From bb56283063b586d311ec7a55c21013d882134ba7 Mon Sep 17 00:00:00 2001 From: Hare Date: Sat, 5 Sep 2026 22:06:59 +0900 Subject: [PATCH 01/39] feat: persist worker submit activation queue --- Cargo.lock | 1 + crates/client/src/client.rs | 7 +- crates/client/src/transport/in_process.rs | 7 +- crates/client/src/transport/unix_socket.rs | 10 +- crates/client/src/transport/websocket.rs | 7 +- crates/protocol/src/lib.rs | 172 +++- crates/protocol/src/typescript.rs | 13 +- .../session-store/src/legacy_session_log.rs | 1 + crates/session-store/src/public_snapshot.rs | 7 +- crates/session-store/src/segment.rs | 1 + crates/session-store/src/segment_log.rs | 11 +- crates/standalone/tests/host.rs | 16 +- crates/tui/src/app.rs | 236 ++--- crates/tui/src/console/mod.rs | 210 ++--- crates/tui/src/ui.rs | 35 +- crates/worker-runtime/src/execution.rs | 37 +- crates/worker-runtime/src/http_server.rs | 12 +- crates/worker-runtime/src/interaction.rs | 30 +- crates/worker-runtime/src/runtime.rs | 112 ++- crates/worker-runtime/src/worker_backend.rs | 274 +++--- crates/worker/Cargo.toml | 1 + crates/worker/examples/worker_protocol.rs | 5 +- crates/worker/src/controller.rs | 516 +++++++---- crates/worker/src/discovery.rs | 27 +- .../src/feature/builtin/worker_observation.rs | 5 +- crates/worker/src/ipc/interceptor.rs | 27 +- crates/worker/src/ipc/notify_buffer.rs | 36 +- crates/worker/src/lib.rs | 10 +- crates/worker/src/segment_log_sink.rs | 1 + crates/worker/src/spawn/comm_tools.rs | 1 + crates/worker/src/spawn/tool.rs | 9 +- crates/worker/src/worker.rs | 857 +++++++++++++++++- crates/worker/tests/compact_events_test.rs | 5 +- crates/worker/tests/controller_test.rs | 292 ++++-- crates/workspace-server/src/hosts.rs | 6 +- docs/development/work-items.md | 2 +- docs/report/test-validity-20260612/pod.md | 2 +- web/workspace/src/lib/generated/protocol.ts | 12 +- .../src/lib/workspace/console/model.test.ts | 1 + .../console/worker-console.ui.test.ts | 22 + .../workers/[workerId]/console/+page.svelte | 111 ++- 41 files changed, 2336 insertions(+), 811 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6038be2..8c6a126e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6650,6 +6650,7 @@ dependencies = [ "serial_test", "session-metrics", "session-store", + "sha2 0.11.0", "tempfile", "thiserror 2.0.18", "ticket", diff --git a/crates/client/src/client.rs b/crates/client/src/client.rs index 517d6a7c..b760ce00 100644 --- a/crates/client/src/client.rs +++ b/crates/client/src/client.rs @@ -120,12 +120,15 @@ mod tests { let mut client = Client::new(socket); client - .send(&Method::run_text("hello")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) .await .expect("send method"); assert!(matches!( decode_method(&client.socket.sent[0]), - Ok(Method::Run { .. }) + Ok(Method::Submit { .. }) )); assert!(matches!( client.next_event().await, diff --git a/crates/client/src/transport/in_process.rs b/crates/client/src/transport/in_process.rs index b1db8e6b..3111808a 100644 --- a/crates/client/src/transport/in_process.rs +++ b/crates/client/src/transport/in_process.rs @@ -89,12 +89,15 @@ mod tests { let mut client = Client::new(socket); client - .send(&Method::run_text("hello")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) .await .expect("send method"); assert!(matches!( peer.next().await.as_deref().map(decode_method), - Some(Ok(Method::Run { .. })) + Some(Ok(Method::Submit { .. })) )); peer.send( diff --git a/crates/client/src/transport/unix_socket.rs b/crates/client/src/transport/unix_socket.rs index 0262bff2..089ed83e 100644 --- a/crates/client/src/transport/unix_socket.rs +++ b/crates/client/src/transport/unix_socket.rs @@ -147,12 +147,18 @@ mod tests { let mut client = Client::new(Socket::connect(&socket_path).await.unwrap()); client - .send(&Method::run_text("hello")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) .await .expect("send method"); let received = server.await.unwrap().expect("method message"); - assert!(matches!(decode_method(&received), Ok(Method::Run { .. }))); + assert!(matches!( + decode_method(&received), + Ok(Method::Submit { .. }) + )); } #[tokio::test] diff --git a/crates/client/src/transport/websocket.rs b/crates/client/src/transport/websocket.rs index e8640573..b4c1ed84 100644 --- a/crates/client/src/transport/websocket.rs +++ b/crates/client/src/transport/websocket.rs @@ -114,7 +114,7 @@ mod tests { assert!(matches!( message, Message::Text(ref text) - if matches!(decode_method(text), Ok(Method::Run { .. })) + if matches!(decode_method(text), Ok(Method::Submit { .. })) )); let event = encode_event(&Event::Status { status: WorkerStatus::Idle, @@ -126,7 +126,10 @@ mod tests { let request = format!("ws://{address}").into_client_request().unwrap(); let mut client = Client::new(Socket::connect(request).await.unwrap()); client - .send(&Method::run_text("hello")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) .await .expect("send method"); assert!(matches!( diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index 9f0ab12c..7171461c 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -11,6 +11,11 @@ use serde::{Deserialize, Serialize}; pub use identity::{WorkerId, WorkerIdParseError}; +/// Allocate an opaque idempotency key for one client Submit request. +pub fn new_submission_request_id() -> String { + uuid::Uuid::now_v7().to_string() +} + fn default_true() -> bool { true } @@ -31,17 +36,22 @@ fn is_false(value: &bool) -> bool { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(tag = "method", content = "params", rename_all = "snake_case")] pub enum Method { - Run { + /// Durably accept typed input for immediate activation or the session FIFO. + /// + /// `submission_request_id` is generated by the authenticated caller and is + /// used only for idempotent retry. Worker allocates the durable + /// `submission_id` returned by [`Event::SubmissionAccepted`]. + Submit { + submission_request_id: String, input: Vec, }, - /// Runtime-internal Run carrying an opaque correlation id that is committed - /// with the resulting UserInput entry. This variant is not serializable on - /// the public Client → Worker protocol. + /// Runtime-internal Submit with the same request identity contract. This + /// variant is not serializable on the public Client → Worker protocol. #[serde(skip)] #[cfg_attr(feature = "typescript", ts(skip))] - RunTracked { + SubmitTracked { + submission_request_id: String, input: Vec, - submission_id: String, }, /// Human-readable text injected into the target Worker's LLM context /// as a non-blocking system message. `auto_run` controls whether an @@ -50,12 +60,25 @@ pub enum Method { /// No side effects beyond LLM context; use `WorkerEvent` for typed /// lifecycle reports. Notify { + notification_request_id: String, message: String, #[serde(default = "default_true", skip_serializing_if = "is_true")] auto_run: bool, }, /// Typed lifecycle report from a child Worker to its direct parent. WorkerEvent(WorkerEvent), + /// Return the authoritative FIFO summary without exposing queued payloads. + ListPendingSubmissions, + /// Remove one queued submission. Running or already activated submissions + /// are immutable and therefore cannot be cancelled here. + CancelPendingSubmission { + submission_id: String, + }, + /// Remove every queued submission while preserving the active run. + ClearPendingSubmissions, + /// Activate the next queued submission while the Worker is idle. This is an + /// explicit recovery operation and never resumes a paused run implicitly. + ContinuePending, Resume, Cancel, /// Stop the in-flight turn and transition to `Paused`. @@ -68,7 +91,7 @@ pub enum Method { /// Request an explicit compaction while the Worker is otherwise idle. /// /// This is a typed control method: clients must not send `compact` as a - /// `Method::Run` user message. + /// `Method::Submit` user message. Compact, /// Ask the Worker to list valid rewind targets from its authoritative session log. ListRewindTargets, @@ -181,7 +204,7 @@ impl WorkerEvent { /// One typed piece of a user submission. /// -/// `Method::Run` and `Event::UserMessage` carry `Vec`. Dumb +/// `Method::Submit` and `Event::UserMessage` carry `Vec`. Dumb /// clients (CLI piping, scripts) only need to produce a single /// `Segment::Text`; richer clients (TUI / GUI) construct typed atoms /// (paste chips, file refs) and @@ -404,12 +427,13 @@ impl Segment { } impl Method { - /// Convenience: a `Run` carrying a single `Segment::Text`. + /// Convenience: a `Submit` carrying a single `Segment::Text`. /// Used by dumb clients, inter-Worker tools, and tests that only have /// a string to forward. - pub fn run_text(s: impl Into) -> Self { - Self::Run { - input: vec![Segment::text(s)], + pub fn submit_text(submission_request_id: impl Into, text: impl Into) -> Self { + Self::Submit { + submission_request_id: submission_request_id.into(), + input: vec![Segment::text(text)], } } } @@ -503,6 +527,37 @@ pub enum ToolResultDisposition { OutcomeUnknown, } +/// Durable acceptance result for one idempotent Submit request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum SubmissionDisposition { + Started, + Queued, +} + +/// Bounded public projection of one pending submission. Payload segments and +/// provenance remain in the session log and are intentionally not exposed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct PendingSubmissionSummary { + pub submission_id: String, + pub accepted_at_ms: u64, + pub segment_count: u32, + pub byte_len: u64, +} + +/// Revisioned session-owned FIFO projection used by snapshots and live events. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct PendingSubmissionsSnapshot { + pub revision: u64, + #[serde(default)] + pub notification_count: u32, + #[serde(default)] + pub submissions: Vec, +} + /// Canonical, storage-independent projection of committed session history. /// /// Worker protocols expose this DTO instead of append-log records. New @@ -511,6 +566,8 @@ pub enum ToolResultDisposition { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] pub struct SessionSnapshot { + #[serde(default)] + pub pending_submissions: PendingSubmissionsSnapshot, pub entries: Vec, } @@ -609,16 +666,27 @@ pub struct SessionToolAttachment { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(tag = "event", content = "data", rename_all = "snake_case")] pub enum Event { - /// A user input message was accepted, persisted as - /// `LogEntry::AnnotatedUserInput`, and is about to start a new turn. - /// Broadcast to every subscribed client so TUI / GUI instances show - /// the same user line that reconnect snapshots would replay from - /// history; clients must not synthesize a separate pending/fake - /// message for accepted runs. - /// - /// Fires exactly once per committed user input, after - /// `InvokeStart { kind: UserSend }` and before the first - /// `TurnStart`. Rejected runs (e.g. `AlreadyRunning`) do not emit. + /// Durable Submit acceptance. A `Started` receipt follows the atomic + /// UserInput commit; a `Queued` receipt follows the durable FIFO checkpoint. + /// Repeating the same request id and exact payload returns the same receipt + /// without appending or activating twice. + SubmissionAccepted { + submission_request_id: String, + submission_id: String, + disposition: SubmissionDisposition, + }, + /// Correlated rejection before durable acceptance. + SubmissionRejected { + submission_request_id: String, + message: String, + }, + /// Revisioned FIFO replacement following enqueue, activation, cancel, or clear. + PendingSubmissionsChanged { + pending: PendingSubmissionsSnapshot, + }, + /// A user input message persisted as `LogEntry::AnnotatedUserInput` and + /// activated for a turn. Broadcast to every subscribed client so TUI / GUI + /// instances show the same user line that reconnect snapshots replay. UserMessage { segments: Vec, }, @@ -641,7 +709,7 @@ pub enum Event { /// /// Marker event for the start of an Invoke range; the range extends /// implicitly until the next `InvokeStart`. Fires for every accepted - /// `Method::Run` (kind=`UserSend`), `Method::Notify` (kind=`Notify`), + /// `Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`), /// `Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other /// IDLE-breaking trigger. Mid-run interrupts (e.g. hook output, /// typed system reminder insertion that doesn't break IDLE) do not @@ -1193,7 +1261,7 @@ pub enum TurnResult { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(rename_all = "snake_case")] pub enum InvokeKind { - /// `Method::Run` — a user submission. + /// `Method::Submit` — a user submission. UserSend, /// `Method::Notify` — free-text notification injected into history. Notify, @@ -1216,7 +1284,7 @@ pub enum RunResult { Finished, Paused, LimitReached, - /// The accepted Method::Run produced no assistant/tool output before + /// The accepted Method::Submit produced no assistant/tool output before /// user interruption, so the Worker rolled the submit-time turn state back /// to its pre-submit snapshot. Clients should treat the Worker as Idle and /// restore the just-submitted input into the editable composer if desired. @@ -1285,26 +1353,30 @@ mod tests { use super::*; #[test] - fn method_run_json_roundtrip() { - let json = r#"{"method":"run","params":{"input":[{"kind":"text","content":"Hello"}]}}"#; + fn method_submit_json_roundtrip_and_run_is_rejected() { + let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#; let method: Method = serde_json::from_str(json).unwrap(); match &method { - Method::Run { input } => { + Method::Submit { input, .. } => { assert_eq!(input.len(), 1); match &input[0] { Segment::Text { content } => assert_eq!(content, "Hello"), other => panic!("expected Text, got {other:?}"), } } - other => panic!("expected Run, got {other:?}"), + other => panic!("expected Submit, got {other:?}"), } let serialized = serde_json::to_string(&method).unwrap(); assert_eq!(serialized, json); + assert!( + serde_json::from_str::(r#"{"method":"run","params":{"input":[]}}"#).is_err() + ); } #[test] - fn method_run_paste_segment_roundtrip() { - let method = Method::Run { + fn method_submit_paste_segment_roundtrip() { + let method = Method::Submit { + submission_request_id: "request-1".to_string(), input: vec![ Segment::text("see "), Segment::Paste { @@ -1318,7 +1390,7 @@ mod tests { let json = serde_json::to_string(&method).unwrap(); let decoded: Method = serde_json::from_str(&json).unwrap(); match decoded { - Method::Run { input } => { + Method::Submit { input, .. } => { assert_eq!(input.len(), 2); match &input[1] { Segment::Paste { @@ -1335,7 +1407,7 @@ mod tests { other => panic!("expected Paste, got {other:?}"), } } - other => panic!("expected Run, got {other:?}"), + other => panic!("expected Submit, got {other:?}"), } } @@ -1389,8 +1461,9 @@ mod tests { } #[test] - fn method_run_flow_segment_roundtrip() { - let method = Method::Run { + fn method_submit_flow_segment_roundtrip() { + let method = Method::Submit { + submission_request_id: "request-1".to_string(), input: vec![ Segment::Flow { selector: "builtin:coder-review".to_string(), @@ -1404,7 +1477,7 @@ mod tests { let decoded = serde_json::from_str::(&json).unwrap(); assert!(matches!( decoded, - Method::Run { input } + Method::Submit { input, .. } if matches!( input.as_slice(), [ @@ -1416,15 +1489,15 @@ mod tests { } #[test] - fn runtime_tracked_run_is_not_public_protocol_json() { - let method = Method::RunTracked { + fn runtime_tracked_submit_is_not_public_protocol_json() { + let method = Method::SubmitTracked { input: vec![Segment::text("private")], - submission_id: "submission-1".to_string(), + submission_request_id: "request-1".to_string(), }; assert!(serde_json::to_string(&method).is_err()); assert!( serde_json::from_str::( - r#"{"method":"run_tracked","input":[],"submission_id":"forged"}"#, + r#"{"method":"submit_tracked","input":[],"submission_id":"forged"}"#, ) .is_err() ); @@ -1442,16 +1515,16 @@ mod tests { } #[test] - fn method_run_with_unknown_segment_decodes() { - let json = r#"{"method":"run","params":{"input":[{"kind":"text","content":"hi"},{"kind":"future_thing","x":1}]}}"#; + fn method_submit_with_unknown_segment_decodes() { + let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"hi"},{"kind":"future_thing","x":1}]}}"#; let method: Method = serde_json::from_str(json).unwrap(); match method { - Method::Run { input } => { + Method::Submit { input, .. } => { assert_eq!(input.len(), 2); assert!(matches!(input[0], Segment::Text { .. })); assert!(matches!(input[1], Segment::Unknown)); } - other => panic!("expected Run, got {other:?}"), + other => panic!("expected Submit, got {other:?}"), } } @@ -1648,11 +1721,11 @@ mod tests { #[test] fn method_notify_json_roundtrip_defaults_to_auto_run() { - let json = r#"{"method":"notify","params":{"message":"turn done"}}"#; + let json = r#"{"method":"notify","params":{"notification_request_id":"notification-1","message":"turn done"}}"#; let method: Method = serde_json::from_str(json).unwrap(); assert!(matches!( method, - Method::Notify { ref message, auto_run: true } if message == "turn done" + Method::Notify { ref message, auto_run: true, .. } if message == "turn done" )); let serialized = serde_json::to_string(&method).unwrap(); assert_eq!(serialized, json); @@ -1660,11 +1733,11 @@ mod tests { #[test] fn method_notify_weak_json_roundtrip_serializes_auto_run_false() { - let json = r#"{"method":"notify","params":{"message":"progress","auto_run":false}}"#; + let json = r#"{"method":"notify","params":{"notification_request_id":"notification-1","message":"progress","auto_run":false}}"#; let method: Method = serde_json::from_str(json).unwrap(); assert!(matches!( method, - Method::Notify { ref message, auto_run: false } if message == "progress" + Method::Notify { ref message, auto_run: false, .. } if message == "progress" )); assert_eq!(serde_json::to_string(&method).unwrap(), json); } @@ -1725,6 +1798,7 @@ mod tests { fn event_snapshot_format() { let event = Event::Snapshot { session: SessionSnapshot { + pending_submissions: PendingSubmissionsSnapshot::default(), entries: vec![SessionSnapshotEntry { entry_id: "entry-1".into(), timestamp: 1, @@ -1776,6 +1850,7 @@ mod tests { let event = Event::Snapshot { session: SessionSnapshot { + pending_submissions: PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: Greeting { @@ -1844,6 +1919,7 @@ mod tests { fn event_segment_rotated_roundtrip() { let event = Event::SegmentRotated { session: SessionSnapshot { + pending_submissions: PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, }; diff --git a/crates/protocol/src/typescript.rs b/crates/protocol/src/typescript.rs index bdac40ca..bf63c8d6 100644 --- a/crates/protocol/src/typescript.rs +++ b/crates/protocol/src/typescript.rs @@ -8,11 +8,11 @@ use crate::{ CompletionKind, ErrorCode, Event, Greeting, InFlightBlock, InFlightSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnapshot, InvokeKind, MemoryWorkerEvent, Method, PasteArtifactAvailability, PasteArtifactMediaType, - PasteArtifactRef, Permission, RewindSummary, RewindTarget, RewindTargetId, RunResult, - ScopeRule, Segment, SessionContentPart, SessionEntryProvenance, SessionMessageRole, - SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment, - ToolResultDisposition, TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, - WorkerStatus, + PasteArtifactRef, PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission, + RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart, + SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry, + SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition, + TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, WorkerStatus, subscription::{ EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame, SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest, @@ -75,6 +75,9 @@ pub fn generated_protocol_types() -> String { push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); diff --git a/crates/session-store/src/legacy_session_log.rs b/crates/session-store/src/legacy_session_log.rs index 40dd0fd5..bb9dd1dd 100644 --- a/crates/session-store/src/legacy_session_log.rs +++ b/crates/session-store/src/legacy_session_log.rs @@ -183,6 +183,7 @@ fn canonicalize_history_entry( item, metadata: legacy_metadata(segment_id, line_index, 0), }, + extensions: Vec::new(), }, } } diff --git a/crates/session-store/src/public_snapshot.rs b/crates/session-store/src/public_snapshot.rs index f3b61754..b1793584 100644 --- a/crates/session-store/src/public_snapshot.rs +++ b/crates/session-store/src/public_snapshot.rs @@ -71,7 +71,7 @@ pub fn project_session_snapshot(session_id: SessionId, log: &[LogEntry]) -> Sess entries.push(history_entry(entry, *ts, data)); } } - LogEntry::AnnotatedSystemItem { ts, entry } => entries.push(system_entry( + LogEntry::AnnotatedSystemItem { ts, entry, .. } => entries.push(system_entry( &entry.item, entry.metadata.entry_id.0.clone(), *ts, @@ -100,7 +100,10 @@ pub fn project_session_snapshot(session_id: SessionId, log: &[LogEntry]) -> Sess } } - SessionSnapshot { entries } + SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries, + } } fn extend_history( diff --git a/crates/session-store/src/segment.rs b/crates/session-store/src/segment.rs index 163b2317..75d997c2 100644 --- a/crates/session-store/src/segment.rs +++ b/crates/session-store/src/segment.rs @@ -287,6 +287,7 @@ pub fn append_system_item( LogEntry::AnnotatedSystemItem { ts: segment_log::now_millis(), entry, + extensions: Vec::new(), }, ) } diff --git a/crates/session-store/src/segment_log.rs b/crates/session-store/src/segment_log.rs index 9daf85c2..ee509cbb 100644 --- a/crates/session-store/src/segment_log.rs +++ b/crates/session-store/src/segment_log.rs @@ -112,6 +112,8 @@ pub enum LogEntry { AnnotatedSystemItem { ts: u64, entry: LoggedSystemHistoryEntry, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + extensions: Vec, }, /// Turn boundary. Records the turn count after increment. @@ -312,12 +314,19 @@ pub fn collect_state(entries: &[LogEntry]) -> RestoredState { state.annotated_history.push(entry.clone()); state.history.push(Item::from(entry.item.clone())); } - LogEntry::AnnotatedSystemItem { entry, .. } => { + LogEntry::AnnotatedSystemItem { + entry, extensions, .. + } => { state.annotated_history.push(LoggedHistoryEntry { item: LoggedItem::from(entry.item.to_history_item()), metadata: entry.metadata.clone(), }); state.history.push(entry.item.to_history_item()); + state.extensions.extend( + extensions + .iter() + .map(|extension| (extension.domain.clone(), extension.payload.clone())), + ); } LogEntry::TurnEnd { turn_count, .. } => { if let Some(active_turn_count) = &mut state.active_run_turn_count { diff --git a/crates/standalone/tests/host.rs b/crates/standalone/tests/host.rs index 510cc68c..45c8f620 100644 --- a/crates/standalone/tests/host.rs +++ b/crates/standalone/tests/host.rs @@ -99,7 +99,10 @@ async fn in_process_host_runs_text_and_read_tool_then_shuts_down() { let mut protocol_client = host.connect(); protocol_client - .send(&Method::run_text("read the probe")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "read the probe", + )) .await .expect("submit input"); @@ -336,11 +339,15 @@ async fn standalone_restore_preserves_history_tasks_notifications_and_cwd_scope( let worker_id = host.worker_id(); let mut protocol_client = host.connect(); protocol_client - .send(&Method::run_text("first request")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "first request", + )) .await?; wait_for_run_end(&mut protocol_client).await?; protocol_client .send(&Method::Notify { + notification_request_id: protocol::new_submission_request_id(), message: "persisted notification".to_string(), auto_run: true, }) @@ -394,7 +401,10 @@ async fn standalone_restore_preserves_history_tasks_notifications_and_cwd_scope( assert!(snapshot.contains("persisted notification"), "{snapshot}"); protocol_client - .send(&Method::run_text("continue after restore")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "continue after restore", + )) .await?; wait_for_run_end(&mut protocol_client).await?; let request = second_inspection diff --git a/crates/tui/src/app.rs b/crates/tui/src/app.rs index b1f4fd04..2fecd888 100644 --- a/crates/tui/src/app.rs +++ b/crates/tui/src/app.rs @@ -102,23 +102,6 @@ struct RollbackSubmitState { turn_before: usize, } -#[derive(Clone)] -pub struct QueuedInput { - segments: Vec, - preview: String, -} - -impl QueuedInput { - fn new(segments: Vec) -> Self { - let preview = Segment::flatten_to_text(&segments); - Self { segments, preview } - } - - pub fn preview(&self) -> &str { - &self.preview - } -} - struct ComposerInputHistory { entries: VecDeque>, browse: Option, @@ -272,7 +255,7 @@ pub struct App { /// Current transient actionbar notice. Notices are local UI state only: /// they are never appended to transcript/session history or LLM context. actionbar_notice: Option, - /// Normal composer input that is submitted as `Method::Run`. + /// Normal composer input that is submitted as `Method::Submit`. pub input: InputBuffer, /// Separate command-line input. It is never submitted as a user message. pub command_input: InputBuffer, @@ -333,9 +316,8 @@ pub struct App { /// Top entry index of the task pane's visible window. Clamped on /// render so it never points past the end of the list. pub task_pane_scroll: usize, - /// TUI-local FIFO of user inputs submitted while the Worker is already running. - /// Entries have not been sent to the Worker yet, so they remain editable/cancellable locally. - queued_inputs: VecDeque, + /// Authoritative WorkerSession FIFO summary received from snapshot/live events. + pending_submissions: protocol::PendingSubmissionsSnapshot, /// TUI-local readline-style composer input history. This is intentionally /// client-side only: recalled entries are plain drafts until submitted again. input_history: ComposerInputHistory, @@ -395,7 +377,7 @@ impl App { text_selection: TextSelectionState::default(), task_pane_open: false, task_pane_scroll: 0, - queued_inputs: VecDeque::new(), + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), input_history: ComposerInputHistory::new(), input_history_store: None, pending_submit_rollback: None, @@ -768,18 +750,12 @@ impl App { return None; } self.record_input_history(segments.clone()); - if self.running { - self.queued_inputs.push_back(QueuedInput::new(segments)); - self.input.clear(); - self.completion = None; - return None; - } self.input.clear(); Some(self.method_for_run(segments)) } pub fn restore_unsent_run(&mut self, method: &Method) { - let Method::Run { input } = method else { + let Method::Submit { input, .. } = method else { return; }; self.pending_submit_rollback = None; @@ -787,8 +763,9 @@ impl App { self.input.replace_with_segments(input); self.completion = None; } else { - self.queued_inputs - .push_front(QueuedInput::new(input.clone())); + self.push_error( + "Submit transport failed; current Composer was preserved and the unsent input was not queued.", + ); } } @@ -804,7 +781,10 @@ impl App { block_start: self.blocks.len(), turn_before: self.turn_index, }); - Method::Run { input: segments } + Method::Submit { + submission_request_id: protocol::new_submission_request_id(), + input: segments, + } } fn record_input_history(&mut self, segments: Vec) { @@ -825,7 +805,7 @@ impl App { } pub fn queued_input_count(&self) -> usize { - self.queued_inputs.len() + self.pending_submissions.submissions.len() } #[cfg(test)] @@ -911,35 +891,10 @@ impl App { } pub fn next_queued_input_preview(&self) -> Option<&str> { - self.queued_inputs.front().map(QueuedInput::preview) - } - - pub fn clear_queued_inputs(&mut self) -> usize { - let cleared = self.queued_inputs.len(); - self.queued_inputs.clear(); - cleared - } - - pub fn restore_next_queued_input_to_composer(&mut self) -> bool { - if self.queued_inputs.is_empty() { - return false; - } - if !self.input.is_empty() { - self.push_error("Composer is not empty; clear it before editing queued input."); - return false; - } - let Some(queued) = self.queued_inputs.pop_front() else { - return false; - }; - self.input_history.cancel_browse(); - self.input.replace_with_segments(&queued.segments); - self.completion = None; - true - } - - fn pop_next_queued_run(&mut self) -> Option { - let queued = self.queued_inputs.pop_front()?; - Some(self.method_for_run(queued.segments)) + self.pending_submissions + .submissions + .first() + .map(|submission| submission.submission_id.as_str()) } pub fn clear_actionbar_notice(&mut self) { @@ -1123,6 +1078,11 @@ impl App { } match event { + Event::SubmissionAccepted { .. } => {} + Event::SubmissionRejected { message, .. } => self.push_error(message), + Event::PendingSubmissionsChanged { pending } => { + self.pending_submissions = pending; + } Event::UserMessage { segments } => { self.turn_index += 1; self.blocks.push(Block::TurnHeader { @@ -1372,9 +1332,6 @@ impl App { WorkerStatus::Idle } }); - if matches!(result, RunResult::Finished | RunResult::LimitReached) { - return self.pop_next_queued_run(); - } } } Event::CompactStart { .. } => { @@ -1449,6 +1406,7 @@ impl App { internal_workers, } => { self.rewind_refresh_fence = false; + self.pending_submissions = session.pending_submissions.clone(); self.restore_snapshot(&session, greeting, in_flight); self.replace_internal_worker_snapshots(internal_workers); self.set_worker_status(status); @@ -2681,7 +2639,10 @@ mod rewind_refresh_tests { }); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::text("selected rewind input")], summary: summary(3), }); @@ -2700,7 +2661,10 @@ mod rewind_refresh_tests { }); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::text("rewound input")], summary: summary(1), }); @@ -2743,7 +2707,10 @@ mod rewind_refresh_tests { }); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::text("rewound input")], summary: summary(2), }); @@ -2877,7 +2844,7 @@ mod composer_history_persistence_tests { path: "src/lib.rs".into(), }, ]); - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); let mut reloaded = App::new_with_input_history_store("test".into(), store); assert!(reloaded.browse_input_history_older()); @@ -2958,7 +2925,7 @@ mod composer_history_persistence_tests { app.insert_char(c); } match app.submit_input() { - Some(Method::Run { input }) => input, + Some(Method::Submit { input, .. }) => input, other => panic!("expected Run, got {other:?}"), } } @@ -3424,72 +3391,43 @@ mod completion_flow_tests { } #[test] - fn running_submit_is_queued_locally_and_clears_composer() { + fn running_submit_is_sent_to_the_worker_and_not_queued_locally() { let mut app = App::new("test".into()); app.set_worker_status(WorkerStatus::Running); insert_text(&mut app, "queued turn"); - assert!(app.submit_input().is_none()); + let method = app.submit_input(); - assert_eq!(app.queued_input_count(), 1); - assert_eq!(app.next_queued_input_preview(), Some("queued turn")); + assert!(matches!(method, Some(Method::Submit { .. }))); + assert_eq!(app.queued_input_count(), 0); assert_eq!(input_text(&app), ""); } #[test] - fn finished_run_auto_sends_next_queued_input() { + fn pending_submission_projection_is_worker_authoritative() { let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - insert_text(&mut app, "next turn"); - assert!(app.submit_input().is_none()); - - let method = app.handle_worker_event(Event::RunEnd { - result: RunResult::Finished, + app.handle_worker_event(Event::PendingSubmissionsChanged { + pending: protocol::PendingSubmissionsSnapshot { + revision: 3, + notification_count: 0, + submissions: vec![protocol::PendingSubmissionSummary { + submission_id: "submission-1".into(), + accepted_at_ms: 7, + segment_count: 2, + byte_len: 42, + }], + }, }); - match method { - Some(Method::Run { input }) => { - assert_eq!(Segment::flatten_to_text(&input), "next turn"); - } - other => panic!("expected queued Run, got {other:?}"), - } - assert_eq!(app.queued_input_count(), 0); - } - - #[test] - fn limit_reached_run_auto_sends_next_queued_input() { - let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - insert_text(&mut app, "next after limit"); - assert!(app.submit_input().is_none()); - - let method = app.handle_worker_event(Event::RunEnd { - result: RunResult::LimitReached, - }); - - match method { - Some(Method::Run { input }) => { - assert_eq!(Segment::flatten_to_text(&input), "next after limit"); - } - other => panic!("expected queued Run, got {other:?}"), - } - assert_eq!(app.queued_input_count(), 0); - } - - #[test] - fn paused_and_rolled_back_run_do_not_auto_send_queue() { - for result in [RunResult::Paused, RunResult::RolledBack] { - let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - insert_text(&mut app, "held turn"); - assert!(app.submit_input().is_none()); - - let method = app.handle_worker_event(Event::RunEnd { result }); - - assert!(method.is_none()); - assert_eq!(app.queued_input_count(), 1); - assert_eq!(app.next_queued_input_preview(), Some("held turn")); - } + assert_eq!(app.queued_input_count(), 1); + assert_eq!(app.next_queued_input_preview(), Some("submission-1")); + assert!( + app.handle_worker_event(Event::RunEnd { + result: RunResult::Finished, + }) + .is_none() + ); + assert_eq!(app.queued_input_count(), 1); } #[test] @@ -3501,24 +3439,6 @@ mod completion_flow_tests { assert_eq!(app.queued_input_count(), 0); } - #[test] - fn queued_input_can_be_restored_to_composer_or_cleared() { - let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - insert_text(&mut app, "edit me"); - assert!(app.submit_input().is_none()); - - assert!(app.restore_next_queued_input_to_composer()); - assert_eq!(app.queued_input_count(), 0); - assert_eq!(input_text(&app), "edit me"); - - app.input.clear(); - insert_text(&mut app, "clear me"); - assert!(app.submit_input().is_none()); - assert_eq!(app.clear_queued_inputs(), 1); - assert_eq!(app.queued_input_count(), 0); - } - fn insert_text(app: &mut App, text: &str) { for c in text.chars() { app.insert_char(c); @@ -3530,7 +3450,7 @@ mod completion_flow_tests { app.insert_char(c); } match app.submit_input() { - Some(Method::Run { input }) => input, + Some(Method::Submit { input, .. }) => input, other => panic!("expected Run, got {other:?}"), } } @@ -3675,6 +3595,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, status: WorkerStatus::Running, @@ -3783,6 +3704,7 @@ mod completion_flow_tests { revision, status: WorkerStatus::Idle, session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, in_flight: protocol::InFlightSnapshot::default(), @@ -4000,6 +3922,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, status: WorkerStatus::Idle, @@ -4051,6 +3974,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, status: WorkerStatus::Idle, @@ -4064,6 +3988,7 @@ mod completion_flow_tests { }, revision: 4, session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, status: WorkerStatus::Running, @@ -4222,6 +4147,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting, @@ -4437,23 +4363,23 @@ mod completion_flow_tests { } #[test] - fn input_history_records_queued_inputs_and_suppresses_consecutive_duplicates() { + fn input_history_records_running_submits_and_suppresses_consecutive_duplicates() { let mut app = App::new("test".into()); app.running = true; for c in "repeat".chars() { app.insert_char(c); } - assert!(app.submit_input().is_none()); + assert!(app.submit_input().is_some()); assert_eq!(app.input_history_len(), 1); - assert_eq!(app.queued_input_count(), 1); + assert_eq!(app.queued_input_count(), 0); for c in "repeat".chars() { app.insert_char(c); } - assert!(app.submit_input().is_none()); + assert!(app.submit_input().is_some()); assert_eq!(app.input_history_len(), 1); - assert_eq!(app.queued_input_count(), 2); + assert_eq!(app.queued_input_count(), 0); app.insert_char(' '); assert!(app.submit_input().is_none()); @@ -4481,7 +4407,7 @@ mod completion_flow_tests { }, ]; app.input.replace_with_segments(&original); - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); assert!(app.browse_input_history_older()); assert_eq!(app.input.submit_segments(), original); @@ -4493,7 +4419,7 @@ mod completion_flow_tests { for c in "sent".chars() { app.insert_char(c); } - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); for c in "draft".chars() { app.insert_char(c); @@ -4511,7 +4437,7 @@ mod completion_flow_tests { for c in "sent".chars() { app.insert_char(c); } - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); assert!(app.browse_input_history_older()); assert!(app.input_history_is_browsing()); @@ -4528,17 +4454,19 @@ mod completion_flow_tests { for c in "first".chars() { app.insert_char(c); } - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); for c in "second".chars() { app.insert_char(c); } - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); assert!(app.browse_input_history_older()); assert!(app.browse_input_history_older()); let method = app.submit_input(); match method { - Some(Method::Run { input }) => assert_eq!(Segment::flatten_to_text(&input), "first"), + Some(Method::Submit { input, .. }) => { + assert_eq!(Segment::flatten_to_text(&input), "first") + } other => panic!("expected recalled run, got {other:?}"), } assert_eq!(app.input_history_len(), 3); diff --git a/crates/tui/src/console/mod.rs b/crates/tui/src/console/mod.rs index 0106b403..3f0d6ddd 100644 --- a/crates/tui/src/console/mod.rs +++ b/crates/tui/src/console/mod.rs @@ -270,8 +270,8 @@ impl ConsoleConnection { async fn send(&mut self, method: &Method) -> Result<(), Box> { let mut prepared = method.clone(); let carries_attachments = - matches!(prepared, Method::Run { .. }) && !self.pending_attachments.is_empty(); - if let Method::Run { input } = &mut prepared { + matches!(prepared, Method::Submit { .. }) && !self.pending_attachments.is_empty(); + if let Method::Submit { input, .. } = &mut prepared { input.extend( self.pending_attachments .iter() @@ -569,6 +569,7 @@ async fn run_e2e_rewind_fixture( app.connected = true; app.handle_worker_event(Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, status: WorkerStatus::Idle, @@ -697,6 +698,7 @@ async fn run_e2e_rewind_fixture( if submitted_at.elapsed() >= apply_delay { app.handle_worker_event(Event::RewindApplied { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, input: vec![Segment::text("rewind-live-refresh")], @@ -916,7 +918,7 @@ async fn run_loop( } fn attachment_command_path(method: &Method) -> Option { - let Method::Run { input } = method else { + let Method::Submit { input, .. } = method else { return None; }; let [Segment::Text { content }] = input.as_slice() else { @@ -927,7 +929,7 @@ fn attachment_command_path(method: &Method) -> Option { } fn is_clear_attachments_command(method: &Method) -> bool { - let Method::Run { input } = method else { + let Method::Submit { input, .. } = method else { return false; }; matches!( @@ -941,7 +943,7 @@ async fn send_console_method( client: &mut ConsoleConnection, method: &Method, ) -> Result<(), Box> { - if matches!(method, Method::Run { .. }) && client.has_active_uploads() { + if matches!(method, Method::Submit { .. }) && client.has_active_uploads() { app.restore_unsent_run(method); app.flash_actionbar_notice( "Attachment upload is still in progress; wait or use /clear-attachments.", @@ -953,7 +955,7 @@ async fn send_console_method( } let sends_attachments = - matches!(method, Method::Run { .. }) && !client.pending_attachments.is_empty(); + matches!(method, Method::Submit { .. }) && !client.pending_attachments.is_empty(); if let Err(error) = client.send(method).await { if sends_attachments { app.restore_unsent_run(method); @@ -1151,15 +1153,10 @@ fn handle_key(app: &mut App, key: KeyEvent) -> Option { KeyCode::Char(c) if c.eq_ignore_ascii_case(&'q') && alt && !ctrl && !app.is_command_mode() => { - if app.restore_next_queued_input_to_composer() { - Some(app.refresh_completion()) - } else { - Some(None) - } + Some(Some(Method::ContinuePending)) } KeyCode::Char(c) if c.eq_ignore_ascii_case(&'c') && alt && !ctrl => { - app.clear_queued_inputs(); - Some(None) + Some(Some(Method::ClearPendingSubmissions)) } KeyCode::Char('c') if ctrl => Some(handle_pause_or_quit(app)), KeyCode::Char('x') if ctrl => Some(handle_cancel_or_shutdown(app)), @@ -1427,7 +1424,6 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option { WorkerStatus::Running | WorkerStatus::Paused ) { app.shutdown_confirm = None; - app.clear_queued_inputs(); return Some(Method::Cancel); } if let Some(pressed_at) = app.shutdown_confirm @@ -1450,7 +1446,6 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option { /// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running). fn handle_pause_or_quit(app: &mut App) -> Option { if app.worker_status == WorkerStatus::Running { - app.clear_queued_inputs(); return Some(Method::Pause); } if let Some(t) = app.quit_confirm @@ -1476,8 +1471,8 @@ mod tests { use crate::text_selection::{HistoryViewport, SelectionRow}; use async_trait::async_trait; use protocol::{ - Event, RewindTarget, RewindTargetId, RunResult, Segment, UploadedFileAvailability, - UploadedFileRef, WorkerStatus, + Event, RewindTarget, RewindTargetId, Segment, UploadedFileAvailability, UploadedFileRef, + WorkerStatus, }; #[test] @@ -1490,7 +1485,8 @@ mod tests { #[test] fn client_local_attachment_commands_are_typed_and_do_not_send_the_path() { - let attach = Method::Run { + let attach = Method::Submit { + submission_request_id: protocol::new_submission_request_id(), input: vec![Segment::text("/attach /tmp/report.md")], }; assert_eq!( @@ -1499,7 +1495,8 @@ mod tests { ); assert!(!is_clear_attachments_command(&attach)); - let clear = Method::Run { + let clear = Method::Submit { + submission_request_id: protocol::new_submission_request_id(), input: vec![Segment::text("/clear-attachments")], }; assert!(is_clear_attachments_command(&clear)); @@ -1605,7 +1602,7 @@ mod tests { } #[tokio::test] - async fn queued_attachment_send_failure_restores_draft_without_exiting_console() { + async fn running_attachment_submit_failure_restores_draft_without_exiting_console() { let file = UploadedFileRef { artifact_id: "artifact-queued".into(), file_name: "queued.txt".into(), @@ -1631,13 +1628,10 @@ mod tests { let mut app = App::new("worker".into()); app.set_worker_status(WorkerStatus::Running); app.input.insert_str("queued inspect"); - assert!(app.submit_input().is_none()); - let method = app - .handle_worker_event(Event::RunEnd { - result: RunResult::Finished, - }) - .expect("queued run must be released"); + .submit_input() + .expect("running Submit is sent immediately"); + send_console_method(&mut app, &mut connection, &method) .await .unwrap(); @@ -1960,7 +1954,7 @@ mod tests { } #[test] - fn running_enter_queues_instead_of_sending_run() { + fn running_enter_sends_submit_to_worker() { let mut app = App::new("agent".to_string()); app.set_worker_status(WorkerStatus::Running); for c in "queued".chars() { @@ -1973,102 +1967,80 @@ mod tests { ); } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); + assert!(matches!( + handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)), + Some(Method::Submit { .. }) + )); - assert_eq!(app.queued_input_count(), 1); - assert_eq!(app.next_queued_input_preview(), Some("queued")); + assert_eq!(app.queued_input_count(), 0); assert_eq!(input_text(&app), ""); } #[test] - fn queued_input_keybindings_restore_and_clear() { - let mut app = App::new("agent".to_string()); - app.set_worker_status(WorkerStatus::Running); - for c in "edit queued".chars() { - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE) - ) - .is_none() - ); - } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); + fn pending_queue_shortcuts_send_worker_operations() { + let mut app = App::new("test".into()); + app.handle_worker_event(Event::PendingSubmissionsChanged { + pending: protocol::PendingSubmissionsSnapshot { + revision: 2, + notification_count: 0, + submissions: vec![protocol::PendingSubmissionSummary { + submission_id: "submission-1".into(), + accepted_at_ms: 1, + segment_count: 1, + byte_len: 6, + }], + }, + }); - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char('q'), KeyModifiers::ALT) - ) - .is_none() + let continue_next = handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('q'), KeyModifiers::ALT), ); - assert_eq!(app.queued_input_count(), 0); - assert_eq!(input_text(&app), "edit queued"); - - app.input.clear(); - for c in "clear queued".chars() { - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE) - ) - .is_none() - ); - } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); + assert!(matches!(continue_next, Some(Method::ContinuePending))); assert_eq!(app.queued_input_count(), 1); - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char('c'), KeyModifiers::ALT) - ) - .is_none() + let clear = handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('c'), KeyModifiers::ALT), ); - assert_eq!(app.queued_input_count(), 0); + assert!(matches!(clear, Some(Method::ClearPendingSubmissions))); + assert_eq!(app.queued_input_count(), 1); } #[test] - fn pause_and_cancel_clear_queued_input() { - let mut app = App::new("agent".to_string()); + fn pause_and_cancel_preserve_authoritative_pending_queue() { + let mut app = App::new("test".into()); + app.handle_worker_event(Event::PendingSubmissionsChanged { + pending: protocol::PendingSubmissionsSnapshot { + revision: 2, + notification_count: 0, + submissions: vec![protocol::PendingSubmissionSummary { + submission_id: "submission-1".into(), + accepted_at_ms: 1, + segment_count: 1, + byte_len: 6, + }], + }, + }); app.set_worker_status(WorkerStatus::Running); - for c in "queued".chars() { - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE) - ) - .is_none() - ); - } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); + assert!(matches!( + handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL), + ), + Some(Method::Pause) + )); assert_eq!(app.queued_input_count(), 1); - let pause = handle_key( - &mut app, - KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL), - ); - assert!(matches!(pause, Some(Method::Pause))); - assert_eq!(app.queued_input_count(), 0); - - for c in "queued again".chars() { - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE) - ) - .is_none() - ); - } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); + app.set_worker_status(WorkerStatus::Running); + assert!(matches!( + handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), + ), + Some(Method::Cancel) + )); assert_eq!(app.queued_input_count(), 1); - - let cancel = handle_key( - &mut app, - KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), - ); - assert!(matches!(cancel, Some(Method::Cancel))); - assert_eq!(app.queued_input_count(), 0); } #[test] @@ -2535,13 +2507,19 @@ mod tests { let mut app = App::new("agent".to_string()); app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, status: WorkerStatus::Idle, in_flight: Default::default(), internal_workers: Vec::new(), }); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::Text { content: "retry this".into(), }], @@ -2562,7 +2540,10 @@ mod tests { let mut app = App::new("agent".to_string()); app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, status: WorkerStatus::Idle, in_flight: Default::default(), internal_workers: Vec::new(), @@ -2570,7 +2551,10 @@ mod tests { type_keys(&mut app, "draft"); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::Text { content: "retry this".into(), }], @@ -2918,12 +2902,12 @@ mod tests { type_keys(&mut app, "first"); assert!(matches!( handle_key(&mut app, key(KeyCode::Enter)), - Some(Method::Run { .. }) + Some(Method::Submit { .. }) )); type_keys(&mut app, "second"); assert!(matches!( handle_key(&mut app, key(KeyCode::Enter)), - Some(Method::Run { .. }) + Some(Method::Submit { .. }) )); assert_eq!(input_text(&app), ""); @@ -2954,7 +2938,7 @@ mod tests { type_keys(&mut app, "sent"); assert!(matches!( handle_key(&mut app, key(KeyCode::Enter)), - Some(Method::Run { .. }) + Some(Method::Submit { .. }) )); type_keys(&mut app, "draft\nbody"); app.move_cursor_start(); diff --git a/crates/tui/src/ui.rs b/crates/tui/src/ui.rs index 35b4fcbc..577fafe9 100644 --- a/crates/tui/src/ui.rs +++ b/crates/tui/src/ui.rs @@ -1880,7 +1880,7 @@ fn actionbar_left_item(app: &App, now: Instant) -> Option<(String, Style)> { } if app.queued_input_count() > 0 { return Some(( - "Alt-q edit queued Alt-c clear queued".to_string(), + "Alt-q continue queued Alt-c clear queued".to_string(), Style::default().fg(Color::DarkGray), )); } @@ -2136,9 +2136,24 @@ mod tests { use super::*; use crate::app::{ActionbarNoticeLevel, ActionbarNoticeSource, App}; use crate::block::{ToolCallBlock, ToolCallState}; - use protocol::WorkerStatus; + use protocol::Event; use std::time::{Duration, Instant}; + fn set_pending_submission(app: &mut App, id: &str) { + app.handle_worker_event(Event::PendingSubmissionsChanged { + pending: protocol::PendingSubmissionsSnapshot { + revision: 1, + notification_count: 0, + submissions: vec![protocol::PendingSubmissionSummary { + submission_id: id.into(), + accepted_at_ms: 1, + segment_count: 1, + byte_len: 1, + }], + }, + }); + } + #[test] fn run_status_line_matches_console_metrics_and_spinner_frame() { let now = Instant::now(); @@ -2251,15 +2266,11 @@ mod tests { #[test] fn queue_status_text_includes_count_and_preview() { let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - for c in "queued preview".chars() { - app.insert_char(c); - } - assert!(app.submit_input().is_none()); + set_pending_submission(&mut app, "submission-1"); assert_eq!( queue_status_text(&app), - Some("queued: 1 — queued preview".to_string()) + Some("queued: 1 — submission-1".to_string()) ); } @@ -2289,14 +2300,10 @@ mod tests { Some("Worker keeps running. Press Ctrl-C again to exit TUI.".into()) ); - app.set_worker_status(WorkerStatus::Running); - for c in "queued turn".chars() { - app.insert_char(c); - } - assert!(app.submit_input().is_none()); + set_pending_submission(&mut app, "submission-1"); assert_eq!( actionbar_left_item(&app, now).map(|(text, _)| text), - Some("Alt-q edit queued Alt-c clear queued".into()) + Some("Alt-q continue queued Alt-c clear queued".into()) ); app.enter_command_mode(); diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index 47660977..c85e9f29 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -41,14 +41,12 @@ pub enum WorkerExecutionOperation { Cancel, } -/// Evidence that a user input reached the durable Worker session boundary. -/// -/// This is intentionally distinct from accepting a method on the Worker's -/// in-memory channel. For Flow submissions, the committed UserInput entry also -/// carries the initial Flow runtime-state extension. +/// Evidence that a Submit request reached the durable Worker session boundary. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct WorkerInputCommitAck { +pub struct WorkerSubmissionAck { + pub submission_request_id: String, pub submission_id: String, + pub disposition: protocol::SubmissionDisposition, } /// Typed execution result class. Results are transient operation outcomes and @@ -61,7 +59,7 @@ pub struct WorkerExecutionResult { #[serde(default, skip_serializing_if = "Option::is_none")] pub message: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub input_commit: Option, + pub submission: Option, } /// Backend result class for a Worker execution operation. @@ -85,22 +83,26 @@ impl WorkerExecutionResult { outcome: WorkerExecutionOutcome::Accepted, run_state, message: None, - input_commit: None, + submission: None, } } - pub fn accepted_input_committed( + pub fn accepted_submission( operation: WorkerExecutionOperation, run_state: WorkerExecutionRunState, + submission_request_id: impl Into, submission_id: impl Into, + disposition: protocol::SubmissionDisposition, ) -> Self { Self { operation, outcome: WorkerExecutionOutcome::Accepted, run_state, message: None, - input_commit: Some(WorkerInputCommitAck { + submission: Some(WorkerSubmissionAck { + submission_request_id: submission_request_id.into(), submission_id: submission_id.into(), + disposition, }), } } @@ -111,7 +113,7 @@ impl WorkerExecutionResult { outcome: WorkerExecutionOutcome::Busy, run_state: WorkerExecutionRunState::Busy, message: Some(message.into()), - input_commit: None, + submission: None, } } @@ -121,7 +123,7 @@ impl WorkerExecutionResult { outcome: WorkerExecutionOutcome::Rejected, run_state: WorkerExecutionRunState::Stopped, message: Some(message.into()), - input_commit: None, + submission: None, } } @@ -131,7 +133,7 @@ impl WorkerExecutionResult { outcome: WorkerExecutionOutcome::Errored, run_state: WorkerExecutionRunState::Errored, message: Some(message.into()), - input_commit: None, + submission: None, } } @@ -141,7 +143,7 @@ impl WorkerExecutionResult { outcome: WorkerExecutionOutcome::Unsupported, run_state: WorkerExecutionRunState::Stopped, message: Some(message.into()), - input_commit: None, + submission: None, } } @@ -618,14 +620,17 @@ mod tests { use super::*; #[test] - fn input_commit_ack_survives_json_round_trip() { - let result = WorkerExecutionResult::accepted_input_committed( + fn submission_ack_survives_json_round_trip() { + let result = WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, WorkerExecutionRunState::Busy, + "request-1", "submission-1", + protocol::SubmissionDisposition::Started, ); let json = serde_json::to_string(&result).unwrap(); + assert!(json.contains("\"submission_request_id\":\"request-1\"")); assert!(json.contains("\"submission_id\":\"submission-1\"")); assert_eq!( serde_json::from_str::(&json).unwrap(), diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 2217e973..575df269 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -2735,11 +2735,13 @@ mod tests { _handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - if let Some(submission_id) = input.submission_id { - WorkerExecutionResult::accepted_input_committed( + if let Some(submission_id) = input.submission_request_id { + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, WorkerExecutionRunState::Idle, + submission_id.clone(), submission_id, + protocol::SubmissionDisposition::Started, ) } else { WorkerExecutionResult::accepted( @@ -3059,11 +3061,13 @@ mod ws_tests { _handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - if let Some(submission_id) = input.submission_id { - WorkerExecutionResult::accepted_input_committed( + if let Some(submission_id) = input.submission_request_id { + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, WorkerExecutionRunState::Idle, + submission_id.clone(), submission_id, + protocol::SubmissionDisposition::Started, ) } else { WorkerExecutionResult::accepted( diff --git a/crates/worker-runtime/src/interaction.rs b/crates/worker-runtime/src/interaction.rs index d76c675d..52e0168e 100644 --- a/crates/worker-runtime/src/interaction.rs +++ b/crates/worker-runtime/src/interaction.rs @@ -25,10 +25,10 @@ impl WorkerInputKind { pub struct WorkerInput { pub kind: WorkerInputKind, pub content: String, - /// Runtime-generated correlation id. This is never accepted from public - /// JSON input and is consumed only by the execution backend. - #[serde(skip)] - pub submission_id: Option, + /// Authenticated client-generated idempotency key. Runtime generates one + /// only for trusted internal callers that omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub submission_request_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub segments: Option>, } @@ -38,7 +38,7 @@ impl WorkerInput { Self { kind: WorkerInputKind::User, content: content.into(), - submission_id: None, + submission_request_id: None, segments: None, } } @@ -47,7 +47,7 @@ impl WorkerInput { Self { kind: WorkerInputKind::Notify, content: content.into(), - submission_id: None, + submission_request_id: None, segments: None, } } @@ -57,6 +57,21 @@ impl WorkerInput { mod tests { use super::WorkerInput; + #[test] + fn submission_request_id_round_trips_for_authenticated_client_retry() { + let input: WorkerInput = serde_json::from_value(serde_json::json!({ + "kind": "user", + "content": "message", + "submission_request_id": "request-1" + })) + .unwrap(); + assert_eq!(input.submission_request_id.as_deref(), Some("request-1")); + assert_eq!( + serde_json::to_value(input).unwrap()["submission_request_id"], + "request-1" + ); + } + #[test] fn notify_is_an_operation_and_legacy_system_kind_is_rejected() { assert_eq!( @@ -78,4 +93,7 @@ mod tests { pub struct WorkerInteractionAck { pub worker_ref: WorkerRef, pub status: WorkerStatus, + /// Present for User Submit and absent for non-Submit interactions. + #[serde(skip_serializing_if = "Option::is_none")] + pub submission: Option, } diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index fd5a0b16..9701aae1 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -748,8 +748,12 @@ impl Runtime { let state = self.lock()?; state.worker(&worker_ref)?.request.initial_input.clone() } { - let expected_submission_id = Uuid::now_v7().to_string(); - initial_input.submission_id = Some(expected_submission_id.clone()); + let expected_submission_id = initial_input + .submission_request_id + .clone() + .filter(|request_id| !request_id.trim().is_empty()) + .unwrap_or_else(|| Uuid::now_v7().to_string()); + initial_input.submission_request_id = Some(expected_submission_id.clone()); let dispatch_result = backend.dispatch_input(&handle, initial_input.clone()); if !dispatch_result.is_accepted() { let _ = backend.stop_worker(&handle); @@ -763,9 +767,9 @@ impl Runtime { }); } let has_commit_ack = dispatch_result - .input_commit + .submission .as_ref() - .is_some_and(|ack| ack.submission_id == expected_submission_id); + .is_some_and(|ack| ack.submission_request_id == expected_submission_id); if !has_commit_ack { let _ = backend.stop_worker(&handle); self.rollback_failed_create(&worker_ref)?; @@ -1146,13 +1150,18 @@ impl Runtime { mut input: WorkerInput, ) -> Result { validate_worker_input(&input)?; - let expected_submission_id = if input.kind == WorkerInputKind::User { - let submission_id = Uuid::now_v7().to_string(); - input.submission_id = Some(submission_id.clone()); - Some(submission_id) - } else { - None - }; + let expected_submission_id = + if matches!(input.kind, WorkerInputKind::User | WorkerInputKind::Notify) { + let submission_id = input + .submission_request_id + .clone() + .filter(|request_id| !request_id.trim().is_empty()) + .unwrap_or_else(|| Uuid::now_v7().to_string()); + input.submission_request_id = Some(submission_id.clone()); + Some(submission_id) + } else { + None + }; self.ensure_worker_execution(worker_ref)?; let (backend, handle) = { let state = self.lock()?; @@ -1191,13 +1200,13 @@ impl Runtime { } if let Some(expected_submission_id) = expected_submission_id && dispatch_result - .input_commit + .submission .as_ref() - .is_none_or(|ack| ack.submission_id != expected_submission_id) + .is_none_or(|ack| ack.submission_request_id != expected_submission_id) { let result = WorkerExecutionResult::rejected( WorkerExecutionOperation::Input, - "execution backend did not acknowledge the committed Runtime submission id", + "execution backend did not acknowledge the committed Runtime submission request id", ); self.record_execution_result(worker_ref, result.clone())?; return Err(RuntimeError::WorkerExecutionRejected { @@ -1209,6 +1218,7 @@ impl Runtime { }); } + let submission = dispatch_result.submission.clone(); let mut state = self.lock()?; state.ensure_running()?; let worker = state.worker_mut(worker_ref)?; @@ -1225,6 +1235,7 @@ impl Runtime { Ok(WorkerInteractionAck { worker_ref: worker_ref.clone(), status, + submission, }) } @@ -1706,6 +1717,7 @@ impl Runtime { } Ok(protocol::Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -3250,17 +3262,9 @@ fn validate_worker_input(input: &WorkerInput) -> Result<(), RuntimeError> { #[cfg(feature = "ws-server")] fn input_protocol_event(input: &WorkerInput) -> Option { match input.kind { - WorkerInputKind::User => Some(protocol::Event::UserMessage { - segments: input.segments.clone().unwrap_or_else(|| { - vec![protocol::Segment::Text { - content: input.content.clone(), - }] - }), - }), - // The committed `SystemItem::Notification` is the sole agent-visible - // and Console-visible authority for Notify. A synthetic observation - // here would display the same notification twice. - WorkerInputKind::Notify => None, + // Submit is projected only after the Worker commits UserInput. Queued + // payloads must never become model- or client-visible history early. + WorkerInputKind::User | WorkerInputKind::Notify => None, WorkerInputKind::Compact | WorkerInputKind::ListRewindTargets | WorkerInputKind::RegisterPeer => Some(protocol::Event::SystemItem { @@ -3435,6 +3439,7 @@ mod tests { ); let snapshot = protocol::Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -3475,7 +3480,7 @@ mod tests { let input = WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(vec![protocol::Segment::Flow { selector: "builtin:coder-review".to_string(), }]), @@ -3488,7 +3493,7 @@ mod tests { let input = WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(Vec::new()), }; assert!(matches!( @@ -3503,7 +3508,7 @@ mod tests { request.initial_input = Some(WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(vec![protocol::Segment::Flow { selector: "builtin:coder-review".to_string(), }]), @@ -4005,7 +4010,7 @@ mod tests { _handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - let submission_id = input.submission_id.clone(); + let submission_id = input.submission_request_id.clone(); self.dispatched_inputs.lock().unwrap().push(input); let mut result = self .dispatch_result @@ -4013,19 +4018,21 @@ mod tests { .unwrap() .clone() .unwrap_or_else(|| { - WorkerExecutionResult::accepted_input_committed( + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, WorkerExecutionRunState::Idle, + "request-test", "test-submission", + protocol::SubmissionDisposition::Started, ) }); if !self .preserve_commit_ack_submission_id .load(Ordering::SeqCst) && let (Some(ack), Some(submission_id)) = - (result.input_commit.as_mut(), submission_id) + (result.submission.as_mut(), submission_id) { - ack.submission_id = submission_id; + ack.submission_request_id = submission_id; } result } @@ -4717,10 +4724,12 @@ mod tests { #[test] fn create_worker_uses_committed_input_ack_run_state() { let (runtime, backend) = runtime_and_backend(); - backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed( + backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, WorkerExecutionRunState::Idle, + "request-test", "test-submission", + protocol::SubmissionDisposition::Started, )); let mut request = task_request("committed initial input is already idle"); request.initial_input = Some(WorkerInput::user("start the ticket")); @@ -4731,13 +4740,15 @@ mod tests { } #[test] - fn create_worker_rejects_mismatched_input_commit_acknowledgement() { + fn create_worker_rejects_mismatched_submission_acknowledgement() { let (runtime, backend) = runtime_and_backend(); backend.preserve_commit_ack_submission_id(); - backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed( + backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, WorkerExecutionRunState::Busy, + "request-test", "forged-submission", + protocol::SubmissionDisposition::Started, )); let mut request = task_request("mismatched initial input commit ack"); request.initial_input = Some(WorkerInput::user("start the ticket")); @@ -4866,6 +4877,7 @@ mod tests { &detail.worker_ref, protocol::Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: vec![protocol::SessionSnapshotEntry { entry_id: "restored-log-entry".to_owned(), timestamp: 1, @@ -4937,10 +4949,14 @@ mod tests { _handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - WorkerExecutionResult::accepted_input_committed( + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, WorkerExecutionRunState::Idle, - input.submission_id.expect("Runtime submission id"), + "request-test", + input + .submission_request_id + .expect("Runtime submission request id"), + protocol::SubmissionDisposition::Started, ) } } @@ -5030,7 +5046,7 @@ mod tests { request.initial_input = Some(WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(vec![ protocol::Segment::Flow { selector: "builtin:coder-review".to_string(), @@ -5068,7 +5084,7 @@ mod tests { let input = WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(vec![protocol::Segment::Flow { selector: "builtin:coder-review".to_string(), }]), @@ -5083,11 +5099,11 @@ mod tests { assert_eq!(dispatched[0].kind, input.kind); assert_eq!(dispatched[0].content, input.content); assert_eq!(dispatched[0].segments, input.segments); - let submission_id = dispatched[0] - .submission_id + let submission_request_id = dispatched[0] + .submission_request_id .as_deref() - .expect("Runtime submission id"); - Uuid::parse_str(submission_id).expect("submission id UUID"); + .expect("Runtime submission request id"); + Uuid::parse_str(submission_request_id).expect("submission request id UUID"); } #[cfg(feature = "ws-server")] @@ -5113,11 +5129,7 @@ mod tests { let observations = runtime .read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero()) .unwrap(); - assert_eq!(observations.len(), 1); - assert!(matches!( - observations[0].payload, - protocol::Event::UserMessage { .. } - )); + assert!(observations.is_empty()); runtime .observe_worker_event( @@ -5135,8 +5147,8 @@ mod tests { let observations = runtime .read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero()) .unwrap(); - assert_eq!(observations.len(), 2); - let protocol::Event::SystemItem { item } = &observations[1].payload else { + assert_eq!(observations.len(), 1); + let protocol::Event::SystemItem { item } = &observations[0].payload else { panic!("committed notification observation must be a system item"); }; assert_eq!(item["kind"], "notification"); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 95d0c941..07800e82 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -39,7 +39,7 @@ use crate::working_directory::{ }; use async_trait::async_trait; use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus}; -use session_store::{CombinedStore, LogEntry, WorkerAggregateStore, WorkerSessionStore}; +use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore}; #[cfg(test)] use session_store::{FsStore, FsWorkerStore}; use tokio::runtime::Runtime; @@ -57,11 +57,10 @@ use worker::feature::builtin::{ #[cfg(feature = "ws-server")] use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session}; use worker::{ - PreparedWorker, PromptCatalogSource, SegmentLogSink, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, - Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, - WorkerControllerTransport, WorkerError, WorkerFilesystemAuthority, WorkerHandle, - WorkerSharedState, WorkerWorkspaceContext, WorkspaceClient, WorkspaceId, - bash_output_dir_for_worker_id, + PreparedWorker, PromptCatalogSource, SegmentLogSink, Worker, WorkerBootstrap, + WorkerBootstrapError, WorkerBootstrapLayout, WorkerControllerTransport, WorkerError, + WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, WorkerWorkspaceContext, + WorkspaceClient, WorkspaceId, bash_output_dir_for_worker_id, }; const DEFAULT_BACKEND_ID: &str = "worker-crate"; @@ -70,17 +69,6 @@ const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10); // returns a typed execution error instead of leaving the outer waiter to time out. const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9); -fn user_input_has_submission(entry: &LogEntry, submission_id: &str) -> bool { - let extensions = match entry { - LogEntry::AnnotatedUserInput { extensions, .. } => extensions, - _ => return false, - }; - extensions.iter().any(|extension| { - extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN - && extension.payload["submission_id"].as_str() == Some(submission_id) - }) -} - pub struct RuntimeWorkerController { pub handle: WorkerHandle, pub shutdown: Arc>>, @@ -1342,126 +1330,75 @@ where .unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message)) } - fn send_user_input_and_wait_for_commit( + fn send_submit_and_wait_for_acceptance( &self, operation: WorkerExecutionOperation, worker: WorkerHandle, method: Method, - submission_id: String, + submission_request_id: String, accepted_run_state: WorkerExecutionRunState, ) -> WorkerExecutionResult { - let acknowledged_submission_id = submission_id.clone(); + let request_id = submission_request_id.clone(); self.run_on_adapter_runtime(async move { - // Subscribe before enqueueing the input so the acknowledgement cannot - // race with a fast Worker commit. The opaque submission id is stored in - // the same UserInput entry as the transformed Flow input and its state. - let (_, mut committed_entries) = worker.sink.subscribe_with_snapshot(); - let committed_probe = worker.clone(); + // Subscribe before enqueueing so a fast durable acceptance cannot + // race the Runtime acknowledgement. let mut events = worker.subscribe(); worker .send(method) .await .map_err(|err| format!("failed to send Worker method: {err}"))?; - let timeout_probe = committed_probe.clone(); - let timeout_submission_id = submission_id.clone(); - let acknowledgement = tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move { - let input_was_committed = || { - committed_probe - .committed_entries() - .iter() - .any(|entry| user_input_has_submission(entry, &submission_id)) - }; + tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move { loop { - tokio::select! { - entry = committed_entries.recv() => { - match entry { - Ok(entry) if user_input_has_submission(&entry, &submission_id) => { - return Ok(()); - } - Ok(_) => {} - Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { - if input_was_committed() { - return Ok(()); - } - return Err(format!( - "worker input commit acknowledgement lagged by {skipped} entry event(s)" - )); - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - if input_was_committed() { - return Ok(()); - } - return Err( - "worker entry stream closed before user input was committed" - .to_string(), - ); - } - } + match events.recv().await { + Ok(Event::SubmissionAccepted { + submission_request_id, + submission_id, + disposition, + }) if submission_request_id == request_id => { + return Ok((submission_id, disposition)); } - event = events.recv() => { - match event { - Ok(Event::Error { message, .. }) => { - if input_was_committed() { - return Ok(()); - } - return Err(format!( - "worker rejected user input before session commit: {message}" - )); - } - Ok(Event::Shutdown) => { - if input_was_committed() { - return Ok(()); - } - return Err( - "worker shut down before user input was committed".to_string() - ); - } - Ok(_) => {} - Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { - if input_was_committed() { - return Ok(()); - } - return Err(format!( - "worker input commit acknowledgement lagged by {skipped} protocol event(s)" - )); - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - if input_was_committed() { - return Ok(()); - } - return Err( - "worker event stream closed before user input was committed" - .to_string(), - ); - } - } + Ok(Event::SubmissionRejected { + submission_request_id, + message, + }) if submission_request_id == request_id => { + return Err(format!("worker rejected Submit: {message}")); + } + Ok(Event::Error { message, .. }) => { + return Err(format!( + "worker rejected Submit before durable acceptance: {message}" + )); + } + Ok(Event::Shutdown) => { + return Err( + "worker shut down before Submit was durably accepted".to_string() + ); + } + Ok(_) => {} + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + return Err(format!( + "worker Submit acknowledgement lagged by {skipped} protocol event(s)" + )); + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + return Err( + "worker event stream closed before Submit was durably accepted" + .to_string(), + ); } } } }) - .await; - - match acknowledgement { - Ok(result) => result, - Err(_) => { - if timeout_probe - .committed_entries() - .iter() - .any(|entry| user_input_has_submission(entry, &timeout_submission_id)) - { - Ok(()) - } else { - Err("timed out waiting for worker user input commit".to_string()) - } - } - } + .await + .map_err(|_| "timed out waiting for durable Worker Submit acceptance".to_string())? }) - .map(|_| { - WorkerExecutionResult::accepted_input_committed( + .map(|(submission_id, disposition)| { + WorkerExecutionResult::accepted_submission( operation, accepted_run_state, - acknowledged_submission_id, + submission_request_id, + submission_id, + disposition, ) }) .unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message)) @@ -1582,8 +1519,8 @@ impl Drop for WorkerRuntimeExecutionBackend { fn method_starts_turn(method: &Method) -> bool { matches!( method, - Method::Run { .. } - | Method::RunTracked { .. } + Method::Submit { .. } + | Method::SubmitTracked { .. } | Method::Notify { auto_run: true, .. } | Method::Resume | Method::Compact @@ -1609,8 +1546,8 @@ fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExec fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState { match method { - Method::Run { .. } - | Method::RunTracked { .. } + Method::Submit { .. } + | Method::SubmitTracked { .. } | Method::Notify { auto_run: true, .. } | Method::Resume | Method::Compact => WorkerExecutionRunState::Busy, @@ -1963,6 +1900,9 @@ where WorkerExecutionOperation::Input, worker, Method::Notify { + notification_request_id: input + .submission_request_id + .unwrap_or_else(protocol::new_submission_request_id), message: input.content, auto_run: true, }, @@ -1975,21 +1915,23 @@ where return result; } - if worker.shared_state.get_status() != WorkerStatus::Idle - || busy + let is_user_submit = input.kind == WorkerInputKind::User; + let status = worker.shared_state.get_status(); + let claimed_here = status == WorkerStatus::Idle + && busy .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_err() - { + .is_ok(); + if !is_user_submit && !claimed_here { return WorkerExecutionResult::busy( WorkerExecutionOperation::Input, - "Worker is already running; runtime adapter v0 does not queue input", + "Worker is already running", ); } - let (method, submission_id) = match input.kind { + let (method, submission_request_id) = match input.kind { WorkerInputKind::User => { let Some(submission_id) = input - .submission_id + .submission_request_id .filter(|submission_id| !submission_id.trim().is_empty()) else { busy.store(false, Ordering::SeqCst); @@ -1999,11 +1941,11 @@ where ); }; ( - Method::RunTracked { + Method::SubmitTracked { + submission_request_id: submission_id.clone(), input: input.segments.unwrap_or_else(|| { vec![Segment::text(input.content.trim().to_string())] }), - submission_id: submission_id.clone(), }, Some(submission_id), ) @@ -2021,21 +1963,21 @@ where ), }; let accepted_run_state = match method { - Method::Run { .. } - | Method::RunTracked { .. } + Method::Submit { .. } + | Method::SubmitTracked { .. } | Method::Notify { .. } | Method::Compact => WorkerExecutionRunState::Busy, _ => WorkerExecutionRunState::Idle, }; let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle; - let waits_for_user_input_commit = submission_id.is_some(); + let waits_for_submission_acceptance = submission_request_id.is_some(); - let result = if waits_for_user_input_commit { - self.send_user_input_and_wait_for_commit( + let result = if waits_for_submission_acceptance { + self.send_submit_and_wait_for_acceptance( WorkerExecutionOperation::Input, worker, method, - submission_id.expect("tracked Run has submission id"), + submission_request_id.expect("Submit must have a submission request id"), accepted_run_state, ) } else { @@ -2046,7 +1988,9 @@ where accepted_run_state, ) }; - if accepted_is_idle || result.outcome != crate::execution::WorkerExecutionOutcome::Accepted + if accepted_is_idle + || (claimed_here + && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted) { busy.store(false, Ordering::SeqCst); } @@ -3400,6 +3344,60 @@ mod tests { ); } + #[test] + fn running_worker_accepts_a_second_submit_as_queued() { + let client = MockClient::sequential(vec![MockResponse::Hang(vec![])]); + let runtime_base = tempfile::tempdir().unwrap(); + let cwd = tempfile::tempdir().unwrap(); + let store = tempfile::tempdir().unwrap(); + let factory = MockFactory { + client, + runtime_base: runtime_base.path().to_path_buf(), + cwd: cwd.path().to_path_buf(), + store_dir: store.path().join("sessions"), + worker_metadata_dir: store.path().join("workers"), + observed_cwds: Arc::new(Mutex::new(Vec::new())), + observed_workspace_clients: Arc::new(Mutex::new(Vec::new())), + }; + let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()); + let runtime = + EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let detail = runtime + .create_worker(create_request("queued-submit")) + .unwrap(); + + let mut first_input = WorkerInput::user("first"); + first_input.submission_request_id = Some("request-first".into()); + let first = runtime + .send_input(&detail.worker_ref, first_input.clone()) + .unwrap(); + assert_eq!( + first.submission.as_ref().map(|ack| ack.disposition), + Some(protocol::SubmissionDisposition::Started) + ); + let retry = runtime.send_input(&detail.worker_ref, first_input).unwrap(); + assert_eq!(retry.submission, first.submission); + let mut conflicting_retry = WorkerInput::user("different"); + conflicting_retry.submission_request_id = Some("request-first".into()); + assert!( + runtime + .send_input(&detail.worker_ref, conflicting_retry) + .is_err(), + "same request id with a different payload must fail" + ); + + let mut second_input = WorkerInput::user("second"); + second_input.submission_request_id = Some("request-second".into()); + let second = runtime + .send_input(&detail.worker_ref, second_input) + .unwrap(); + assert_eq!( + second.submission.as_ref().map(|ack| ack.disposition), + Some(protocol::SubmissionDisposition::Queued) + ); + } + #[test] fn create_with_initial_input_returns_after_session_commit() { let client = MockClient::new(simple_text_events()); @@ -3449,8 +3447,10 @@ mod tests { }; extensions .iter() - .find(|extension| extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN) - .and_then(|extension| extension.payload["submission_id"].as_str()) + .find(|extension| extension.domain == "worker.pending_activations.v1") + .and_then(|extension| { + extension.payload["receipts"][0]["submission_id"].as_str() + }) }) .expect("committed input submission id"); uuid::Uuid::parse_str(submission_id).expect("opaque submission id is a UUID"); diff --git a/crates/worker/Cargo.toml b/crates/worker/Cargo.toml index 87437d17..87046834 100644 --- a/crates/worker/Cargo.toml +++ b/crates/worker/Cargo.toml @@ -20,6 +20,7 @@ protocol = { workspace = true, features = ["json-schema"] } client = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +sha2 = { workspace = true } reqwest = { version = "0.13", default-features = false, features = ["blocking", "native-tls"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["fs", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } diff --git a/crates/worker/examples/worker_protocol.rs b/crates/worker/examples/worker_protocol.rs index 1e2c618f..cd00370a 100644 --- a/crates/worker/examples/worker_protocol.rs +++ b/crates/worker/examples/worker_protocol.rs @@ -101,7 +101,10 @@ async fn main() -> Result<(), Box> { // Send a run method handle - .send(Method::run_text("What is the capital of France?")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "What is the capital of France?", + )) .await?; // Wait for completion diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 4e431d3d..2794b35c 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -5,7 +5,7 @@ use std::sync::atomic::Ordering; use agen::EngineError; use agen::llm_client::client::LlmClient; use session_store::WorkerMetadataStore; -use session_store::{LogEntry, SessionExtension, Store}; +use session_store::{LogEntry, Store}; use tokio::sync::{broadcast, mpsc, oneshot}; use crate::discovery::WorkerDiscovery; @@ -23,16 +23,12 @@ use crate::shutdown_after_idle::{ }; use crate::spawn::registry::SpawnedWorkerRegistry; use crate::spawn::tool::sub_worker_spawn_tool; -use crate::worker::{ - SystemItemCommitter, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, WorkerError, - WorkerRunResult, -}; +use crate::worker::{SystemItemCommitter, Worker, WorkerError, WorkerRunResult}; use protocol::{ AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent, CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus, CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice, - ErrorCode, Event, Method, RewindTargetId, RunResult, Segment, TurnResult, UploadedFileRef, - WorkerStatus, + ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef, WorkerStatus, }; use workdir::{ CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot, @@ -58,6 +54,7 @@ pub struct WorkerHandle { spawned_registry: Arc, artifact_store: Arc, session_id: session_store::SessionId, + pending_activations: Arc>, } impl WorkerHandle { @@ -131,8 +128,15 @@ impl WorkerHandle { let in_flight = snapshot_from_guard(&in_flight_guard); (entries, entry_rx, in_flight) }; + let mut session = + session_store::public_snapshot::project_current_session_snapshot(&entries); + session.pending_submissions = self + .pending_activations + .lock() + .expect("pending activation state poisoned") + .snapshot(); let event = Event::Snapshot { - session: session_store::public_snapshot::project_current_session_snapshot(&entries), + session, greeting: self.shared_state.greeting.clone(), status: self.shared_state.get_status(), in_flight, @@ -213,21 +217,41 @@ async fn finish_controller_run( /// `Worker::*` entry point — `RunForNotification` carries none because /// `worker.run_for_notification()` drains the NotifyBuffer on its own. enum PendingRun { - Run(Vec), - RunTracked { - input: Vec, - extension: SessionExtension, - }, + Submit(crate::worker::PendingSubmission), /// Self-initiated turn kicked from the notify buffer. The carried /// `InvokeKind` is the trigger that flipped the Worker from IDLE /// (Notify or WorkerEvent) and is recorded by the Invoke marker /// committed at the start of `worker.run_for_notification`. - RunForNotification(protocol::InvokeKind), + RunForNotification { + invoke_kind: protocol::InvokeKind, + notification_request_id: Option, + }, Resume, } +fn prepare_pending_run( + pending_submissions: &crate::worker::PendingSubmissionHandle, + notify_buffer: &NotifyBuffer, +) -> Result, crate::worker::PendingSubmissionError> { + Ok(match pending_submissions.prepare_next_activation()? { + Some(crate::worker::PendingActivation::Submission(submission)) => { + Some(PendingRun::Submit(submission)) + } + Some(crate::worker::PendingActivation::Notification(notification)) => { + let extension = pending_submissions.notification_activation_extension(); + let notification_request_id = notification.notification_request_id.clone(); + notify_buffer.push_durable_notify(notification.message, extension); + Some(PendingRun::RunForNotification { + invoke_kind: protocol::InvokeKind::Notify, + notification_request_id: Some(notification_request_id), + }) + } + None => None, + }) +} + impl PendingRun { - /// Whether this turn was kicked off by the parent (via `Method::Run` + /// Whether this turn was kicked off by the parent (via `Method::Submit` /// or `Method::Resume`). Used by [`drive_turn`] to gate upward /// `WorkerEvent::TurnEnded` / `WorkerEvent::Errored` reports so the parent /// only sees completion signals for work it actually delegated. @@ -235,16 +259,12 @@ impl PendingRun { /// notify buffer (Notify / inbound WorkerEvent) and stays silent. fn is_parent_originated(&self) -> bool { match self { - PendingRun::Run(_) | PendingRun::RunTracked { .. } | PendingRun::Resume => true, - PendingRun::RunForNotification(_) => false, + PendingRun::Submit(_) | PendingRun::Resume => true, + PendingRun::RunForNotification { .. } => false, } } } -fn should_auto_run_notification(status: WorkerStatus, auto_run: bool) -> bool { - auto_run && status == WorkerStatus::Idle -} - // --------------------------------------------------------------------------- // WorkerController — actor that owns a Worker // --------------------------------------------------------------------------- @@ -552,6 +572,7 @@ impl WorkerController { let artifact_store: Arc = Arc::new(worker.store().clone()); let session_id = worker.session_id(); + let pending_activations = worker.pending_activation_state(); let handle = WorkerHandle { method_tx, working_event_tx: working_event_tx.clone(), @@ -563,6 +584,7 @@ impl WorkerController { spawned_registry: spawned_registry.clone(), artifact_store, session_id, + pending_activations, }; let socket_server = match transport { @@ -1291,6 +1313,7 @@ async fn controller_loop( spawned_registry.clone(), ); let mut pending: Option = None; + let pending_submissions = worker.pending_submission_handle(); loop { // Top-of-iteration: if an event handler staged a run, fire it @@ -1307,8 +1330,8 @@ async fn controller_loop( // interrupted/error turn from being carried into the next snapshot. worker.clear_in_flight_events(); let parent_originated = run.is_parent_originated(); - let user_input_run = matches!(&run, PendingRun::Run(_) | PendingRun::RunTracked { .. }); - if !user_input_run { + let user_input_submit = matches!(&run, PendingRun::Submit(_)); + if !user_input_submit { set_controller_status( &shared_state, &runtime_dir, @@ -1317,37 +1340,21 @@ async fn controller_loop( ) .await; } - let (mut new_status, shutdown) = match run { - PendingRun::Run(input) => { + let notification_request_id = match &run { + PendingRun::RunForNotification { + notification_request_id, + .. + } => notification_request_id.clone(), + _ => None, + }; + let (mut new_status, shutdown, may_drain_pending) = match run { + PendingRun::Submit(submission) => { let (input_commit_tx, input_commit_rx) = oneshot::channel(); + let committed_submission = submission.clone(); + let extension = pending_submissions.activation_extension(); drive_turn( worker.run_with_input_extensions_and_commit_hook( - input, - Vec::new(), - move || { - let _ = input_commit_tx.send(()); - }, - ), - &mut method_rx, - &working_event_tx, - &cancel_tx, - &pause_tx, - &shared_state, - &runtime_dir, - Some(input_commit_rx), - ¬ify_buffer, - self_parent_socket.as_ref(), - &spawner_name, - &spawned_registry, - parent_originated, - ) - .await - } - PendingRun::RunTracked { input, extension } => { - let (input_commit_tx, input_commit_rx) = oneshot::channel(); - drive_turn( - worker.run_with_input_extensions_and_commit_hook( - input, + submission.input, vec![extension], move || { let _ = input_commit_tx.send(()); @@ -1359,8 +1366,9 @@ async fn controller_loop( &pause_tx, &shared_state, &runtime_dir, - Some(input_commit_rx), + Some((input_commit_rx, committed_submission)), ¬ify_buffer, + &pending_submissions, self_parent_socket.as_ref(), &spawner_name, &spawned_registry, @@ -1368,9 +1376,9 @@ async fn controller_loop( ) .await } - PendingRun::RunForNotification(kind) => { + PendingRun::RunForNotification { invoke_kind, .. } => { drive_turn( - worker.run_for_notification(kind), + worker.run_for_notification(invoke_kind), &mut method_rx, &working_event_tx, &cancel_tx, @@ -1379,6 +1387,7 @@ async fn controller_loop( &runtime_dir, None, ¬ify_buffer, + &pending_submissions, self_parent_socket.as_ref(), &spawner_name, &spawned_registry, @@ -1397,6 +1406,7 @@ async fn controller_loop( &runtime_dir, None, ¬ify_buffer, + &pending_submissions, self_parent_socket.as_ref(), &spawner_name, &spawned_registry, @@ -1405,10 +1415,32 @@ async fn controller_loop( .await } }; - if !shutdown && new_status == WorkerStatus::Idle && notify_buffer.has_auto_run_pending() - { - pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify)); - new_status = WorkerStatus::Running; + if let Some(notification_request_id) = notification_request_id { + pending_submissions.finish_notification_activation(¬ification_request_id); + } + + if !shutdown && may_drain_pending && new_status == WorkerStatus::Idle { + match prepare_pending_run(&pending_submissions, ¬ify_buffer) { + Ok(Some(next)) => { + pending = Some(next); + new_status = WorkerStatus::Running; + } + Ok(None) => { + if notify_buffer.has_auto_run_pending() { + pending = Some(PendingRun::RunForNotification { + invoke_kind: protocol::InvokeKind::Notify, + notification_request_id: None, + }); + new_status = WorkerStatus::Running; + } + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } + } } finish_controller_run( &mut worker, @@ -1435,61 +1467,118 @@ async fn controller_loop( }; match method { - Method::Run { input } => { - if shared_state.get_status() == WorkerStatus::Running { - // Defensive: the inner select! inside drive_turn - // already rejects `Run` while a turn is live, so - // this branch is only reachable across a race window - // around status flips. - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::AlreadyRunning, - message: "Worker is already executing a turn".into(), - }); - continue; - } - // Stage the run without a speculative user-message echo. - // `Worker::run` validates the input, commits - // `LogEntry::AnnotatedUserInput`, and the session-log sink turns that - // committed entry into the live `Event::UserMessage`. That - // keeps every client ordered against `SegmentStart` replay and - // makes persisted history the single source of visible user - // input. Paused→Run cleanup (orphan tool_result closure + - // interrupt system note) is applied inside `Worker::run` itself - // when the worker's `last_run_interrupted` flag is set. - pending = Some(PendingRun::Run(input)); - } - - Method::RunTracked { + Method::Submit { + submission_request_id, input, - submission_id, - } => { - // Runtime-correlated submissions retain their opaque id in the - // same durable UserInput record used for Flow state. - let extension = SessionExtension::new( - WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, - serde_json::json!({ "submission_id": submission_id }), - ); - pending = Some(PendingRun::RunTracked { input, extension }); } - - Method::Notify { message, auto_run } => { - // Client-side live echo is delivered as `Event::SystemItem` - // once the interceptor commits the corresponding - // `LogEntry::AnnotatedSystemItem` entry — drained out of the - // notify buffer + broadcast through the sink. No - // separate echo here. - worker.push_notify(message, auto_run); - // RUNNING: the in-flight turn drains the buffer at its next - // pending_history_appends; if an auto-run notification remains - // at turn end, the Controller stages a follow-up notification - // turn. Paused notifications remain queued until Resume/Run. - // IDLE: `auto_run` notifications stage RunForNotification; - // weak progress notices stay queued until an explicit run. - if should_auto_run_notification(shared_state.get_status(), auto_run) { - pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify)); + | Method::SubmitTracked { + submission_request_id, + input, + } => { + let request_id = submission_request_id.clone(); + match pending_submissions.accept(submission_request_id, input, true) { + Ok(acceptance) => { + if let Some(activation) = acceptance.activation { + pending = Some(PendingRun::Submit(activation)); + } else { + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: acceptance.submission_request_id, + submission_id: acceptance.submission_id, + disposition: acceptance.disposition, + }); + } + } + Err(error) => { + let _ = working_event_tx.send(Event::SubmissionRejected { + submission_request_id: request_id, + message: error.to_string(), + }); + } } } + Method::Notify { + notification_request_id, + message, + auto_run, + } => { + if auto_run { + match pending_submissions.accept_notification(notification_request_id, message) + { + Ok(true) => match prepare_pending_run(&pending_submissions, ¬ify_buffer) + { + Ok(Some(next)) => pending = Some(next), + Ok(None) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } + }, + Ok(false) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + } else { + worker.push_notify(message, false); + } + } + + Method::ListPendingSubmissions => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Method::CancelPendingSubmission { submission_id } => { + match pending_submissions.cancel(&submission_id) { + Ok(pending_snapshot) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_snapshot, + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + } + Method::ClearPendingSubmissions => match pending_submissions.clear() { + Ok(pending_snapshot) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_snapshot, + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } + }, + Method::ContinuePending => { + match prepare_pending_run(&pending_submissions, ¬ify_buffer) { + Ok(Some(next)) => pending = Some(next), + Ok(None) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: "pending activation queue is empty".into(), + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } + } + } Method::Resume => { if shared_state.get_status() != WorkerStatus::Paused { let _ = working_event_tx.send(Event::Error { @@ -1703,9 +1792,10 @@ async fn controller_loop( // notification is not stranded. Matches the // `Method::Notify` idle path. if shared_state.get_status() == WorkerStatus::Idle { - pending = Some(PendingRun::RunForNotification( - protocol::InvokeKind::WorkerEvent, - )); + pending = Some(PendingRun::RunForNotification { + invoke_kind: protocol::InvokeKind::WorkerEvent, + notification_request_id: None, + }); } } } @@ -1788,12 +1878,12 @@ async fn handle_inbound_worker_event( /// as `Errored` — only the worker-execution `Err` branch below fires. /// /// `parent_originated` further restricts both upward reports to turns -/// the parent actually delegated (`Method::Run` / `Method::Resume`). +/// the parent actually delegated (`Method::Submit` / `Method::Resume`). /// `Method::Notify` / inbound `WorkerEvent` auto-kicks complete silently /// so the parent's history does not get flooded with child-internal /// turn boundaries. #[allow(clippy::too_many_arguments)] -async fn drive_turn( +async fn drive_turn( worker_future: F, method_rx: &mut mpsc::Receiver, working_event_tx: &broadcast::Sender, @@ -1801,15 +1891,17 @@ async fn drive_turn( pause_tx: &mpsc::Sender<()>, shared_state: &Arc, runtime_dir: &RuntimeDir, - mut input_commit_rx: Option>, + mut input_commit: Option<(oneshot::Receiver<()>, crate::worker::PendingSubmission)>, notify_buffer: &NotifyBuffer, + pending_submissions: &crate::worker::PendingSubmissionHandle, parent_socket: Option<&PathBuf>, self_name: &str, spawned_registry: &Arc, parent_originated: bool, -) -> (WorkerStatus, bool) +) -> (WorkerStatus, bool, bool) where F: std::future::Future>, + St: Store + Clone, { tokio::pin!(worker_future); let mut shutdown_requested = false; @@ -1822,13 +1914,25 @@ where // Running snapshot contract deterministic even for immediate clients. biased; committed = async { - input_commit_rx + input_commit .as_mut() + .map(|(receiver, _)| receiver) .expect("input commit receiver guarded by select condition") .await - }, if input_commit_rx.is_some() => { - input_commit_rx = None; + }, if input_commit.is_some() => { + let submission = input_commit.take().map(|(_, submission)| submission); if committed.is_ok() { + if let Some(submission) = submission { + pending_submissions.finish_activation(&submission.submission_id); + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: submission.submission_request_id, + submission_id: submission.submission_id, + disposition: protocol::SubmissionDisposition::Started, + }); + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } set_controller_status( shared_state, runtime_dir, @@ -1836,11 +1940,33 @@ where WorkerStatus::Running, ) .await; + } else if let Some(submission) = submission { + pending_submissions.abort_activation(submission); } } result = &mut worker_future => { + if let Some((mut receiver, submission)) = input_commit.take() { + match receiver.try_recv() { + Ok(()) => { + pending_submissions.finish_activation(&submission.submission_id); + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: submission.submission_request_id, + submission_id: submission.submission_id, + disposition: protocol::SubmissionDisposition::Started, + }); + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Err(_) => pending_submissions.abort_activation(submission), + } + } return match result { Ok(r) => { + let may_drain_pending = matches!( + &r, + WorkerRunResult::Finished | WorkerRunResult::LimitReached + ); let (status, run_result) = match r { WorkerRunResult::Finished if pause_requested => { (WorkerStatus::Paused, RunResult::Paused) @@ -1851,7 +1977,7 @@ where WorkerRunResult::RolledBack => (WorkerStatus::Idle, RunResult::RolledBack), WorkerRunResult::Interrupted { .. } if pause_requested => { let _ = working_event_tx.send(Event::RunEnd { result: RunResult::Paused }); - return (WorkerStatus::Paused, shutdown_requested); + return (WorkerStatus::Paused, shutdown_requested, false); } WorkerRunResult::Interrupted { code, message } => { let _ = working_event_tx.send(Event::Error { @@ -1867,7 +1993,7 @@ where }, ); } - return (WorkerStatus::Idle, shutdown_requested); + return (WorkerStatus::Idle, shutdown_requested, false); } }; let _ = working_event_tx.send(Event::RunEnd { result: run_result }); @@ -1879,7 +2005,7 @@ where }, ); } - (status, shutdown_requested) + (status, shutdown_requested, may_drain_pending) } Err(WorkerError::Engine(EngineError::Cancelled)) if pause_requested => { // User-initiated Pause. Report the transition to @@ -1888,7 +2014,7 @@ where // that channel is reserved for worker runtime // failures, not deliberate interruptions. let _ = working_event_tx.send(Event::RunEnd { result: RunResult::Paused }); - (WorkerStatus::Paused, shutdown_requested) + (WorkerStatus::Paused, shutdown_requested, false) } Err(e) => { let code = worker_error_code(&e); @@ -1906,11 +2032,11 @@ where }, ); } - (WorkerStatus::Idle, shutdown_requested) + (WorkerStatus::Idle, shutdown_requested, false) } }; } - method = method_rx.recv() => { + method = method_rx.recv(), if input_commit.is_none() => { match method { Some(Method::Cancel) => { let _ = cancel_tx.try_send(()); @@ -1923,12 +2049,71 @@ where shutdown_requested = true; let _ = cancel_tx.try_send(()); } - Some(Method::Run { .. } | Method::RunTracked { .. } | Method::Resume) => { + Some(Method::Submit { + submission_request_id, + input, + } + | Method::SubmitTracked { + submission_request_id, + input, + }) => { + let request_id = submission_request_id.clone(); + match pending_submissions.accept(submission_request_id, input, false) { + Ok(acceptance) => { + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: acceptance.submission_request_id, + submission_id: acceptance.submission_id, + disposition: acceptance.disposition, + }); + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::SubmissionRejected { + submission_request_id: request_id, + message: error.to_string(), + }); + } + } + } + Some(Method::Resume | Method::ContinuePending) => { let _ = working_event_tx.send(Event::Error { code: ErrorCode::AlreadyRunning, message: "Worker is already executing a turn".into(), }); } + Some(Method::ListPendingSubmissions) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Some(Method::CancelPendingSubmission { submission_id }) => { + match pending_submissions.cancel(&submission_id) { + Ok(pending) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { pending }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + } + Some(Method::ClearPendingSubmissions) => { + match pending_submissions.clear() { + Ok(pending) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { pending }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } + } + } Some(Method::Compact | Method::ListRewindTargets | Method::RewindTo { .. }) => { let _ = working_event_tx.send(Event::Error { code: ErrorCode::AlreadyRunning, @@ -1936,11 +2121,28 @@ where .into(), }); } - Some(Method::Notify { message, auto_run }) => { - // Live echo arrives via `Event::SystemItem` once - // the in-flight turn's next `pending_history_appends` - // drains this entry through the interceptor. - notify_buffer.push_notify(message, auto_run); + Some(Method::Notify { + notification_request_id, + message, + auto_run, + }) => { + if auto_run { + if let Err(error) = pending_submissions.accept_notification( + notification_request_id, + message, + ) { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } else { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + } else { + notify_buffer.push_notify(message, false); + } } Some(Method::ListCompletions { .. }) => {} Some(Method::ListWorkers | Method::RestoreWorker { .. } | Method::RegisterPeer { .. }) => { @@ -1969,7 +2171,7 @@ where None => { let _ = cancel_tx.try_send(()); shared_state.set_status(WorkerStatus::Idle); - return (WorkerStatus::Idle, false); + return (WorkerStatus::Idle, false, false); } } } @@ -2134,21 +2336,16 @@ mod tests { #[test] fn pending_run_parent_origin_table() { - assert!(PendingRun::Run(Vec::new()).is_parent_originated()); assert!(PendingRun::Resume.is_parent_originated()); assert!( - !PendingRun::RunForNotification(protocol::InvokeKind::Notify).is_parent_originated() + !PendingRun::RunForNotification { + invoke_kind: protocol::InvokeKind::Notify, + notification_request_id: None, + } + .is_parent_originated() ); } - #[test] - fn notification_auto_run_gate_only_allows_idle_auto_run() { - assert!(should_auto_run_notification(WorkerStatus::Idle, true)); - assert!(!should_auto_run_notification(WorkerStatus::Idle, false)); - assert!(!should_auto_run_notification(WorkerStatus::Running, true)); - assert!(!should_auto_run_notification(WorkerStatus::Paused, true)); - } - struct DriveTurnEnv { // Held to keep the channel alive; without this `method_rx.recv()` // would observe channel-closed and confuse the select! arm. @@ -2161,6 +2358,7 @@ mod tests { _pause_rx: mpsc::Receiver<()>, shared_state: Arc, notify_buffer: NotifyBuffer, + pending_submissions: crate::worker::PendingSubmissionHandle, spawned_registry: Arc, parent_socket_path: PathBuf, runtime_dir: Arc, @@ -2194,6 +2392,8 @@ mod tests { }, )); let notify_buffer = NotifyBuffer::new(); + let pending_submissions = + crate::worker::PendingSubmissionHandle::for_test(&temp.path().join("pending-sessions")); let spawned_registry = SpawnedWorkerRegistry::new(runtime_dir.clone()); let parent_socket_path = temp.path().join("parent.sock"); @@ -2207,6 +2407,7 @@ mod tests { _pause_rx: pause_rx, shared_state, notify_buffer, + pending_submissions, spawned_registry, parent_socket_path, runtime_dir, @@ -2225,6 +2426,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -2259,7 +2461,7 @@ mod tests { let recv = tokio::spawn(recv_worker_event(listener, Duration::from_secs(2))); let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2269,6 +2471,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, @@ -2302,7 +2505,7 @@ mod tests { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; let started_at = std::time::Instant::now(); - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2312,6 +2515,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, None, "child-worker", &env.spawned_registry, @@ -2332,7 +2536,7 @@ mod tests { let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener"); let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, _) = drive_turn( + let (status, _, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2342,6 +2546,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, @@ -2370,7 +2575,7 @@ mod tests { "boom from test".into(), ))) }; - let (status, _) = drive_turn( + let (status, _, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2380,6 +2585,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, @@ -2414,7 +2620,7 @@ mod tests { "boom from notify".into(), ))) }; - let (status, _) = drive_turn( + let (status, _, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2424,6 +2630,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, @@ -2456,7 +2663,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2466,6 +2673,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "parent", &env.spawned_registry, @@ -2495,7 +2703,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2505,6 +2713,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "parent", &env.spawned_registry, @@ -2522,6 +2731,7 @@ mod tests { let mut env = make_env().await; env._method_tx .send(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), message: "continue".into(), auto_run: true, }) @@ -2532,7 +2742,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2542,6 +2752,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "parent", &env.spawned_registry, @@ -2551,8 +2762,8 @@ mod tests { assert_eq!(status, WorkerStatus::Idle); assert!(!shutdown); - assert_eq!(env.notify_buffer.len(), 1); - assert!(env.notify_buffer.has_auto_run_pending()); + assert_eq!(env.notify_buffer.len(), 0); + assert_eq!(env.pending_submissions.snapshot().notification_count, 1); } #[tokio::test] @@ -2568,7 +2779,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2578,6 +2789,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, diff --git a/crates/worker/src/discovery.rs b/crates/worker/src/discovery.rs index 32da104d..effc1f6f 100644 --- a/crates/worker/src/discovery.rs +++ b/crates/worker/src/discovery.rs @@ -1012,7 +1012,15 @@ async fn send_peer_notify(socket_path: &Path, message: String) -> io::Result<()> } async fn send_notify(socket_path: &Path, message: String, auto_run: bool) -> io::Result<()> { - connect_and_send(socket_path, &Method::Notify { message, auto_run }).await + connect_and_send( + socket_path, + &Method::Notify { + notification_request_id: protocol::new_submission_request_id(), + message, + auto_run, + }, + ) + .await } fn json_content(value: &T) -> Result { @@ -1482,6 +1490,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1517,6 +1526,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1536,7 +1546,10 @@ mod tests { .await .unwrap(); let method = reader.next::().await.unwrap().unwrap(); - if let Method::Notify { message, auto_run } = method { + if let Method::Notify { + message, auto_run, .. + } = method + { assert!(auto_run); tx.send(message).await.unwrap(); } else { @@ -1608,6 +1621,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1634,6 +1648,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1653,7 +1668,10 @@ mod tests { .await .unwrap(); let method = reader.next::().await.unwrap().unwrap(); - if let Method::Notify { message, auto_run } = method { + if let Method::Notify { + message, auto_run, .. + } = method + { assert!(!auto_run); tx.send(message).await.unwrap(); } else { @@ -1738,6 +1756,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1790,6 +1809,8 @@ mod tests { let _ = writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default( + ), entries: Vec::new(), }, greeting: protocol::Greeting { diff --git a/crates/worker/src/feature/builtin/worker_observation.rs b/crates/worker/src/feature/builtin/worker_observation.rs index 2dc059f6..bda05e09 100644 --- a/crates/worker/src/feature/builtin/worker_observation.rs +++ b/crates/worker/src/feature/builtin/worker_observation.rs @@ -803,7 +803,10 @@ mod tests { .collect(); Ok(WorkerSessionCapture { segment_id: "segment".to_string(), - session: protocol::SessionSnapshot { entries }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries, + }, }) } } diff --git a/crates/worker/src/ipc/interceptor.rs b/crates/worker/src/ipc/interceptor.rs index 30bf4e12..5853a2f7 100644 --- a/crates/worker/src/ipc/interceptor.rs +++ b/crates/worker/src/ipc/interceptor.rs @@ -176,12 +176,16 @@ impl WorkerInterceptor { /// `Item::system_message`s reach the worker via /// `ContinueWith` / `pending_history_appends`, so on-disk order /// matches worker-history order. - fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> { + fn commit_system_items_with_extensions( + &self, + items: &[(SystemItem, Vec)], + ) -> Result<(), session_store::StoreError> { let Some(writer) = self.log_writer.as_ref() else { return Ok(()); }; - for item in items { - let entry = writer.commit_system_item(item.clone())?; + for (item, extensions) in items { + let entry = + writer.commit_system_item_with_extensions(item.clone(), extensions.clone())?; self.pending_committed_history .lock() .expect("pending committed history poisoned") @@ -190,6 +194,16 @@ impl WorkerInterceptor { Ok(()) } + fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> { + self.commit_system_items_with_extensions( + &items + .iter() + .cloned() + .map(|item| (item, Vec::new())) + .collect::>(), + ) + } + fn current_turn_index(&self) -> usize { self.next_turn_index .load(Ordering::Relaxed) @@ -327,7 +341,8 @@ impl Interceptor for WorkerInterceptor { projection_digest: projection.catalog_digest.clone(), logical_name: "internal.notify_wrapper".to_string(), }; - let mut system_items: Vec = Vec::with_capacity(drained.len()); + let mut system_items: Vec<(SystemItem, Vec)> = + Vec::with_capacity(drained.len()); let mut items: Vec = Vec::with_capacity(drained.len()); for entry in &drained { let system_item = match build_system_item_with_provenance( @@ -345,9 +360,9 @@ impl Interceptor for WorkerInterceptor { } }; items.push(system_item.to_history_item()); - system_items.push(system_item); + system_items.push((system_item, entry.extensions())); } - if let Err(error) = self.commit_system_items(&system_items) { + if let Err(error) = self.commit_system_items_with_extensions(&system_items) { self.pending_notifies.requeue_front(drained); return Err(InterceptorError::new( InterceptorErrorCategory::Dependency, diff --git a/crates/worker/src/ipc/notify_buffer.rs b/crates/worker/src/ipc/notify_buffer.rs index 7b97e4ed..cbaedc5d 100644 --- a/crates/worker/src/ipc/notify_buffer.rs +++ b/crates/worker/src/ipc/notify_buffer.rs @@ -25,7 +25,7 @@ use std::collections::VecDeque; use std::sync::{Arc, Mutex}; use protocol::WorkerEvent; -use session_store::SystemItem; +use session_store::{SessionExtension, SystemItem}; use tracing::warn; use crate::prompt::catalog::{CatalogError, PromptCatalog}; @@ -41,8 +41,23 @@ const CAPACITY: usize = 128; /// is available. #[derive(Debug, Clone)] pub enum PendingNotify { - Notify { message: String, auto_run: bool }, - WorkerEvent { event: WorkerEvent }, + Notify { + message: String, + auto_run: bool, + extensions: Vec, + }, + WorkerEvent { + event: WorkerEvent, + }, +} + +impl PendingNotify { + pub(crate) fn extensions(&self) -> Vec { + match self { + PendingNotify::Notify { extensions, .. } => extensions.clone(), + PendingNotify::WorkerEvent { .. } => Vec::new(), + } + } } /// Shared, mutex-guarded buffer of pending entries. @@ -62,7 +77,19 @@ impl NotifyBuffer { /// oldest entry is dropped and a `tracing::warn` is emitted — the /// caller should never hit this in normal operation. pub fn push_notify(&self, message: String, auto_run: bool) { - self.push_entry(PendingNotify::Notify { message, auto_run }); + self.push_entry(PendingNotify::Notify { + message, + auto_run, + extensions: Vec::new(), + }); + } + + pub fn push_durable_notify(&self, message: String, extension: SessionExtension) { + self.push_entry(PendingNotify::Notify { + message, + auto_run: true, + extensions: vec![extension], + }); } /// Push a typed worker-event entry onto the queue. @@ -202,6 +229,7 @@ mod tests { let entry = PendingNotify::Notify { message: "hello".into(), auto_run: false, + extensions: Vec::new(), }; let catalog = PromptCatalog::builtins_only().unwrap(); let item = build_system_item(&entry, &catalog).unwrap(); diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index d5f7f8d1..c83ef56a 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -57,9 +57,9 @@ pub use session_history::{ }; pub use shared_state::WorkerSharedState; pub use worker::{ - LocalWorkingDirectory, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, WorkerError, - WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceClient, - WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspacePromptCatalogResolution, - WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, WorkspaceWorkerDiscoveryRequest, - apply_worker_manifest, marker_workspace_client, unavailable_workspace_client, + LocalWorkingDirectory, Worker, WorkerError, WorkerFilesystemAuthority, WorkerRunResult, + WorkerWorkspaceContext, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError, + WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, + WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, marker_workspace_client, + unavailable_workspace_client, }; diff --git a/crates/worker/src/segment_log_sink.rs b/crates/worker/src/segment_log_sink.rs index 91c2792b..20233983 100644 --- a/crates/worker/src/segment_log_sink.rs +++ b/crates/worker/src/segment_log_sink.rs @@ -291,6 +291,7 @@ mod tests { prompt_provenance: None, }, ), + extensions: Vec::new(), } } diff --git a/crates/worker/src/spawn/comm_tools.rs b/crates/worker/src/spawn/comm_tools.rs index 27e6845f..0389f288 100644 --- a/crates/worker/src/spawn/comm_tools.rs +++ b/crates/worker/src/spawn/comm_tools.rs @@ -72,6 +72,7 @@ mod tests { fn snapshot(entries: Vec) -> Event { Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: entries .into_iter() .enumerate() diff --git a/crates/worker/src/spawn/tool.rs b/crates/worker/src/spawn/tool.rs index c2498874..7fab828e 100644 --- a/crates/worker/src/spawn/tool.rs +++ b/crates/worker/src/spawn/tool.rs @@ -58,7 +58,7 @@ struct SubWorkerSpawnInput { /// a host path and grants no authority. When omitted, the Workdir root is used. #[serde(default)] cwd: Option, - /// First message sent to the spawned SubWorker via `Method::Run`. + /// First message sent to the spawned SubWorker via `Method::Submit`. task: String, /// Allow rules delegated to the spawned SubWorker. Must be a subset of the /// spawner's explicit delegation authority; direct tool scope alone is not @@ -235,7 +235,11 @@ impl ParentNotificationTarget { }; tokio::spawn(async move { if let Err(error) = parent_method_tx - .send(Method::Notify { message, auto_run }) + .send(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), + message, + auto_run, + }) .await { tracing::warn!( @@ -1267,6 +1271,7 @@ enabled = false Method::Notify { message, auto_run: true, + .. } if message.contains("SubWorker `reviewer-child` turn ended with status Idle") )); assert!(!runtime.path().join("reviewer-child/sock").exists()); diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index d4a3b85c..cc93bb3a 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -1,3 +1,4 @@ +use std::collections::VecDeque; #[cfg(test)] use std::path::Path; use std::path::PathBuf; @@ -68,6 +69,130 @@ const LARGE_PASTE_INLINE_MAX_BYTES: usize = 32 * 1024; const WORKER_ORCHESTRATION_INSTRUCTION_ID: &str = "worker.orchestration"; const WORKER_ORCHESTRATION_PROMPT_REF: &str = "common.worker_orchestration"; const FEATURE_HOOK_CHAIN_TIMEOUT: Duration = Duration::from_secs(30); +const SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN: &str = "worker.pending_activations.v1"; +const MAX_PENDING_SUBMISSIONS: usize = 32; +const MAX_PENDING_SUBMISSION_BYTES: u64 = 1024 * 1024; +const MAX_PENDING_ARTIFACT_REFS: usize = 64; +const MAX_ACTIVATION_REQUEST_ID_BYTES: usize = 128; +const MAX_SUBMISSION_RECEIPTS: usize = 128; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub(crate) struct PendingSubmission { + pub(crate) submission_request_id: String, + pub(crate) submission_id: String, + payload_digest: String, + accepted_at_ms: u64, + activation_sequence: u64, + provenance: WorkerHistoryProvenance, + #[serde(default)] + was_queued: bool, + pub(crate) input: Vec, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct SubmissionReceipt { + submission_request_id: String, + submission_id: String, + payload_digest: String, + disposition: protocol::SubmissionDisposition, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub(crate) struct PendingNotification { + pub(crate) notification_request_id: String, + pub(crate) message: String, + payload_digest: String, + accepted_at_ms: u64, + activation_sequence: u64, + provenance: WorkerHistoryProvenance, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct NotificationReceipt { + notification_request_id: String, + payload_digest: String, +} + +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub(crate) struct PendingActivationState { + revision: u64, + next_activation_sequence: u64, + /// A prepared activation remains in checkpoints until the same atomic + /// UserInput record commits the clearing checkpoint. Restore puts it back + /// at the FIFO head. + activating: Option, + activating_notification: Option, + pending: VecDeque, + pending_notifications: VecDeque, + receipts: VecDeque, + notification_receipts: VecDeque, +} + +impl PendingActivationState { + pub(crate) fn snapshot(&self) -> protocol::PendingSubmissionsSnapshot { + protocol::PendingSubmissionsSnapshot { + revision: self.revision, + notification_count: u32::try_from(self.pending_notifications.len()).unwrap_or(u32::MAX), + submissions: self + .pending + .iter() + .map(|pending| protocol::PendingSubmissionSummary { + submission_id: pending.submission_id.clone(), + accepted_at_ms: pending.accepted_at_ms, + segment_count: u32::try_from(pending.input.len()).unwrap_or(u32::MAX), + byte_len: submission_payload_len(&pending.input), + }) + .collect(), + } + } + + fn remember_notification_receipt(&mut self, receipt: NotificationReceipt) { + self.notification_receipts.push_back(receipt); + while self.notification_receipts.len() > MAX_SUBMISSION_RECEIPTS { + self.notification_receipts.pop_front(); + } + } + + fn remember_receipt(&mut self, receipt: SubmissionReceipt) { + self.receipts.push_back(receipt); + while self.receipts.len() > MAX_SUBMISSION_RECEIPTS { + self.receipts.pop_front(); + } + } +} + +fn submission_payload_len(input: &[Segment]) -> u64 { + serde_json::to_vec(input) + .map(|bytes| u64::try_from(bytes.len()).unwrap_or(u64::MAX)) + .unwrap_or(u64::MAX) +} + +fn submission_payload_digest(input: &[Segment]) -> String { + use sha2::Digest as _; + sha2::Sha256::digest(serde_json::to_vec(input).unwrap_or_default()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn submission_artifact_ref_count(input: &[Segment]) -> usize { + input + .iter() + .filter(|segment| { + matches!( + segment, + Segment::PasteArtifact { .. } | Segment::UploadedFile { .. } + ) + }) + .count() +} + +fn pending_activation_extension(state: &PendingActivationState) -> SessionExtension { + SessionExtension { + domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(), + payload: serde_json::to_value(state).expect("pending activation state must serialize"), + } +} fn hook_run_exit(exit: &EngineRunExit) -> RunCommittedExit { match exit { @@ -970,15 +1095,489 @@ where } } -/// Type-erased commit handle for the interceptor. Lets the -/// interceptor commit `SystemItem`s without being generic over the +#[derive(Debug, Clone)] +pub(crate) enum PendingActivation { + Submission(PendingSubmission), + Notification(PendingNotification), +} + +#[derive(Debug, Clone)] +pub(crate) struct SubmissionAcceptance { + pub(crate) submission_request_id: String, + pub(crate) submission_id: String, + pub(crate) disposition: protocol::SubmissionDisposition, + pub(crate) activation: Option, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PendingSubmissionError { + #[error("submission_request_id must not be empty")] + EmptyRequestId, + #[error("activation request id exceeds {MAX_ACTIVATION_REQUEST_ID_BYTES} bytes")] + RequestIdLimit, + #[error("submission input must contain at least one typed segment")] + EmptyInput, + #[error("submission request id was already used with a different payload")] + IdempotencyConflict, + #[error("pending submission queue is full (maximum {MAX_PENDING_SUBMISSIONS})")] + CountLimit, + #[error("pending submission bytes exceed {MAX_PENDING_SUBMISSION_BYTES}")] + ByteLimit, + #[error("pending submission artifact references exceed {MAX_PENDING_ARTIFACT_REFS}")] + ArtifactLimit, + #[error("pending submission not found: {0}")] + NotFound(String), + #[error("pending submission state persistence failed: {0}")] + Store(#[from] StoreError), +} + +#[derive(Clone)] +pub(crate) struct PendingSubmissionHandle { + state: Arc>, + writer: LogWriterHandle, +} + +impl PendingSubmissionHandle +where + St: Store + Clone, +{ + fn persist_locked(&self, state: &PendingActivationState) -> Result<(), PendingSubmissionError> { + self.writer.append_entry_locked(LogEntry::Extension { + ts: segment_log::now_millis(), + domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(), + payload: serde_json::to_value(state).expect("pending activation state must serialize"), + })?; + Ok(()) + } + + pub(crate) fn accept( + &self, + submission_request_id: String, + input: Vec, + activate_now: bool, + ) -> Result { + if submission_request_id.trim().is_empty() { + return Err(PendingSubmissionError::EmptyRequestId); + } + if submission_request_id.len() > MAX_ACTIVATION_REQUEST_ID_BYTES { + return Err(PendingSubmissionError::RequestIdLimit); + } + if input.is_empty() { + return Err(PendingSubmissionError::EmptyInput); + } + let payload_digest = submission_payload_digest(&input); + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut current = self + .state + .lock() + .expect("pending activation state poisoned"); + let original = current.clone(); + if let Some(receipt) = current + .receipts + .iter() + .find(|receipt| receipt.submission_request_id == submission_request_id) + { + if receipt.payload_digest != payload_digest { + return Err(PendingSubmissionError::IdempotencyConflict); + } + return Ok(SubmissionAcceptance { + submission_request_id, + submission_id: receipt.submission_id.clone(), + disposition: receipt.disposition, + activation: None, + }); + } + + let submission_id = uuid::Uuid::now_v7().to_string(); + let pending = PendingSubmission { + submission_request_id: submission_request_id.clone(), + submission_id: submission_id.clone(), + payload_digest: payload_digest.clone(), + accepted_at_ms: segment_log::now_millis(), + activation_sequence: current.next_activation_sequence, + provenance: WorkerHistoryProvenance::LegacyUnknown, + was_queued: !activate_now, + input, + }; + current.next_activation_sequence = current.next_activation_sequence.saturating_add(1); + let disposition = if activate_now { + protocol::SubmissionDisposition::Started + } else { + protocol::SubmissionDisposition::Queued + }; + current.remember_receipt(SubmissionReceipt { + submission_request_id: submission_request_id.clone(), + submission_id: submission_id.clone(), + payload_digest, + disposition, + }); + current.revision = current.revision.saturating_add(1); + + if activate_now { + current.activating = Some(pending.clone()); + } else { + let count = current + .pending + .len() + .saturating_add(current.pending_notifications.len()) + .saturating_add(1); + if count > MAX_PENDING_SUBMISSIONS { + *current = original; + return Err(PendingSubmissionError::CountLimit); + } + let bytes = current + .pending + .iter() + .map(|pending| submission_payload_len(&pending.input)) + .sum::() + .saturating_add( + current + .pending_notifications + .iter() + .map(|pending| u64::try_from(pending.message.len()).unwrap_or(u64::MAX)) + .sum::(), + ) + .saturating_add(submission_payload_len(&pending.input)); + if bytes > MAX_PENDING_SUBMISSION_BYTES { + *current = original; + return Err(PendingSubmissionError::ByteLimit); + } + let artifact_refs = current + .pending + .iter() + .map(|pending| submission_artifact_ref_count(&pending.input)) + .sum::() + .saturating_add(submission_artifact_ref_count(&pending.input)); + if artifact_refs > MAX_PENDING_ARTIFACT_REFS { + *current = original; + return Err(PendingSubmissionError::ArtifactLimit); + } + current.pending.push_back(pending.clone()); + if let Err(error) = self.persist_locked(¤t) { + *current = original; + return Err(error); + } + } + Ok(SubmissionAcceptance { + submission_request_id, + submission_id, + disposition, + activation: activate_now.then_some(pending), + }) + } + + pub(crate) fn accept_notification( + &self, + notification_request_id: String, + message: String, + ) -> Result { + if notification_request_id.trim().is_empty() { + return Err(PendingSubmissionError::EmptyRequestId); + } + if notification_request_id.len() > MAX_ACTIVATION_REQUEST_ID_BYTES { + return Err(PendingSubmissionError::RequestIdLimit); + } + let payload_digest = submission_payload_digest(&[Segment::text(message.clone())]); + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if let Some(receipt) = state + .notification_receipts + .iter() + .find(|receipt| receipt.notification_request_id == notification_request_id) + { + if receipt.payload_digest != payload_digest { + return Err(PendingSubmissionError::IdempotencyConflict); + } + return Ok(false); + } + if state + .pending + .len() + .saturating_add(state.pending_notifications.len()) + >= MAX_PENDING_SUBMISSIONS + { + return Err(PendingSubmissionError::CountLimit); + } + let queued_bytes = state + .pending + .iter() + .map(|pending| submission_payload_len(&pending.input)) + .sum::() + .saturating_add( + state + .pending_notifications + .iter() + .map(|pending| u64::try_from(pending.message.len()).unwrap_or(u64::MAX)) + .sum::(), + ) + .saturating_add(u64::try_from(message.len()).unwrap_or(u64::MAX)); + if queued_bytes > MAX_PENDING_SUBMISSION_BYTES { + return Err(PendingSubmissionError::ByteLimit); + } + let original = state.clone(); + let activation_sequence = state.next_activation_sequence; + state.next_activation_sequence = state.next_activation_sequence.saturating_add(1); + state.pending_notifications.push_back(PendingNotification { + notification_request_id: notification_request_id.clone(), + message, + payload_digest: payload_digest.clone(), + accepted_at_ms: segment_log::now_millis(), + activation_sequence, + provenance: WorkerHistoryProvenance::BackendInstruction { + operation_id: Some(notification_request_id.clone()), + }, + }); + state.remember_notification_receipt(NotificationReceipt { + notification_request_id, + payload_digest, + }); + state.revision = state.revision.saturating_add(1); + if let Err(error) = self.persist_locked(&state) { + *state = original; + return Err(error); + } + Ok(true) + } + + pub(crate) fn prepare_next_activation( + &self, + ) -> Result, PendingSubmissionError> { + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if state.activating.is_some() || state.activating_notification.is_some() { + return Ok(None); + } + let submission_sequence = state.pending.front().map(|item| item.activation_sequence); + let notification_sequence = state + .pending_notifications + .front() + .map(|item| item.activation_sequence); + if notification_sequence.is_some() + && (submission_sequence.is_none() || notification_sequence < submission_sequence) + { + let notification = state + .pending_notifications + .pop_front() + .expect("notification sequence came from queue head"); + state.activating_notification = Some(notification.clone()); + state.revision = state.revision.saturating_add(1); + return Ok(Some(PendingActivation::Notification(notification))); + } + if submission_sequence.is_some() { + let pending = state + .pending + .pop_front() + .expect("submission sequence came from queue head"); + state.activating = Some(pending.clone()); + state.revision = state.revision.saturating_add(1); + return Ok(Some(PendingActivation::Submission(pending))); + } + Ok(None) + } + + pub(crate) fn abort_activation(&self, pending: PendingSubmission) { + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if state.activating.as_ref().map(|item| &item.submission_id) != Some(&pending.submission_id) + { + return; + } + state.activating = None; + if pending.was_queued { + state.pending.push_front(pending); + } else { + state + .receipts + .retain(|receipt| receipt.submission_id != pending.submission_id); + } + state.revision = state.revision.saturating_add(1); + } + + pub(crate) fn activation_extension(&self) -> SessionExtension { + let state = self + .state + .lock() + .expect("pending activation state poisoned"); + let mut committed = state.clone(); + if let Some(activating) = &committed.activating + && let Some(receipt) = committed + .receipts + .iter_mut() + .find(|receipt| receipt.submission_id == activating.submission_id) + { + receipt.disposition = protocol::SubmissionDisposition::Started; + } + committed.activating = None; + committed.revision = committed.revision.saturating_add(1); + pending_activation_extension(&committed) + } + + pub(crate) fn finish_activation(&self, submission_id: &str) { + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if state + .activating + .as_ref() + .map(|item| item.submission_id.as_str()) + == Some(submission_id) + { + if let Some(receipt) = state + .receipts + .iter_mut() + .find(|receipt| receipt.submission_id == submission_id) + { + receipt.disposition = protocol::SubmissionDisposition::Started; + } + state.activating = None; + state.revision = state.revision.saturating_add(1); + } + } + + pub(crate) fn notification_activation_extension(&self) -> SessionExtension { + let state = self + .state + .lock() + .expect("pending activation state poisoned"); + let mut committed = state.clone(); + committed.activating_notification = None; + committed.revision = committed.revision.saturating_add(1); + pending_activation_extension(&committed) + } + + pub(crate) fn finish_notification_activation(&self, notification_request_id: &str) { + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if state + .activating_notification + .as_ref() + .map(|item| item.notification_request_id.as_str()) + == Some(notification_request_id) + { + state.activating_notification = None; + state.revision = state.revision.saturating_add(1); + } + } + + pub(crate) fn snapshot(&self) -> protocol::PendingSubmissionsSnapshot { + self.state + .lock() + .expect("pending activation state poisoned") + .snapshot() + } + + pub(crate) fn cancel( + &self, + submission_id: &str, + ) -> Result { + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + let original = state.clone(); + let Some(index) = state + .pending + .iter() + .position(|pending| pending.submission_id == submission_id) + else { + return Err(PendingSubmissionError::NotFound(submission_id.to_owned())); + }; + state.pending.remove(index); + state.revision = state.revision.saturating_add(1); + if let Err(error) = self.persist_locked(&state) { + *state = original; + return Err(error); + } + Ok(state.snapshot()) + } + + pub(crate) fn clear( + &self, + ) -> Result { + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + let original = state.clone(); + state.pending.clear(); + state.pending_notifications.clear(); + state.revision = state.revision.saturating_add(1); + if let Err(error) = self.persist_locked(&state) { + *state = original; + return Err(error); + } + Ok(state.snapshot()) + } +} + +impl PendingSubmissionHandle { + #[cfg(test)] + pub(crate) fn for_test(root: &std::path::Path) -> Self { + let store = session_store::FsStore::new(root).expect("test session store"); + let session_id = session_store::new_session_id(); + let segment_id = session_store::new_segment_id(); + store + .create_segment(session_id, segment_id, &[]) + .expect("test session segment"); + Self { + state: Arc::new(Mutex::new(PendingActivationState::default())), + writer: LogWriterHandle { + store, + state: SegmentState::new(session_id, segment_id, 0), + sink: SegmentLogSink::new(), + in_flight: None, + }, + } + } +} + +/// Type-erased commit handle for the interceptor. Lets the interceptor commit `SystemItem`s without being generic over the /// concrete `Store` type. pub trait SystemItemCommitter: Send + Sync { fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError>; - fn commit_system_item( + fn commit_system_item_with_extensions( &self, item: SystemItem, + extensions: Vec, ) -> Result, StoreError> { let metadata = new_history_metadata( WorkerHistoryProvenance::BackendInstruction { operation_id: None }, @@ -991,6 +1590,7 @@ pub trait SystemItemCommitter: Send + Sync { item, metadata: metadata.clone(), }, + extensions, })?; Ok(HistoryEntry::new(history_item, metadata)) } @@ -1027,8 +1627,6 @@ where } } -pub const WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN: &str = "worker.input-submission.v1"; - #[derive(Clone)] struct PreparedFlowProjection { selector: String, @@ -1049,6 +1647,7 @@ pub struct WorkerSession { session_id: SessionId, revision: u64, history: History, + pending_activations: Arc>, } impl WorkerSession { @@ -1058,9 +1657,39 @@ impl WorkerSession { session_id, revision, history: History::from_entries(entries), + pending_activations: Arc::new(Mutex::new(PendingActivationState::default())), } } + fn restore_pending_activations(&mut self, extensions: &[(String, serde_json::Value)]) { + let Some(payload) = extensions.iter().rev().find_map(|(domain, payload)| { + (domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN).then_some(payload) + }) else { + return; + }; + if let Ok(mut state) = serde_json::from_value::(payload.clone()) { + if let Some(activating) = state.activating.take() { + state.pending.push_front(activating); + state.revision = state.revision.saturating_add(1); + } + if let Some(activating) = state.activating_notification.take() { + state.pending_notifications.push_front(activating); + state.revision = state.revision.saturating_add(1); + } + *self + .pending_activations + .lock() + .expect("pending activation state poisoned") = state; + } + } + + pub fn pending_submissions(&self) -> protocol::PendingSubmissionsSnapshot { + self.pending_activations + .lock() + .expect("pending activation state poisoned") + .snapshot() + } + pub fn session_id(&self) -> SessionId { self.session_id } @@ -1309,6 +1938,21 @@ impl Worker { } } + pub(crate) fn pending_activation_state(&self) -> Arc> { + self.session.pending_activations.clone() + } + + pub(crate) fn pending_submission_handle(&self) -> PendingSubmissionHandle { + PendingSubmissionHandle { + state: self.session.pending_activations.clone(), + writer: self.log_writer_handle(), + } + } + + pub fn pending_submissions(&self) -> protocol::PendingSubmissionsSnapshot { + self.session.pending_submissions() + } + /// Attach a type-erased system-item commit handle. The controller /// calls this once during spawn so the interceptor can commit /// `SystemItem`s directly without owning a generic store handle. @@ -1670,6 +2314,7 @@ impl Worker { }, metadata: skill_metadata.clone(), }, + extensions: Vec::new(), })?; let history_entry = HistoryEntry::new(agen::Item::system_message(body), skill_metadata); let mut annotate = history_annotator( @@ -1960,6 +2605,28 @@ impl Worker { .truncate(loc.session_id, loc.segment_id, truncate_entries)?; self.segment_state.set_entries_written(truncate_entries); self.sink.truncate_silent(truncate_entries); + let pending_state = self + .session + .pending_activations + .lock() + .expect("pending activation state poisoned") + .clone(); + if !pending_state.pending.is_empty() + || pending_state.activating.is_some() + || pending_state.activating_notification.is_some() + || !pending_state.receipts.is_empty() + { + let checkpoint = LogEntry::Extension { + ts: segment_log::now_millis(), + domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(), + payload: serde_json::to_value(&pending_state).map_err(|error| { + RewindError::Invalid(format!( + "serialize pending submissions during rewind: {error}" + )) + })?, + }; + self.commit_entry(checkpoint)?; + } let history_entries = restore_history_entries(loc.session_id, loc.segment_id, &retained) .map_err(|error| RewindError::Invalid(error.into()))?; @@ -2525,7 +3192,7 @@ impl Worker { /// Convenience: run with a single `Segment::Text`. /// /// Equivalent to `run(vec![Segment::text(s)])`. The dumb-client - /// counterpart of [`protocol::Method::run_text`]; primarily for + /// counterpart of [`protocol::Method::submit_text`]; primarily for /// tests and tools that have only a string in hand. pub async fn run_text(&mut self, s: impl Into) -> Result where @@ -3042,6 +3709,7 @@ impl Worker { }, metadata: interrupt_metadata.clone(), }, + extensions: Vec::new(), })?; let interrupt_entry = HistoryEntry::new(agen::Item::system_message(system_note), interrupt_metadata); @@ -4428,6 +5096,22 @@ impl Worker { { initial_entries.push(checkpoint); } + initial_entries.push(LogEntry::Extension { + ts: segment_log::now_millis(), + domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(), + payload: serde_json::to_value( + &*self + .session + .pending_activations + .lock() + .expect("pending activation state poisoned"), + ) + .map_err(|error| { + WorkerError::InvalidState(format!( + "serialize pending submissions during compaction: {error}" + )) + })?, + }); if let Some(flow_state) = self .flow_runtime_state .lock() @@ -5248,6 +5932,9 @@ where history_persistence_wired: false, log_writer: None, }; + worker + .session + .restore_pending_activations(&state.extensions); worker.apply_permissions_from_manifest(); worker.apply_prune_from_manifest(); worker.write_worker_metadata_active(SegmentLocation { @@ -8453,6 +9140,164 @@ mod build_summary_prompt_tests { ); } + #[test] + fn pending_submission_queue_is_durable_idempotent_and_bounded() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let input = vec![Segment::text("queued")]; + let accepted = handle + .accept("request-1".into(), input.clone(), false) + .unwrap(); + assert_eq!( + accepted.disposition, + protocol::SubmissionDisposition::Queued + ); + assert_eq!(handle.snapshot().submissions.len(), 1); + + let replay = handle + .accept("request-1".into(), input.clone(), false) + .unwrap(); + assert_eq!(replay.submission_id, accepted.submission_id); + assert!(replay.activation.is_none()); + assert_eq!(handle.snapshot().submissions.len(), 1); + assert!(matches!( + handle.accept("request-1".into(), vec![Segment::text("different")], false), + Err(PendingSubmissionError::IdempotencyConflict) + )); + + let entries = handle + .writer + .store + .read_all( + handle.writer.state.session_id(), + handle.writer.state.segment_id(), + ) + .unwrap(); + let payload = entries + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN => { + Some(payload.clone()) + } + _ => None, + }) + .unwrap(); + let restored: PendingActivationState = serde_json::from_value(payload).unwrap(); + assert_eq!(restored.pending.len(), 1); + assert_eq!(restored.pending[0].submission_id, accepted.submission_id); + + let snapshot = handle.cancel(&accepted.submission_id).unwrap(); + assert!(snapshot.submissions.is_empty()); + assert!(matches!( + handle.cancel(&accepted.submission_id), + Err(PendingSubmissionError::NotFound(_)) + )); + + for index in 0..MAX_PENDING_SUBMISSIONS { + handle + .accept( + format!("limit-{index}"), + vec![Segment::text(format!("value-{index}"))], + false, + ) + .unwrap(); + } + assert!(matches!( + handle.accept("over-limit".into(), vec![Segment::text("too much")], false), + Err(PendingSubmissionError::CountLimit) + )); + assert_eq!(handle.snapshot().submissions.len(), MAX_PENDING_SUBMISSIONS); + let cleared = handle.clear().unwrap(); + assert!(cleared.submissions.is_empty()); + assert_eq!(cleared.notification_count, 0); + } + + #[test] + fn notification_and_submit_share_activation_order_and_notification_dedupes() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + assert!( + handle + .accept_notification("notification-1".into(), "notice".into()) + .unwrap() + ); + assert!( + !handle + .accept_notification("notification-1".into(), "notice".into()) + .unwrap() + ); + assert!(matches!( + handle.accept_notification("notification-1".into(), "different".into()), + Err(PendingSubmissionError::IdempotencyConflict) + )); + handle + .accept("request-1".into(), vec![Segment::text("submit")], false) + .unwrap(); + + let first = handle.prepare_next_activation().unwrap().unwrap(); + assert!(matches!( + first, + PendingActivation::Notification(PendingNotification { ref message, .. }) + if message == "notice" + )); + let committed = handle.notification_activation_extension(); + let committed_state: PendingActivationState = + serde_json::from_value(committed.payload).unwrap(); + assert!(committed_state.pending_notifications.is_empty()); + assert!(committed_state.activating_notification.is_none()); + handle.finish_notification_activation("notification-1"); + let second = handle.prepare_next_activation().unwrap().unwrap(); + assert!(matches!(second, PendingActivation::Submission(_))); + } + + #[test] + fn restoring_an_in_flight_activation_requeues_it_at_the_fifo_head() { + let mut session = WorkerSession::new(session_store::new_session_id(), Vec::new()); + let state = PendingActivationState { + revision: 4, + next_activation_sequence: 2, + activating: Some(PendingSubmission { + submission_request_id: "request-1".into(), + submission_id: "submission-1".into(), + payload_digest: submission_payload_digest(&[Segment::text("first")]), + accepted_at_ms: 1, + activation_sequence: 0, + provenance: WorkerHistoryProvenance::LegacyUnknown, + was_queued: false, + input: vec![Segment::text("first")], + }), + activating_notification: None, + pending: VecDeque::from([PendingSubmission { + submission_request_id: "request-2".into(), + submission_id: "submission-2".into(), + payload_digest: submission_payload_digest(&[Segment::text("second")]), + accepted_at_ms: 2, + activation_sequence: 1, + provenance: WorkerHistoryProvenance::LegacyUnknown, + was_queued: true, + input: vec![Segment::text("second")], + }]), + pending_notifications: VecDeque::new(), + receipts: VecDeque::new(), + notification_receipts: VecDeque::new(), + }; + session.restore_pending_activations(&[( + SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.into(), + serde_json::to_value(state).unwrap(), + )]); + let state = session + .pending_activations + .lock() + .expect("pending activation state poisoned"); + assert!(state.activating.is_none()); + assert_eq!(state.pending.len(), 2); + assert_eq!(state.pending[0].submission_id, "submission-1"); + assert_eq!(state.pending[1].submission_id, "submission-2"); + } + fn minimal_manifest() -> WorkerManifest { let toml_str = r#" [worker] diff --git a/crates/worker/tests/compact_events_test.rs b/crates/worker/tests/compact_events_test.rs index 47fbd351..8265f9e2 100644 --- a/crates/worker/tests/compact_events_test.rs +++ b/crates/worker/tests/compact_events_test.rs @@ -630,7 +630,10 @@ async fn controller_compact_method_emits_start_and_done() { let mut rx = handle.subscribe(); handle - .send(Method::run_text("seed history")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "seed history", + )) .await .expect("send run"); loop { diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 403ca3cb..6faa2fdd 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -617,7 +617,13 @@ async fn feature_flags_default_to_core_tool_surface_only() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; @@ -672,7 +678,13 @@ permission = "write" let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; @@ -758,7 +770,13 @@ permission = "write" let worker = make_worker_with_pwd_and_manifest(client, &manifest).await.0; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; @@ -814,7 +832,13 @@ async fn builtin_orchestrator_exposes_worker_remove_and_workdir_delete() { .await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; let installed = request_tool_names(&request); @@ -863,7 +887,13 @@ permission = "write" .0; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; @@ -916,7 +946,13 @@ permission = "write" ) .await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; let names = request_tool_names(&request); @@ -963,7 +999,13 @@ async fn run_end_returns_to_idle_without_busy_status() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); let mut saw_run_end = false; let mut saw_idle_status = false; @@ -1005,7 +1047,13 @@ async fn provider_stream_error_records_run_errored() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("ping")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "ping", + )) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( @@ -1054,7 +1102,10 @@ async fn snapshot_includes_user_input_for_in_flight_turn() { let mut events = handle.subscribe(); handle - .send(Method::run_text("hello in-flight")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "hello in-flight", + )) .await .unwrap(); tokio::time::timeout(std::time::Duration::from_secs(2), async { @@ -1119,7 +1170,13 @@ async fn attach_snapshot_includes_current_status() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path()) @@ -1157,7 +1214,13 @@ async fn run_updates_shared_state_to_idle_after_completion() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); // Wait for the run to complete tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -1171,7 +1234,13 @@ async fn run_populates_history() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -1189,7 +1258,13 @@ async fn events_are_broadcast() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); let mut saw_turn_start = false; let mut saw_text_delta = false; @@ -1224,10 +1299,8 @@ async fn events_are_broadcast() { } #[tokio::test] -async fn double_run_returns_error() { - // Keep the first turn in-flight until the test drops the handle. A - // finite stream can finish before the second Method reaches the - // controller in the full test suite, making this assertion racy. +async fn submit_while_running_is_durably_queued() { + // Keep the first turn in-flight until the second Submit is accepted. let events = vec![ LlmEvent::text_block_start(0), LlmEvent::text_delta(0, "slow..."), @@ -1237,35 +1310,44 @@ async fn double_run_returns_error() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - // Send first run and wait until the controller has entered Running. - handle.send(Method::run_text("first")).await.unwrap(); + handle + .send(Method::submit_text("request-first", "first")) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; + handle + .send(Method::submit_text("request-second", "second")) + .await + .unwrap(); - // Now the second run must be rejected by drive_turn's live Method arm. - handle.send(Method::run_text("second")).await.unwrap(); - - // Look for the error event - let mut saw_already_running = false; let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); - loop { - tokio::select! { - event = rx.recv() => { - match event { - Ok(Event::Error { code, .. }) => { - if code == worker::ErrorCode::AlreadyRunning { - saw_already_running = true; - break; - } - } - Err(_) => break, - _ => {} - } + let mut accepted = None; + let mut pending_count = None; + while tokio::time::Instant::now() < deadline { + match tokio::time::timeout(std::time::Duration::from_millis(100), rx.recv()).await { + Ok(Ok(Event::SubmissionAccepted { + submission_request_id, + disposition, + .. + })) if submission_request_id == "request-second" => accepted = Some(disposition), + Ok(Ok(Event::PendingSubmissionsChanged { pending })) + if pending.submissions.len() == 1 => + { + pending_count = Some(1) } - _ = tokio::time::sleep_until(deadline) => break, + Ok(Ok(Event::Error { code, message })) if code == worker::ErrorCode::AlreadyRunning => { + panic!("Submit was busy-rejected: {message}") + } + _ => {} + } + if accepted.is_some() && pending_count.is_some() { + break; } } - assert!(saw_already_running, "should see already_running error"); + assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued)); + assert_eq!(pending_count, Some(1)); + handle.send(Method::Pause).await.unwrap(); } #[tokio::test] @@ -1353,7 +1435,8 @@ async fn run_with_paste_segment_inlines_content_and_emits_typed_user_message() { protocol::Segment::text(" thanks"), ]; handle - .send(Method::Run { + .send(Method::Submit { + submission_request_id: protocol::new_submission_request_id(), input: segments.clone(), }) .await @@ -1425,7 +1508,13 @@ async fn run_with_resolvable_file_ref_attaches_system_message_after_user() { path: "notes.md".into(), }, ]; - handle.send(Method::Run { input: segments }).await.unwrap(); + handle + .send(Method::Submit { + submission_request_id: protocol::new_submission_request_id(), + input: segments, + }) + .await + .unwrap(); // Wait for the turn to complete. let mut rx = handle.subscribe(); @@ -1473,7 +1562,8 @@ async fn run_with_file_ref_uses_manifest_file_upload_limit() { let handle = spawn_controller(worker).await; handle - .send(Method::Run { + .send(Method::Submit { + submission_request_id: protocol::new_submission_request_id(), input: vec![protocol::Segment::FileRef { path: "long.txt".into(), }], @@ -1526,7 +1616,13 @@ async fn run_with_unresolved_segment_emits_alert_and_placeholder() { path: "src/lib.rs".into(), }, ]; - handle.send(Method::Run { input: segments }).await.unwrap(); + handle + .send(Method::Submit { + submission_request_id: protocol::new_submission_request_id(), + input: segments, + }) + .await + .unwrap(); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); let mut saw_alert_for_file_ref = false; @@ -1574,6 +1670,7 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() { handle .send(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), message: "turn finished".into(), auto_run: true, }) @@ -1614,6 +1711,19 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() { saw_notify_in_mirror, "Method::Notify should commit a SystemItem::Notification entry; mirror = {entries:?}" ); + let queue_checkpoint_is_atomic = entries.iter().any(|entry| match entry { + LogEntry::AnnotatedSystemItem { extensions, .. } => extensions.iter().any(|extension| { + extension.domain == "worker.pending_activations.v1" + && extension.payload["pending_notifications"] + .as_array() + .is_some_and(Vec::is_empty) + }), + _ => false, + }); + assert!( + queue_checkpoint_is_atomic, + "notification history and queue claim must share one log entry" + ); // Exactly one request was made; it must contain the formatted // notification as one of the items (committed to history by @@ -1662,6 +1772,7 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() { handle .send(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), message: "progress snapshot".into(), auto_run: false, }) @@ -1675,7 +1786,13 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() { "weak Notify must not stage RunForNotification while idle" ); - handle.send(Method::run_text("continue")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "continue", + )) + .await + .unwrap(); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); loop { if !client_for_assert.captured_requests().is_empty() { @@ -1855,9 +1972,16 @@ async fn notify_while_running_does_not_emit_already_running_error() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("start")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "start", + )) + .await + .unwrap(); handle .send(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), message: "ping".into(), auto_run: true, }) @@ -1924,7 +2048,13 @@ async fn socket_run_receives_events() { let mut writer = JsonLineWriter::new(writer); // Send run method via socket - writer.write(&Method::run_text("Hello")).await.unwrap(); + writer + .write(&Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); // Collect events let mut saw_turn_start = false; @@ -2231,7 +2361,13 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) + .await + .unwrap(); // Wait for the partial text_delta to confirm the first stream is // live before we pause. @@ -2320,7 +2456,7 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { assert!(!has_tool_call, "no orphan tool_call in history"); } -/// Paused with an orphan `tool_use` in history + a fresh `Method::Run` +/// Paused with an orphan `tool_use` in history + a fresh `Method::Submit` /// must produce a wire-valid next LLM request: the orphan is closed /// with a synthetic `tool_result`, a system note is inserted, and the /// new user input is appended. @@ -2357,7 +2493,13 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("first")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "first", + )) + .await + .unwrap(); // Wait for ToolCallDone — the ToolCall is committed to history // right before the Engine enters tool execution and pends. @@ -2388,7 +2530,13 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() { // `last_run_interrupted` and runs its interrupt-prep step, which // closes the orphan + injects a system note before the fresh user // message. - handle.send(Method::run_text("new request")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "new request", + )) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2519,7 +2667,13 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("first")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "first", + )) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2587,7 +2741,10 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { ); handle - .send(Method::run_text("fresh request")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "fresh request", + )) .await .unwrap(); assert!( @@ -2676,7 +2833,13 @@ async fn empty_turn_cancel_rolls_back_submit_entries_and_emits_signal() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("rollback me")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "rollback me", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; handle.send(Method::Cancel).await.unwrap(); @@ -2709,7 +2872,10 @@ async fn empty_turn_pause_rolls_back_and_snapshot_does_not_restore_input() { let mut rx = handle.subscribe(); handle - .send(Method::run_text("pause rollback")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "pause rollback", + )) .await .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; @@ -2743,7 +2909,13 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("first kept")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "first kept", + )) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2757,7 +2929,10 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() { wait_for_status(&handle, WorkerStatus::Idle).await; handle - .send(Method::run_text("second rolled back")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "second rolled back", + )) .await .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; @@ -2804,7 +2979,10 @@ async fn pause_after_assistant_token_does_not_rollback() { let mut rx = handle.subscribe(); handle - .send(Method::run_text("keep this turn")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "keep this turn", + )) .await .unwrap(); assert!( diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 7f530a8d..f86a1d00 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -533,7 +533,7 @@ fn initial_worker_input(segments: &[Segment]) -> Option { Some(EmbeddedWorkerInput { kind: EmbeddedWorkerInputKind::User, content: Segment::flatten_to_text(segments), - submission_id: None, + submission_request_id: None, segments: Some(segments.to_vec()), }) } @@ -2625,7 +2625,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer, }, content: request.content, - submission_id: None, + submission_request_id: None, segments: request.segments, }; match self.runtime.send_input(&worker_ref, input) { @@ -3726,7 +3726,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer, }, content: request.content, - submission_id: None, + submission_request_id: None, segments: request.segments, }; match self.post_json::<_, RuntimeHttpWorkerInputResponse>( diff --git a/docs/development/work-items.md b/docs/development/work-items.md index 61c174ff..34bec74f 100644 --- a/docs/development/work-items.md +++ b/docs/development/work-items.md @@ -286,7 +286,7 @@ User triggers a Ticket action in yoi panel -> client Ticket role launcher reads .yoi/workspace.toml [ticket] settings -> launcher selects the role Profile -> launcher spawns the role Worker - -> launcher sends Method::Run with Text segments + -> launcher sends Method::Submit with Text segments -> launcher waits for run-acceptance evidence -> Dashboard reports success/failure ``` diff --git a/docs/report/test-validity-20260612/pod.md b/docs/report/test-validity-20260612/pod.md index d5c5cfc8..087ea4d2 100644 --- a/docs/report/test-validity-20260612/pod.md +++ b/docs/report/test-validity-20260612/pod.md @@ -34,7 +34,7 @@ - いくつかの async/socket tests はまだ fixed sleeps と process-wide environment mutation (`YOI_RUNTIME_DIR`, `YOI_HOME`, `XDG_RUNTIME_DIR`) に依存している。一部ファイルでは env-changing tests を guards で serialize しているが、fixed timing と global env は高負荷 CI や parallel execution 下で flakiness risk のまま。 - Real provider wire behavior は意図的に `pod` の外側だが、crate と streaming edge cases の interaction はまだ大部分が mock されている。tests は重要な `Worker` outcomes をカバーしているが、malformed/partial provider streams を real provider adapter 経由では exercise していない。 - Prompt tests は数が多く有用だが、一部は behavior-coupled というより prose-coupled である。critical safety wording に対して、その文字列を意図的に stable contract として扱う場合だけこれは許容できる。そうでなければ maintenance noise になる。 -- startup profile resolution、socket server、`Method::Run`、session persistence、shutdown、restore を跨ぐ full lifecycle integration は slice ごとにしかカバーされておらず、1 つの scenario としてはカバーされていない。 +- startup profile resolution、socket server、`Method::Submit`、session persistence、shutdown、restore を跨ぐ full lifecycle integration は slice ごとにしかカバーされておらず、1 つの scenario としてはカバーされていない。 ## 追加を提案するもの diff --git a/web/workspace/src/lib/generated/protocol.ts b/web/workspace/src/lib/generated/protocol.ts index 8cd81901..1e744d20 100644 --- a/web/workspace/src/lib/generated/protocol.ts +++ b/web/workspace/src/lib/generated/protocol.ts @@ -103,7 +103,13 @@ entry_id: string, */ timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array, } & ({ "kind": "user_input", segments: Array, } | { "kind": "message", role: SessionMessageRole, content: Array, } | { "kind": "tool_call", call_id: string, name: string, arguments: string, } | { "kind": "tool_result", call_id: string, summary: string, content?: string | null, is_error: boolean, attachments?: Array, } | { "kind": "system_item", item_kind: string, content: string, data?: unknown, } | { "kind": "run_error", message: string, }); -export type SessionSnapshot = { entries: Array, }; +export type PendingSubmissionSummary = { submission_id: string, accepted_at_ms: number, segment_count: number, byte_len: number, }; + +export type PendingSubmissionsSnapshot = { revision: number, notification_count: number, submissions: Array, }; + +export type SubmissionDisposition = "started" | "queued"; + +export type SessionSnapshot = { pending_submissions: PendingSubmissionsSnapshot, entries: Array, }; export type InternalWorkerKind = "sub_worker" | { "service": { kind: string, } }; @@ -225,9 +231,9 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod }); -export type Method = { "method": "run", "params": { input: Array, } } | { "method": "notify", "params": { message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; +export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, } } | { "method": "clear_pending_submissions" } | { "method": "continue_pending" } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; -export type Event = { "event": "user_message", "data": { segments: Array, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number, +export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number, /** * The attempt that just failed. 1 origin. */ diff --git a/web/workspace/src/lib/workspace/console/model.test.ts b/web/workspace/src/lib/workspace/console/model.test.ts index e2dfdceb..da4f70d5 100644 --- a/web/workspace/src/lib/workspace/console/model.test.ts +++ b/web/workspace/src/lib/workspace/console/model.test.ts @@ -2150,6 +2150,7 @@ Deno.test("snapshot restores TaskStore state from system history", () => { const event = snapshotEvent("/repo"); if (event.event !== "snapshot") throw new Error("snapshot fixture expected"); event.data.session = { + pending_submissions: { revision: 0, notification_count: 0, submissions: [] }, entries: [{ entry_id: "task-reminder-1", timestamp: 1, diff --git a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts index 869603f3..b031f1d0 100644 --- a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts +++ b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts @@ -1059,3 +1059,25 @@ Deno.test("Web Console switches main and direct SubWorker views from the Tasks r "Worker view selection should expose only direct SubWorker session identities with main fallback", ); }); + +Deno.test("Web Console uses Notify while running and exposes durable pending controls", async () => { + const consolePage = await Deno.readTextFile( + new URL( + "./../../../routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte", + import.meta.url, + ), + ); + + for (const token of [ + 'method: "submit"', + 'method: "notify"', + "notification_request_id: crypto.randomUUID()", + "submission_request_id: crypto.randomUUID()", + 'payload.event === "pending_submissions_changed"', + 'method: "cancel_pending_submission"', + 'method: "clear_pending_submissions"', + 'method: "continue_pending"', + ]) { + assert(consolePage.includes(token), `missing durable pending control token: ${token}`); + } +}); diff --git a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte index 03efcf8d..6c50e2ae 100644 --- a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte @@ -31,7 +31,13 @@ type ConsoleViewMode, type ConsoleViewScroll, } from "$lib/workspace/console/model"; - import type { Event as ProtocolEvent, Method as ProtocolMethod, RewindTarget, Segment } from "$lib/generated/protocol"; + import type { + Event as ProtocolEvent, + Method as ProtocolMethod, + PendingSubmissionsSnapshot, + RewindTarget, + Segment, + } from "$lib/generated/protocol"; import { MAX_FILES_PER_SUBMISSION, uploadAttachment, @@ -152,6 +158,12 @@ "connecting", ); let protocolSubscription: WorkspaceMultiplexerSubscription | null = null; + let pendingSubmissions = $state({ + revision: 0, + notification_count: 0, + submissions: [], + }); + let pendingSubmissionItems = $derived(pendingSubmissions.submissions ?? []); let pendingCompletionRequest: { resolve: (entries: ComposerCompletionEntry[]) => void; reject: (error: Error) => void; @@ -334,6 +346,13 @@ function handleIncomingProtocolEvent(payload: ProtocolEvent) { handleProtocolCommandEvent(payload); + if (payload.event === "snapshot") { + pendingSubmissions = payload.data.session.pending_submissions; + } else if (payload.event === "segment_rotated") { + pendingSubmissions = payload.data.session.pending_submissions; + } else if (payload.event === "pending_submissions_changed") { + pendingSubmissions = payload.data.pending; + } if (payload.event === "error") { queueObservationDiagnostic({ code: payload.data.code, @@ -555,9 +574,20 @@ ): ProtocolMethod { switch (request.kind) { case "user": + if (workerRunning) { + return { + method: "notify", + params: { + notification_request_id: crypto.randomUUID(), + message: request.content, + auto_run: true, + }, + }; + } return { - method: "run", + method: "submit", params: { + submission_request_id: crypto.randomUUID(), input: request.segments ?? [ { kind: "text", content: request.content }, ], @@ -566,7 +596,11 @@ case "notify": return { method: "notify", - params: { message: request.content, auto_run: true }, + params: { + notification_request_id: crypto.randomUUID(), + message: request.content, + auto_run: true, + }, }; case "compact": return { method: "compact" }; @@ -779,7 +813,7 @@ composerInputElement?.recordHistory(value); composerInputElement?.clear(); attachments = []; - if (method.method === "run" || method.method === "notify") { + if (method.method === "submit" || method.method === "notify") { liveWorkerState = "running"; } composerNotice = "Sent through Worker protocol."; @@ -1722,6 +1756,50 @@ {/if} + {#if pendingSubmissionItems.length > 0 || pendingSubmissions.notification_count > 0} +
+ + Pending activations ({pendingSubmissionItems.length} submissions · {pendingSubmissions.notification_count} notifications) + +
    + {#each pendingSubmissionItems as submission (submission.submission_id)} +
  1. + {submission.submission_id} + {submission.segment_count} segments · {submission.byte_len} bytes + +
    2. + {/each} +
+ + +
+ {/if} + {#if workerRunning} Date: Sat, 5 Sep 2026 23:16:19 +0900 Subject: [PATCH 02/39] fix: fence pending queue controls --- crates/protocol/src/lib.rs | 12 ++- crates/tui/src/app.rs | 43 ++++++++++ crates/tui/src/console/mod.rs | 70 ++++++++++++++- crates/tui/src/ui.rs | 5 +- crates/worker/src/controller.rs | 86 ++++++++++++------- crates/worker/src/worker.rs | 72 ++++++++++++++-- crates/worker/tests/controller_test.rs | 31 ++++++- web/workspace/src/lib/generated/protocol.ts | 4 +- .../src/lib/workspace/console/model.test.ts | 7 +- .../console/worker-console.ui.test.ts | 13 +++ .../workers/[workerId]/console/+page.svelte | 72 ++++++++++++---- 11 files changed, 347 insertions(+), 68 deletions(-) diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index 7171461c..e3330ba0 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -73,12 +73,18 @@ pub enum Method { /// are immutable and therefore cannot be cancelled here. CancelPendingSubmission { submission_id: String, + expected_revision: u64, }, /// Remove every queued submission while preserving the active run. - ClearPendingSubmissions, + ClearPendingSubmissions { + expected_revision: u64, + }, /// Activate the next queued submission while the Worker is idle. This is an /// explicit recovery operation and never resumes a paused run implicitly. - ContinuePending, + ContinuePending { + expected_revision: u64, + expected_head_id: String, + }, Resume, Cancel, /// Stop the in-flight turn and transition to `Paused`. @@ -555,6 +561,8 @@ pub struct PendingSubmissionsSnapshot { #[serde(default)] pub notification_count: u32, #[serde(default)] + pub head_id: Option, + #[serde(default)] pub submissions: Vec, } diff --git a/crates/tui/src/app.rs b/crates/tui/src/app.rs index 2fecd888..d23430c2 100644 --- a/crates/tui/src/app.rs +++ b/crates/tui/src/app.rs @@ -754,6 +754,28 @@ impl App { Some(self.method_for_run(segments)) } + pub fn submit_notify_input(&mut self) -> Option { + let segments = self.input.submit_segments(); + if segments_are_blank(&segments) { + return None; + } + if segments + .iter() + .any(|segment| matches!(segment, Segment::UploadedFile { .. })) + { + self.push_error("Notify accepts text only; remove attachments or queue a Submit."); + return None; + } + let message = Segment::flatten_to_text(&segments); + self.record_input_history(segments); + self.input.clear(); + Some(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), + message, + auto_run: true, + }) + } + pub fn restore_unsent_run(&mut self, method: &Method) { let Method::Submit { input, .. } = method else { return; @@ -890,6 +912,26 @@ impl App { } } + pub fn continue_pending_method(&self) -> Option { + Some(Method::ContinuePending { + expected_revision: self.pending_submissions.revision, + expected_head_id: self.pending_submissions.head_id.clone()?, + }) + } + + pub fn clear_pending_method(&self) -> Method { + Method::ClearPendingSubmissions { + expected_revision: self.pending_submissions.revision, + } + } + + pub fn cancel_pending_method(&self, submission_id: String) -> Method { + Method::CancelPendingSubmission { + submission_id, + expected_revision: self.pending_submissions.revision, + } + } + pub fn next_queued_input_preview(&self) -> Option<&str> { self.pending_submissions .submissions @@ -3410,6 +3452,7 @@ mod completion_flow_tests { pending: protocol::PendingSubmissionsSnapshot { revision: 3, notification_count: 0, + head_id: Some("submission-1".into()), submissions: vec![protocol::PendingSubmissionSummary { submission_id: "submission-1".into(), accepted_at_ms: 7, diff --git a/crates/tui/src/console/mod.rs b/crates/tui/src/console/mod.rs index 3f0d6ddd..a7ff703e 100644 --- a/crates/tui/src/console/mod.rs +++ b/crates/tui/src/console/mod.rs @@ -1150,13 +1150,27 @@ fn handle_key(app: &mut App, key: KeyEvent) -> Option { app.clear_command_input(); Some(None) } + KeyCode::Char(c) + if c.eq_ignore_ascii_case(&'d') && alt && !ctrl && !app.is_command_mode() => + { + Some( + app.next_queued_input_preview() + .map(str::to_owned) + .map(|submission_id| app.cancel_pending_method(submission_id)), + ) + } + KeyCode::Char(c) + if c.eq_ignore_ascii_case(&'n') && alt && !ctrl && !app.is_command_mode() => + { + Some(app.submit_notify_input()) + } KeyCode::Char(c) if c.eq_ignore_ascii_case(&'q') && alt && !ctrl && !app.is_command_mode() => { - Some(Some(Method::ContinuePending)) + Some(app.continue_pending_method()) } KeyCode::Char(c) if c.eq_ignore_ascii_case(&'c') && alt && !ctrl => { - Some(Some(Method::ClearPendingSubmissions)) + Some(Some(app.clear_pending_method())) } KeyCode::Char('c') if ctrl => Some(handle_pause_or_quit(app)), KeyCode::Char('x') if ctrl => Some(handle_cancel_or_shutdown(app)), @@ -1976,6 +1990,29 @@ mod tests { assert_eq!(input_text(&app), ""); } + #[test] + fn running_alt_n_sends_explicit_notify_without_implicit_submit_conversion() { + let mut app = App::new("test".into()); + app.set_worker_status(WorkerStatus::Running); + for character in "progress".chars() { + app.insert_char(character); + } + + let method = handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('n'), KeyModifiers::ALT), + ); + assert!(matches!( + method, + Some(Method::Notify { + ref message, + auto_run: true, + .. + }) if message == "progress" + )); + assert_eq!(input_text(&app), ""); + } + #[test] fn pending_queue_shortcuts_send_worker_operations() { let mut app = App::new("test".into()); @@ -1983,6 +2020,7 @@ mod tests { pending: protocol::PendingSubmissionsSnapshot { revision: 2, notification_count: 0, + head_id: Some("submission-1".into()), submissions: vec![protocol::PendingSubmissionSummary { submission_id: "submission-1".into(), accepted_at_ms: 1, @@ -1996,14 +2034,37 @@ mod tests { &mut app, KeyEvent::new(KeyCode::Char('q'), KeyModifiers::ALT), ); - assert!(matches!(continue_next, Some(Method::ContinuePending))); + assert!(matches!( + continue_next, + Some(Method::ContinuePending { + expected_revision: 2, + ref expected_head_id, + }) if expected_head_id == "submission-1" + )); assert_eq!(app.queued_input_count(), 1); + let cancel = handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('d'), KeyModifiers::ALT), + ); + assert!(matches!( + cancel, + Some(Method::CancelPendingSubmission { + expected_revision: 2, + ref submission_id, + }) if submission_id == "submission-1" + )); + let clear = handle_key( &mut app, KeyEvent::new(KeyCode::Char('c'), KeyModifiers::ALT), ); - assert!(matches!(clear, Some(Method::ClearPendingSubmissions))); + assert!(matches!( + clear, + Some(Method::ClearPendingSubmissions { + expected_revision: 2 + }) + )); assert_eq!(app.queued_input_count(), 1); } @@ -2014,6 +2075,7 @@ mod tests { pending: protocol::PendingSubmissionsSnapshot { revision: 2, notification_count: 0, + head_id: Some("submission-1".into()), submissions: vec![protocol::PendingSubmissionSummary { submission_id: "submission-1".into(), accepted_at_ms: 1, diff --git a/crates/tui/src/ui.rs b/crates/tui/src/ui.rs index 577fafe9..58b790f1 100644 --- a/crates/tui/src/ui.rs +++ b/crates/tui/src/ui.rs @@ -1880,7 +1880,7 @@ fn actionbar_left_item(app: &App, now: Instant) -> Option<(String, Style)> { } if app.queued_input_count() > 0 { return Some(( - "Alt-q continue queued Alt-c clear queued".to_string(), + "Alt-n notify Alt-q continue Alt-d cancel queued Alt-c clear queued".to_string(), Style::default().fg(Color::DarkGray), )); } @@ -2144,6 +2144,7 @@ mod tests { pending: protocol::PendingSubmissionsSnapshot { revision: 1, notification_count: 0, + head_id: Some(id.into()), submissions: vec![protocol::PendingSubmissionSummary { submission_id: id.into(), accepted_at_ms: 1, @@ -2303,7 +2304,7 @@ mod tests { set_pending_submission(&mut app, "submission-1"); assert_eq!( actionbar_left_item(&app, now).map(|(text, _)| text), - Some("Alt-q continue queued Alt-c clear queued".into()) + Some("Alt-n notify Alt-q continue Alt-d cancel queued Alt-c clear queued".into()) ); app.enter_command_mode(); diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 2794b35c..4834abd0 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -232,8 +232,9 @@ enum PendingRun { fn prepare_pending_run( pending_submissions: &crate::worker::PendingSubmissionHandle, notify_buffer: &NotifyBuffer, + fence: Option<(u64, &str)>, ) -> Result, crate::worker::PendingSubmissionError> { - Ok(match pending_submissions.prepare_next_activation()? { + Ok(match pending_submissions.prepare_next_activation(fence)? { Some(crate::worker::PendingActivation::Submission(submission)) => { Some(PendingRun::Submit(submission)) } @@ -1420,7 +1421,7 @@ async fn controller_loop( } if !shutdown && may_drain_pending && new_status == WorkerStatus::Idle { - match prepare_pending_run(&pending_submissions, ¬ify_buffer) { + match prepare_pending_run(&pending_submissions, ¬ify_buffer, None) { Ok(Some(next)) => { pending = Some(next); new_status = WorkerStatus::Running; @@ -1505,17 +1506,18 @@ async fn controller_loop( if auto_run { match pending_submissions.accept_notification(notification_request_id, message) { - Ok(true) => match prepare_pending_run(&pending_submissions, ¬ify_buffer) - { - Ok(Some(next)) => pending = Some(next), - Ok(None) => {} - Err(error) => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::Internal, - message: error.to_string(), - }); + Ok(true) => { + match prepare_pending_run(&pending_submissions, ¬ify_buffer, None) { + Ok(Some(next)) => pending = Some(next), + Ok(None) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } } - }, + } Ok(false) => {} Err(error) => { let _ = working_event_tx.send(Event::Error { @@ -1534,8 +1536,24 @@ async fn controller_loop( pending: pending_submissions.snapshot(), }); } - Method::CancelPendingSubmission { submission_id } => { - match pending_submissions.cancel(&submission_id) { + Method::CancelPendingSubmission { + submission_id, + expected_revision, + } => match pending_submissions.cancel(&submission_id, expected_revision) { + Ok(pending_snapshot) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_snapshot, + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + }, + Method::ClearPendingSubmissions { expected_revision } => { + match pending_submissions.clear(expected_revision) { Ok(pending_snapshot) => { let _ = working_event_tx.send(Event::PendingSubmissionsChanged { pending: pending_snapshot, @@ -1549,21 +1567,22 @@ async fn controller_loop( } } } - Method::ClearPendingSubmissions => match pending_submissions.clear() { - Ok(pending_snapshot) => { - let _ = working_event_tx.send(Event::PendingSubmissionsChanged { - pending: pending_snapshot, - }); - } - Err(error) => { + Method::ContinuePending { + expected_revision, + expected_head_id, + } => { + if shared_state.get_status() != WorkerStatus::Idle { let _ = working_event_tx.send(Event::Error { - code: ErrorCode::Internal, - message: error.to_string(), + code: ErrorCode::InvalidRequest, + message: "ContinuePending requires an idle Worker; Resume or Cancel a paused run first".into(), }); + continue; } - }, - Method::ContinuePending => { - match prepare_pending_run(&pending_submissions, ¬ify_buffer) { + match prepare_pending_run( + &pending_submissions, + ¬ify_buffer, + Some((expected_revision, &expected_head_id)), + ) { Ok(Some(next)) => pending = Some(next), Ok(None) => { let _ = working_event_tx.send(Event::Error { @@ -2077,7 +2096,7 @@ where } } } - Some(Method::Resume | Method::ContinuePending) => { + Some(Method::Resume | Method::ContinuePending { .. }) => { let _ = working_event_tx.send(Event::Error { code: ErrorCode::AlreadyRunning, message: "Worker is already executing a turn".into(), @@ -2088,8 +2107,11 @@ where pending: pending_submissions.snapshot(), }); } - Some(Method::CancelPendingSubmission { submission_id }) => { - match pending_submissions.cancel(&submission_id) { + Some(Method::CancelPendingSubmission { + submission_id, + expected_revision, + }) => { + match pending_submissions.cancel(&submission_id, expected_revision) { Ok(pending) => { let _ = working_event_tx.send(Event::PendingSubmissionsChanged { pending }); } @@ -2101,14 +2123,14 @@ where } } } - Some(Method::ClearPendingSubmissions) => { - match pending_submissions.clear() { + Some(Method::ClearPendingSubmissions { expected_revision }) => { + match pending_submissions.clear(expected_revision) { Ok(pending) => { let _ = working_event_tx.send(Event::PendingSubmissionsChanged { pending }); } Err(error) => { let _ = working_event_tx.send(Event::Error { - code: ErrorCode::Internal, + code: ErrorCode::InvalidRequest, message: error.to_string(), }); } diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index cc93bb3a..3bfb9230 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -130,9 +130,20 @@ pub(crate) struct PendingActivationState { impl PendingActivationState { pub(crate) fn snapshot(&self) -> protocol::PendingSubmissionsSnapshot { + let head_id = match (self.pending.front(), self.pending_notifications.front()) { + (Some(submission), Some(notification)) + if notification.activation_sequence < submission.activation_sequence => + { + Some(notification.notification_request_id.clone()) + } + (Some(submission), _) => Some(submission.submission_id.clone()), + (None, Some(notification)) => Some(notification.notification_request_id.clone()), + (None, None) => None, + }; protocol::PendingSubmissionsSnapshot { revision: self.revision, notification_count: u32::try_from(self.pending_notifications.len()).unwrap_or(u32::MAX), + head_id, submissions: self .pending .iter() @@ -1125,6 +1136,13 @@ pub(crate) enum PendingSubmissionError { ByteLimit, #[error("pending submission artifact references exceed {MAX_PENDING_ARTIFACT_REFS}")] ArtifactLimit, + #[error("pending queue revision conflict: expected {expected}, current {current}")] + RevisionConflict { expected: u64, current: u64 }, + #[error("pending queue head conflict: expected {expected}, current {current:?}")] + HeadConflict { + expected: String, + current: Option, + }, #[error("pending submission not found: {0}")] NotFound(String), #[error("pending submission state persistence failed: {0}")] @@ -1141,6 +1159,29 @@ impl PendingSubmissionHandle where St: Store + Clone, { + fn validate_fence( + state: &PendingActivationState, + expected_revision: u64, + expected_head_id: Option<&str>, + ) -> Result<(), PendingSubmissionError> { + if state.revision != expected_revision { + return Err(PendingSubmissionError::RevisionConflict { + expected: expected_revision, + current: state.revision, + }); + } + if let Some(expected) = expected_head_id { + let current = state.snapshot().head_id; + if current.as_deref() != Some(expected) { + return Err(PendingSubmissionError::HeadConflict { + expected: expected.to_owned(), + current, + }); + } + } + Ok(()) + } + fn persist_locked(&self, state: &PendingActivationState) -> Result<(), PendingSubmissionError> { self.writer.append_entry_locked(LogEntry::Extension { ts: segment_log::now_millis(), @@ -1354,6 +1395,7 @@ where pub(crate) fn prepare_next_activation( &self, + fence: Option<(u64, &str)>, ) -> Result, PendingSubmissionError> { let _append_guard = self .writer @@ -1365,6 +1407,9 @@ where .state .lock() .expect("pending activation state poisoned"); + if let Some((expected_revision, expected_head_id)) = fence { + Self::validate_fence(&state, expected_revision, Some(expected_head_id))?; + } if state.activating.is_some() || state.activating_notification.is_some() { return Ok(None); } @@ -1495,6 +1540,7 @@ where pub(crate) fn cancel( &self, submission_id: &str, + expected_revision: u64, ) -> Result { let _append_guard = self .writer @@ -1506,6 +1552,7 @@ where .state .lock() .expect("pending activation state poisoned"); + Self::validate_fence(&state, expected_revision, None)?; let original = state.clone(); let Some(index) = state .pending @@ -1525,6 +1572,7 @@ where pub(crate) fn clear( &self, + expected_revision: u64, ) -> Result { let _append_guard = self .writer @@ -1536,6 +1584,7 @@ where .state .lock() .expect("pending activation state poisoned"); + Self::validate_fence(&state, expected_revision, None)?; let original = state.clone(); state.pending.clear(); state.pending_notifications.clear(); @@ -9189,10 +9238,23 @@ mod build_summary_prompt_tests { assert_eq!(restored.pending.len(), 1); assert_eq!(restored.pending[0].submission_id, accepted.submission_id); - let snapshot = handle.cancel(&accepted.submission_id).unwrap(); + let fence = handle.snapshot(); + assert!(matches!( + handle.cancel(&accepted.submission_id, fence.revision.saturating_sub(1)), + Err(PendingSubmissionError::RevisionConflict { .. }) + )); + assert!(matches!( + handle.prepare_next_activation(Some((fence.revision, "wrong-head"))), + Err(PendingSubmissionError::HeadConflict { .. }) + )); + assert_eq!(handle.snapshot(), fence); + + let snapshot = handle + .cancel(&accepted.submission_id, handle.snapshot().revision) + .unwrap(); assert!(snapshot.submissions.is_empty()); assert!(matches!( - handle.cancel(&accepted.submission_id), + handle.cancel(&accepted.submission_id, handle.snapshot().revision), Err(PendingSubmissionError::NotFound(_)) )); @@ -9210,7 +9272,7 @@ mod build_summary_prompt_tests { Err(PendingSubmissionError::CountLimit) )); assert_eq!(handle.snapshot().submissions.len(), MAX_PENDING_SUBMISSIONS); - let cleared = handle.clear().unwrap(); + let cleared = handle.clear(handle.snapshot().revision).unwrap(); assert!(cleared.submissions.is_empty()); assert_eq!(cleared.notification_count, 0); } @@ -9237,7 +9299,7 @@ mod build_summary_prompt_tests { .accept("request-1".into(), vec![Segment::text("submit")], false) .unwrap(); - let first = handle.prepare_next_activation().unwrap().unwrap(); + let first = handle.prepare_next_activation(None).unwrap().unwrap(); assert!(matches!( first, PendingActivation::Notification(PendingNotification { ref message, .. }) @@ -9249,7 +9311,7 @@ mod build_summary_prompt_tests { assert!(committed_state.pending_notifications.is_empty()); assert!(committed_state.activating_notification.is_none()); handle.finish_notification_activation("notification-1"); - let second = handle.prepare_next_activation().unwrap().unwrap(); + let second = handle.prepare_next_activation(None).unwrap().unwrap(); assert!(matches!(second, PendingActivation::Submission(_))); } diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 6faa2fdd..28fc96e2 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -1322,7 +1322,7 @@ async fn submit_while_running_is_durably_queued() { let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); let mut accepted = None; - let mut pending_count = None; + let mut pending_snapshot = None; while tokio::time::Instant::now() < deadline { match tokio::time::timeout(std::time::Duration::from_millis(100), rx.recv()).await { Ok(Ok(Event::SubmissionAccepted { @@ -1333,21 +1333,44 @@ async fn submit_while_running_is_durably_queued() { Ok(Ok(Event::PendingSubmissionsChanged { pending })) if pending.submissions.len() == 1 => { - pending_count = Some(1) + pending_snapshot = Some(pending) } Ok(Ok(Event::Error { code, message })) if code == worker::ErrorCode::AlreadyRunning => { panic!("Submit was busy-rejected: {message}") } _ => {} } - if accepted.is_some() && pending_count.is_some() { + if accepted.is_some() && pending_snapshot.is_some() { break; } } assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued)); - assert_eq!(pending_count, Some(1)); + let pending_snapshot = pending_snapshot.expect("pending snapshot"); + assert_eq!(pending_snapshot.submissions.len(), 1); handle.send(Method::Pause).await.unwrap(); + wait_for_status(&handle, WorkerStatus::Paused).await; + handle + .send(Method::ContinuePending { + expected_revision: pending_snapshot.revision, + expected_head_id: pending_snapshot.head_id.expect("pending head"), + }) + .await + .unwrap(); + let rejection = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if let Ok(Event::Error { code, message }) = rx.recv().await + && code == worker::ErrorCode::InvalidRequest + && message.contains("requires an idle Worker") + { + break message; + } + } + }) + .await + .expect("paused ContinuePending rejection"); + assert!(rejection.contains("Resume or Cancel")); + assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused); } #[tokio::test] diff --git a/web/workspace/src/lib/generated/protocol.ts b/web/workspace/src/lib/generated/protocol.ts index 1e744d20..6055d3e7 100644 --- a/web/workspace/src/lib/generated/protocol.ts +++ b/web/workspace/src/lib/generated/protocol.ts @@ -105,7 +105,7 @@ timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array, }; +export type PendingSubmissionsSnapshot = { revision: number, notification_count: number, head_id: string | null, submissions: Array, }; export type SubmissionDisposition = "started" | "queued"; @@ -231,7 +231,7 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod }); -export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, } } | { "method": "clear_pending_submissions" } | { "method": "continue_pending" } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; +export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number, /** diff --git a/web/workspace/src/lib/workspace/console/model.test.ts b/web/workspace/src/lib/workspace/console/model.test.ts index da4f70d5..60b98633 100644 --- a/web/workspace/src/lib/workspace/console/model.test.ts +++ b/web/workspace/src/lib/workspace/console/model.test.ts @@ -2150,7 +2150,12 @@ Deno.test("snapshot restores TaskStore state from system history", () => { const event = snapshotEvent("/repo"); if (event.event !== "snapshot") throw new Error("snapshot fixture expected"); event.data.session = { - pending_submissions: { revision: 0, notification_count: 0, submissions: [] }, + pending_submissions: { + revision: 0, + notification_count: 0, + head_id: null, + submissions: [], + }, entries: [{ entry_id: "task-reminder-1", timestamp: 1, diff --git a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts index b031f1d0..92d9531e 100644 --- a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts +++ b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts @@ -1077,7 +1077,20 @@ Deno.test("Web Console uses Notify while running and exposes durable pending con 'method: "cancel_pending_submission"', 'method: "clear_pending_submissions"', 'method: "continue_pending"', + "handleQueueSubmit", + "handleNotifySubmit", + ">Queue Submit", + ">Notify", ]) { assert(consolePage.includes(token), `missing durable pending control token: ${token}`); } + + const userCase = consolePage.slice( + consolePage.indexOf('case "user":'), + consolePage.indexOf('case "compact":'), + ); + assert( + !userCase.includes("workerRunning"), + "ordinary text must remain Submit instead of being implicitly converted to Notify", + ); }); diff --git a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte index 6c50e2ae..60bf188d 100644 --- a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte @@ -161,6 +161,7 @@ let pendingSubmissions = $state({ revision: 0, notification_count: 0, + head_id: null, submissions: [], }); let pendingSubmissionItems = $derived(pendingSubmissions.submissions ?? []); @@ -574,16 +575,6 @@ ): ProtocolMethod { switch (request.kind) { case "user": - if (workerRunning) { - return { - method: "notify", - params: { - notification_request_id: crypto.randomUUID(), - message: request.content, - auto_run: true, - }, - }; - } return { method: "submit", params: { @@ -672,6 +663,14 @@ void submitDraft(composerInputElement?.snapshot() ?? draft); } + function handleQueueSubmit() { + void submitDraft(composerInputElement?.snapshot() ?? draft); + } + + function handleNotifySubmit() { + void submitDraft(composerInputElement?.snapshot() ?? draft, "notify"); + } + function attachmentPath(): string { return `/api/w/${encodeURIComponent(workspaceId)}/runtimes/${encodeURIComponent(runtimeId)}/workers/${encodeURIComponent(workerId)}`; } @@ -771,7 +770,15 @@ if (event.dataTransfer?.files) addAttachmentFiles(event.dataTransfer.files); } - async function submitDraft(value: ComposerDraftSnapshot) { + async function submitDraft( + value: ComposerDraftSnapshot, + delivery: "submit" | "notify" = "submit", + ) { + if (delivery === "notify" && attachments.length > 0) { + composerNotice = null; + sendError = "Notify accepts text only; remove attachments or queue a Submit."; + return; + } const incompleteAttachment = attachments.find((attachment) => attachment.state !== "uploaded" || !attachment.reference ); @@ -805,10 +812,19 @@ return; } + let request: WorkerConsoleInputRequest = command.request; + if (delivery === "notify") { + if (request.kind !== "user") { + composerNotice = null; + sendError = "Notify accepts ordinary text, not a Composer command."; + return; + } + request = { kind: "notify", content: request.content }; + } sending = true; sendError = null; try { - const method = composerRequestToProtocolMethod(command.request); + const method = composerRequestToProtocolMethod(request); sendProtocolMethod(method); composerInputElement?.recordHistory(value); composerInputElement?.clear(); @@ -1772,7 +1788,10 @@ sendControl( { method: "cancel_pending_submission", - params: { submission_id: submission.submission_id }, + params: { + submission_id: submission.submission_id, + expected_revision: pendingSubmissions.revision, + }, }, "Pending submission cancellation", )} @@ -1782,10 +1801,16 @@ @@ -1793,7 +1818,10 @@ type="button" onclick={() => sendControl( - { method: "clear_pending_submissions" }, + { + method: "clear_pending_submissions", + params: { expected_revision: pendingSubmissions.revision }, + }, "Pending submissions clear", )} >Clear all @@ -1932,6 +1960,18 @@
+ {#if workerRunning} + + + {/if} {#if composerNotice} {composerNotice} {/if} From bb4c1dfe4ff0a0735a372cd9b1c87893ebb21c6d Mon Sep 17 00:00:00 2001 From: Hare Date: Sat, 5 Sep 2026 23:25:25 +0900 Subject: [PATCH 03/39] docs: update flow submit protocol example --- docs/design/flow-state-graph.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/design/flow-state-graph.md b/docs/design/flow-state-graph.md index 3fc49249..bd0cea00 100644 --- a/docs/design/flow-state-graph.md +++ b/docs/design/flow-state-graph.md @@ -55,11 +55,12 @@ Workspace Server schema migration v26 removes the legacy `flow_instances`, `flow ## Worker boundary -Flow invocation uses the normal Submit/Run segment vector rather than a Worker-create field: +Flow invocation uses the normal Submit segment vector rather than a Worker-create field: ```json { - "method": "run", + "method": "submit", + "submission_request_id": "018f4f15-5c41-7d3a-8a72-2e755bc71681", "input": [ { "kind": "flow", "selector": "builtin:coder-review" }, { "kind": "text", "content": "Ticket 00001... implementation" } @@ -69,7 +70,7 @@ Flow invocation uses the normal Submit/Run segment vector rather than a Worker-c Runtime accepts exactly one Flow segment only when the resolved Profile enables `feature.flow` and a Workspace client is available. The Worker asks Workspace authority only for an immutable source snapshot, creates the instance locally, replaces the Flow segment with the entered state's instructions, and commits that runtime state atomically with the remaining Submit segments before LLM execution. A Worker with an active Flow rejects the duplicate input without changing its local state or events. -The generic model-facing `WorkerSpawn` accepts `initial_submit: Vec` and routes them unchanged through the shared Workspace spawn request into Runtime `CreateWorkerRequest.initial_input`. It does not have a parallel `initial_text` or a role-specific `SpawnCoder` wrapper. Backend derives the flat content projection from the canonical segment vector, validates Flow shape before spawn, and includes the segment vector in lifecycle idempotency fingerprints. Runtime does not commit Worker creation or report spawn success merely because the initial Run method entered the Worker's in-memory channel: Runtime assigns the Submit an opaque id, the Worker commits that id as an extension on the same `UserInput` entry as any initial `FlowRuntimeState`, and the execution backend must return a matching typed input-commit acknowledgement. Restoring the same Worker never replays spawn initial segments. +The generic model-facing `WorkerSpawn` accepts `initial_submit: Vec` and routes them unchanged through the shared Workspace spawn request into Runtime `CreateWorkerRequest.initial_input`. It does not have a parallel `initial_text` or a role-specific `SpawnCoder` wrapper. Backend derives the flat content projection from the canonical segment vector, validates Flow shape before spawn, and includes the segment vector in lifecycle idempotency fingerprints. Runtime does not commit Worker creation or report spawn success merely because the initial Submit request entered the Worker's in-memory channel: Runtime assigns the Submit an opaque id, the Worker commits that id as an extension on the same `UserInput` entry as any initial `FlowRuntimeState`, and the execution backend must return a matching typed input-commit acknowledgement. Restoring the same Worker never replays spawn initial segments. When an Orchestrator supplies `ticket_id` to generic `WorkerSpawn`, the Worker tool derives the assignment operation id from the durable tool-call id rather than accepting lifecycle authority from model input. The shared Workspace worker-create route projects that request into a Coder Ticket-role intent and atomically applies the existing queued-Ticket assignment operation only after Runtime has returned the input-commit acknowledgement. A spawn or pre-commit input failure therefore leaves the Ticket queued and unassigned. From b038f022d395fa05f7ec472650f60fd96daacfa8 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 01:35:26 +0900 Subject: [PATCH 04/39] fix: harden submit queue durability --- crates/protocol/src/lib.rs | 64 +- crates/session-store/src/fs_store.rs | 117 +++- crates/session-store/src/store.rs | 20 + crates/session-store/src/uploaded_file.rs | 82 ++- crates/worker-runtime/src/worker_backend.rs | 6 + crates/worker/src/controller.rs | 302 +++++++-- crates/worker/src/ipc/interceptor.rs | 26 +- crates/worker/src/ipc/notify_buffer.rs | 25 +- crates/worker/src/worker.rs | 605 ++++++++++++++++-- crates/worker/tests/controller_test.rs | 79 ++- crates/workspace-server/src/hosts.rs | 10 +- .../src/runtime_subscription_tests.rs | 8 +- crates/workspace-server/src/server.rs | 170 ++++- .../src/workspace_subscription.rs | 17 +- 14 files changed, 1370 insertions(+), 161 deletions(-) diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index e3330ba0..e6d790db 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -32,6 +32,38 @@ fn is_false(value: &bool) -> bool { // Method (Client → Worker via Unix Socket) // --------------------------------------------------------------------------- +/// Trusted source identity attached by an authenticated transport boundary. +/// +/// Public clients cannot select this value directly. Runtime/Backend adapters +/// stamp it before forwarding an accepted Submit or Notify to a Worker. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum AuthenticatedInputSource { + Account { + account_id: String, + }, + Worker { + runtime_id: String, + worker_id: String, + }, + Backend { + operation_id: String, + }, +} + +impl AuthenticatedInputSource { + pub fn namespace(&self) -> String { + match self { + Self::Account { account_id } => format!("account:{account_id}"), + Self::Worker { + runtime_id, + worker_id, + } => format!("worker:{runtime_id}:{worker_id}"), + Self::Backend { operation_id } => format!("backend:{operation_id}"), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(tag = "method", content = "params", rename_all = "snake_case")] @@ -45,13 +77,13 @@ pub enum Method { submission_request_id: String, input: Vec, }, - /// Runtime-internal Submit with the same request identity contract. This - /// variant is not serializable on the public Client → Worker protocol. - #[serde(skip)] + /// Authenticated transport form of Submit. Trusted adapters replace + /// public Submit before forwarding it to the Worker. #[cfg_attr(feature = "typescript", ts(skip))] SubmitTracked { submission_request_id: String, input: Vec, + source: AuthenticatedInputSource, }, /// Human-readable text injected into the target Worker's LLM context /// as a non-blocking system message. `auto_run` controls whether an @@ -65,6 +97,15 @@ pub enum Method { #[serde(default = "default_true", skip_serializing_if = "is_true")] auto_run: bool, }, + /// Authenticated transport form of Notify. + #[cfg_attr(feature = "typescript", ts(skip))] + NotifyTracked { + notification_request_id: String, + message: String, + #[serde(default = "default_true", skip_serializing_if = "is_true")] + auto_run: bool, + source: AuthenticatedInputSource, + }, /// Typed lifecycle report from a child Worker to its direct parent. WorkerEvent(WorkerEvent), /// Return the authoritative FIFO summary without exposing queued payloads. @@ -1497,15 +1538,26 @@ mod tests { } #[test] - fn runtime_tracked_submit_is_not_public_protocol_json() { + fn authenticated_submit_round_trips_trusted_source() { let method = Method::SubmitTracked { input: vec![Segment::text("private")], submission_request_id: "request-1".to_string(), + source: AuthenticatedInputSource::Account { + account_id: "account-1".into(), + }, }; - assert!(serde_json::to_string(&method).is_err()); + let json = serde_json::to_string(&method).unwrap(); + let decoded = serde_json::from_str::(&json).unwrap(); + assert!(matches!( + decoded, + Method::SubmitTracked { + source: AuthenticatedInputSource::Account { account_id }, + .. + } if account_id == "account-1" + )); assert!( serde_json::from_str::( - r#"{"method":"submit_tracked","input":[],"submission_id":"forged"}"#, + r#"{"method":"submit_tracked","input":[],"submission_request_id":"forged"}"#, ) .is_err() ); diff --git a/crates/session-store/src/fs_store.rs b/crates/session-store/src/fs_store.rs index a0e80a75..abdf14e4 100644 --- a/crates/session-store/src/fs_store.rs +++ b/crates/session-store/src/fs_store.rs @@ -22,7 +22,8 @@ use crate::store::{Store, StoreError}; use crate::uploaded_file::{ bind_uploaded_file, clear_uploaded_file_binding, copy_committed_uploaded_files, delete_uncommitted_uploaded_files, delete_uploaded_file, list_uploaded_file_refs, - read_uploaded_file, read_uploaded_file_by_id, write_uploaded_file, + pin_uploaded_file, read_uploaded_file, read_uploaded_file_by_id, release_uploaded_file_pin, + write_uploaded_file, }; use crate::{ PasteArtifactLimits, SegmentId, SessionId, UploadedFileLimits, UploadedFileUploadContext, @@ -518,6 +519,32 @@ impl Store for FsStore { } } + fn pin_uploaded_file( + &self, + session_id: SessionId, + reference: &UploadedFileRef, + owner_id: &str, + ) -> Result<(), StoreError> { + let _guard = self + .append_lock + .lock() + .map_err(|_| std::io::Error::other("session store append lock was poisoned"))?; + pin_uploaded_file(&self.paste_artifact_dir(session_id), reference, owner_id) + } + + fn release_uploaded_file_pin( + &self, + session_id: SessionId, + artifact_id: &str, + owner_id: &str, + ) -> Result<(), StoreError> { + let _guard = self + .append_lock + .lock() + .map_err(|_| std::io::Error::other("session store append lock was poisoned"))?; + release_uploaded_file_pin(&self.paste_artifact_dir(session_id), artifact_id, owner_id) + } + fn delete_uploaded_file( &self, session_id: SessionId, @@ -865,6 +892,94 @@ mod tests { assert!(store.read_uploaded_file(owner, &reference).is_err()); } + #[test] + fn pending_upload_pin_survives_cleanup_until_release_or_history_binding() { + let tmp = tempfile::TempDir::new().unwrap(); + let store = FsStore::new(tmp.path()).unwrap(); + let session_id = new_session_id(); + let limits = UploadedFileLimits { + max_file_bytes: 64, + max_session_bytes: 128, + }; + let pending = store + .write_uploaded_file(session_id, "pending.txt", "text/plain", b"pending", limits) + .unwrap(); + store + .pin_uploaded_file(session_id, &pending, "submission-1") + .unwrap(); + assert!(matches!( + store.pin_uploaded_file(session_id, &pending, "submission-other"), + Err(StoreError::ArtifactAlreadyCommitted) + )); + drop(store); + let store = FsStore::new(tmp.path()).unwrap(); + assert_eq!( + store.delete_uncommitted_uploaded_files(session_id).unwrap(), + 0 + ); + assert_eq!( + store + .read_uploaded_file_by_id(session_id, &pending.artifact_id) + .unwrap() + .1, + b"pending" + ); + + let fork_session_id = new_session_id(); + assert_eq!( + store + .copy_committed_uploaded_files(session_id, fork_session_id) + .unwrap(), + 1 + ); + assert_eq!( + store + .read_uploaded_file_by_id(fork_session_id, &pending.artifact_id) + .unwrap() + .1, + b"pending" + ); + + let committed = store + .bind_uploaded_file(session_id, &pending, "entry-1") + .unwrap(); + store + .create_segment( + session_id, + new_segment_id(), + &[LogEntry::InputSegmentsCheckpoint { + ts: 1, + user_segments: vec![vec![protocol::Segment::UploadedFile { + file: committed.clone(), + }]], + }], + ) + .unwrap(); + assert_eq!( + store.delete_uncommitted_uploaded_files(session_id).unwrap(), + 0 + ); + + let releasable = store + .write_uploaded_file(session_id, "cancelled.txt", "text/plain", b"cancel", limits) + .unwrap(); + store + .pin_uploaded_file(session_id, &releasable, "submission-2") + .unwrap(); + store + .release_uploaded_file_pin(session_id, &releasable.artifact_id, "submission-2") + .unwrap(); + assert_eq!( + store.delete_uncommitted_uploaded_files(session_id).unwrap(), + 1 + ); + assert!( + store + .read_uploaded_file_by_id(session_id, &releasable.artifact_id) + .is_err() + ); + } + #[test] fn uploaded_file_validation_and_shared_quota_fail_closed() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/crates/session-store/src/store.rs b/crates/session-store/src/store.rs index a22fd6c7..4c98fa33 100644 --- a/crates/session-store/src/store.rs +++ b/crates/session-store/src/store.rs @@ -226,6 +226,26 @@ pub trait Store: Send + Sync { Err(StoreError::PasteArtifactUnsupported) } + /// Retain an uploaded file while a durable pending operation owns it. + fn pin_uploaded_file( + &self, + _session_id: SessionId, + _reference: &UploadedFileRef, + _owner_id: &str, + ) -> Result<(), StoreError> { + Err(StoreError::PasteArtifactUnsupported) + } + + /// Release a pending-operation pin without changing committed ownership. + fn release_uploaded_file_pin( + &self, + _session_id: SessionId, + _artifact_id: &str, + _owner_id: &str, + ) -> Result<(), StoreError> { + Err(StoreError::PasteArtifactUnsupported) + } + /// Delete an uncommitted uploaded file owned by `session_id`. fn delete_uploaded_file( &self, diff --git a/crates/session-store/src/uploaded_file.rs b/crates/session-store/src/uploaded_file.rs index 4ce37588..a7c544ea 100644 --- a/crates/session-store/src/uploaded_file.rs +++ b/crates/session-store/src/uploaded_file.rs @@ -24,6 +24,12 @@ pub const DEFAULT_MAX_FILES_PER_SUBMISSION: usize = 8; pub const DEFAULT_MAX_SESSION_UPLOADED_FILES: u64 = 256; const MAX_FILE_NAME_CHARS: usize = 255; const MAX_MEDIA_TYPE_BYTES: usize = 127; +fn validate_pending_owner_id(owner_id: &str) -> Result<()> { + if owner_id.is_empty() || owner_id.len() > 256 { + return Err(StoreError::ArtifactIntegrityMismatch); + } + Ok(()) +} #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct UploadedFileLimits { @@ -59,6 +65,8 @@ struct StoredUploadedFile { #[serde(default, skip_serializing_if = "Option::is_none")] source_entry_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pending_owner_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] upload_context: Option, content_base64: String, } @@ -291,6 +299,7 @@ pub(crate) fn write_uploaded_file( byte_len, sha256: sha256.clone(), source_entry_id: None, + pending_owner_id: None, upload_context: context.cloned(), content_base64: BASE64.encode(content), }; @@ -376,6 +385,72 @@ pub(crate) fn clear_uploaded_file_binding( Ok(()) } +pub(crate) fn pin_uploaded_file( + dir: &Path, + reference: &UploadedFileRef, + owner_id: &str, +) -> Result<()> { + validate_pending_owner_id(owner_id)?; + if reference.source_entry_id.is_some() { + return Err(StoreError::ArtifactAlreadyCommitted); + } + let aggregate_lock = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(dir.join(".aggregate.lock"))?; + FileExt::lock_exclusive(&aggregate_lock)?; + let path = record_path(dir, &reference.artifact_id)?; + let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?; + if stored.file_name != reference.file_name + || stored.media_type != reference.media_type + || stored.created_at_ms != reference.created_at_ms + || stored.byte_len != reference.byte_len + || stored.sha256 != reference.sha256 + { + return Err(StoreError::ArtifactIntegrityMismatch); + } + if stored.source_entry_id.is_some() { + return Err(StoreError::ArtifactAlreadyCommitted); + } + if let Some(existing_owner) = stored.pending_owner_id.as_deref() { + return if existing_owner == owner_id { + Ok(()) + } else { + Err(StoreError::ArtifactAlreadyCommitted) + }; + } + stored.pending_owner_id = Some(owner_id.to_owned()); + let temp = dir.join(format!(".{}.file.pin.tmp", reference.artifact_id)); + fs::write(&temp, serde_json::to_vec(&stored)?)?; + fs::rename(temp, path)?; + Ok(()) +} + +pub(crate) fn release_uploaded_file_pin( + dir: &Path, + artifact_id: &str, + owner_id: &str, +) -> Result<()> { + validate_pending_owner_id(owner_id)?; + let aggregate_lock = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(dir.join(".aggregate.lock"))?; + FileExt::lock_exclusive(&aggregate_lock)?; + let path = record_path(dir, artifact_id)?; + let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?; + if stored.source_entry_id.is_some() || stored.pending_owner_id.as_deref() != Some(owner_id) { + return Err(StoreError::ArtifactIntegrityMismatch); + } + stored.pending_owner_id = None; + let temp = dir.join(format!(".{artifact_id}.file.unpin.tmp")); + fs::write(&temp, serde_json::to_vec(&stored)?)?; + fs::rename(temp, path)?; + Ok(()) +} + pub(crate) fn bind_uploaded_file( dir: &Path, reference: &UploadedFileRef, @@ -405,6 +480,7 @@ pub(crate) fn bind_uploaded_file( return Err(StoreError::ArtifactAlreadyCommitted); } stored.source_entry_id = Some(source_entry_id.to_owned()); + stored.pending_owner_id = None; let temp = dir.join(format!(".{}.file.bind.tmp", reference.artifact_id)); fs::write(&temp, serde_json::to_vec(&stored)?)?; fs::rename(&temp, path)?; @@ -455,7 +531,7 @@ pub(crate) fn copy_committed_uploaded_files(source_dir: &Path, target_dir: &Path } let bytes = fs::read(&path)?; let stored: StoredUploadedFile = serde_json::from_slice(&bytes)?; - if stored.source_entry_id.is_none() { + if stored.source_entry_id.is_none() && stored.pending_owner_id.is_none() { continue; } let target = target_dir.join(name); @@ -499,7 +575,7 @@ pub(crate) fn delete_uncommitted_uploaded_files(dir: &Path) -> Result { continue; } let stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?; - if stored.source_entry_id.is_none() { + if stored.source_entry_id.is_none() && stored.pending_owner_id.is_none() { fs::remove_file(path)?; removed = removed .checked_add(1) @@ -523,7 +599,7 @@ pub(crate) fn delete_uploaded_file(dir: &Path, artifact_id: &str) -> Result return Ok(false), Err(error) => return Err(error.into()), }; - if stored.source_entry_id.is_some() { + if stored.source_entry_id.is_some() || stored.pending_owner_id.is_some() { return Err(StoreError::ArtifactAlreadyCommitted); } match fs::remove_file(path) { diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 07800e82..408d8155 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -1522,6 +1522,7 @@ fn method_starts_turn(method: &Method) -> bool { Method::Submit { .. } | Method::SubmitTracked { .. } | Method::Notify { auto_run: true, .. } + | Method::NotifyTracked { auto_run: true, .. } | Method::Resume | Method::Compact ) @@ -1549,6 +1550,7 @@ fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState { Method::Submit { .. } | Method::SubmitTracked { .. } | Method::Notify { auto_run: true, .. } + | Method::NotifyTracked { auto_run: true, .. } | Method::Resume | Method::Compact => WorkerExecutionRunState::Busy, Method::Shutdown => WorkerExecutionRunState::Stopped, @@ -1946,6 +1948,9 @@ where input: input.segments.unwrap_or_else(|| { vec![Segment::text(input.content.trim().to_string())] }), + source: protocol::AuthenticatedInputSource::Backend { + operation_id: submission_id.clone(), + }, }, Some(submission_id), ) @@ -1966,6 +1971,7 @@ where Method::Submit { .. } | Method::SubmitTracked { .. } | Method::Notify { .. } + | Method::NotifyTracked { .. } | Method::Compact => WorkerExecutionRunState::Busy, _ => WorkerExecutionRunState::Idle, }; diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 03a98e5e..14fd26f6 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -229,6 +229,43 @@ enum PendingRun { Resume, } +fn stage_pending_notification( + pending_submissions: &crate::worker::PendingSubmissionHandle, + notify_buffer: &NotifyBuffer, + source_namespace: &str, + notification_request_id: &str, +) -> bool { + let Some(notification) = + pending_submissions.prepare_notification(source_namespace, notification_request_id) + else { + return false; + }; + let extension = pending_submissions.notification_activation_extension(); + notify_buffer.push_durable_notify( + notification.message, + notification.auto_run, + notification.provenance, + extension, + ); + true +} + +fn stage_oldest_passive_notification( + pending_submissions: &crate::worker::PendingSubmissionHandle, + notify_buffer: &NotifyBuffer, +) -> bool { + pending_submissions + .next_passive_notification_identity() + .is_some_and(|(source_namespace, request_id)| { + stage_pending_notification( + pending_submissions, + notify_buffer, + &source_namespace, + &request_id, + ) + }) +} + fn prepare_pending_run( pending_submissions: &crate::worker::PendingSubmissionHandle, notify_buffer: &NotifyBuffer, @@ -241,7 +278,12 @@ fn prepare_pending_run( Some(crate::worker::PendingActivation::Notification(notification)) => { let extension = pending_submissions.notification_activation_extension(); let notification_request_id = notification.notification_request_id.clone(); - notify_buffer.push_durable_notify(notification.message, extension); + notify_buffer.push_durable_notify( + notification.message, + notification.auto_run, + notification.provenance, + extension, + ); Some(PendingRun::RunForNotification { invoke_kind: protocol::InvokeKind::Notify, notification_request_id: Some(notification_request_id), @@ -1314,6 +1356,7 @@ async fn controller_loop( ); let mut pending: Option = None; let pending_submissions = worker.pending_submission_handle(); + stage_oldest_passive_notification(&pending_submissions, ¬ify_buffer); loop { // Top-of-iteration: if an event handler staged a run, fire it @@ -1347,6 +1390,8 @@ async fn controller_loop( } => notification_request_id.clone(), _ => None, }; + let passive_notification_request_id = + pending_submissions.activating_passive_notification_id(); let (mut new_status, shutdown, may_drain_pending) = match run { PendingRun::Submit(submission) => { let (input_commit_tx, input_commit_rx) = oneshot::channel(); @@ -1356,6 +1401,7 @@ async fn controller_loop( worker.run_with_input_extensions_and_commit_hook( submission.input, vec![extension], + submission.provenance, move || { let _ = input_commit_tx.send(()); }, @@ -1415,8 +1461,11 @@ async fn controller_loop( .await } }; - if let Some(notification_request_id) = notification_request_id { + if let Some(notification_request_id) = + notification_request_id.or(passive_notification_request_id) + { pending_submissions.finish_notification_activation(¬ification_request_id); + stage_oldest_passive_notification(&pending_submissions, ¬ify_buffer); } if !shutdown && may_drain_pending && new_status == WorkerStatus::Idle { @@ -1470,13 +1519,47 @@ async fn controller_loop( Method::Submit { submission_request_id, input, - } - | Method::SubmitTracked { - submission_request_id, - input, } => { let request_id = submission_request_id.clone(); - match pending_submissions.accept(submission_request_id, input, true) { + match pending_submissions.accept_from_source( + submission_request_id, + input, + pending_submissions.direct_client_namespace(), + session_store::LoggedSessionHistoryOrigin::LegacyUnknown, + true, + ) { + Ok(acceptance) => { + if let Some(activation) = acceptance.activation { + pending = Some(PendingRun::Submit(activation)); + } else { + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: acceptance.submission_request_id, + submission_id: acceptance.submission_id, + disposition: acceptance.disposition, + }); + } + } + Err(error) => { + let _ = working_event_tx.send(Event::SubmissionRejected { + submission_request_id: request_id, + message: error.to_string(), + }); + } + } + } + Method::SubmitTracked { + submission_request_id, + input, + source, + } => { + let request_id = submission_request_id.clone(); + match pending_submissions.accept_from_source( + submission_request_id, + input, + source.namespace(), + crate::worker::authenticated_input_provenance(&source), + true, + ) { Ok(acceptance) => { if let Some(activation) = acceptance.activation { pending = Some(PendingRun::Submit(activation)); @@ -1502,31 +1585,85 @@ async fn controller_loop( message, auto_run, } => { - if auto_run { - match pending_submissions.accept_notification(notification_request_id, message) - { - Ok(true) => { - match prepare_pending_run(&pending_submissions, ¬ify_buffer, None) { - Ok(Some(next)) => pending = Some(next), - Ok(None) => {} - Err(error) => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::Internal, - message: error.to_string(), - }); - } + let request_id = notification_request_id.clone(); + let source_namespace = pending_submissions.direct_client_namespace(); + match pending_submissions.accept_notification_from_source( + notification_request_id, + message, + source_namespace.clone(), + session_store::LoggedSessionHistoryOrigin::LegacyUnknown, + auto_run, + ) { + Ok(_) if auto_run => { + match prepare_pending_run(&pending_submissions, ¬ify_buffer, None) { + Ok(Some(next)) => pending = Some(next), + Ok(None) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); } } - Ok(false) => {} - Err(error) => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::InvalidRequest, - message: error.to_string(), - }); + } + Ok(_) => { + stage_pending_notification( + &pending_submissions, + ¬ify_buffer, + &source_namespace, + &request_id, + ); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + } + + Method::NotifyTracked { + notification_request_id, + message, + auto_run, + source, + } => { + let request_id = notification_request_id.clone(); + let source_namespace = source.namespace(); + match pending_submissions.accept_notification_from_source( + notification_request_id, + message, + source_namespace.clone(), + crate::worker::authenticated_input_provenance(&source), + auto_run, + ) { + Ok(_) if auto_run => { + match prepare_pending_run(&pending_submissions, ¬ify_buffer, None) { + Ok(Some(next)) => pending = Some(next), + Ok(None) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } } } - } else { - worker.push_notify(message, false); + Ok(_) => { + stage_pending_notification( + &pending_submissions, + ¬ify_buffer, + &source_namespace, + &request_id, + ); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } } } @@ -2070,13 +2207,46 @@ where Some(Method::Submit { submission_request_id, input, - } - | Method::SubmitTracked { - submission_request_id, - input, }) => { let request_id = submission_request_id.clone(); - match pending_submissions.accept(submission_request_id, input, false) { + match pending_submissions.accept_from_source( + submission_request_id, + input, + pending_submissions.direct_client_namespace(), + session_store::LoggedSessionHistoryOrigin::LegacyUnknown, + false, + ) { + Ok(acceptance) => { + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: acceptance.submission_request_id, + submission_id: acceptance.submission_id, + disposition: acceptance.disposition, + }); + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::SubmissionRejected { + submission_request_id: request_id, + message: error.to_string(), + }); + } + } + } + Some(Method::SubmitTracked { + submission_request_id, + input, + source, + }) => { + let request_id = submission_request_id.clone(); + match pending_submissions.accept_from_source( + submission_request_id, + input, + source.namespace(), + crate::worker::authenticated_input_provenance(&source), + false, + ) { Ok(acceptance) => { let _ = working_event_tx.send(Event::SubmissionAccepted { submission_request_id: acceptance.submission_request_id, @@ -2147,23 +2317,69 @@ where message, auto_run, }) => { - if auto_run { - if let Err(error) = pending_submissions.accept_notification( - notification_request_id, - message, - ) { + let request_id = notification_request_id.clone(); + let source_namespace = pending_submissions.direct_client_namespace(); + match pending_submissions.accept_notification_from_source( + notification_request_id, + message, + source_namespace.clone(), + session_store::LoggedSessionHistoryOrigin::LegacyUnknown, + auto_run, + ) { + Ok(_) if !auto_run => { + stage_pending_notification( + &pending_submissions, + notify_buffer, + &source_namespace, + &request_id, + ); + } + Ok(_) => {} + Err(error) => { let _ = working_event_tx.send(Event::Error { code: ErrorCode::InvalidRequest, message: error.to_string(), }); - } else { - let _ = working_event_tx.send(Event::PendingSubmissionsChanged { - pending: pending_submissions.snapshot(), + } + } + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Some(Method::NotifyTracked { + notification_request_id, + message, + auto_run, + source, + }) => { + let request_id = notification_request_id.clone(); + let source_namespace = source.namespace(); + match pending_submissions.accept_notification_from_source( + notification_request_id, + message, + source_namespace.clone(), + crate::worker::authenticated_input_provenance(&source), + auto_run, + ) { + Ok(_) if !auto_run => { + stage_pending_notification( + &pending_submissions, + notify_buffer, + &source_namespace, + &request_id, + ); + } + Ok(_) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), }); } - } else { - notify_buffer.push_notify(message, false); } + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); } Some(Method::ListCompletions { .. }) => {} Some(Method::ListWorkers | Method::RestoreWorker { .. } | Method::RegisterPeer { .. }) => { diff --git a/crates/worker/src/ipc/interceptor.rs b/crates/worker/src/ipc/interceptor.rs index 5853a2f7..86f26230 100644 --- a/crates/worker/src/ipc/interceptor.rs +++ b/crates/worker/src/ipc/interceptor.rs @@ -178,14 +178,21 @@ impl WorkerInterceptor { /// matches worker-history order. fn commit_system_items_with_extensions( &self, - items: &[(SystemItem, Vec)], + items: &[( + SystemItem, + Vec, + Option, + )], ) -> Result<(), session_store::StoreError> { let Some(writer) = self.log_writer.as_ref() else { return Ok(()); }; - for (item, extensions) in items { - let entry = - writer.commit_system_item_with_extensions(item.clone(), extensions.clone())?; + for (item, extensions, history_provenance) in items { + let entry = writer.commit_system_item_with_extensions( + item.clone(), + extensions.clone(), + history_provenance.clone(), + )?; self.pending_committed_history .lock() .expect("pending committed history poisoned") @@ -199,7 +206,7 @@ impl WorkerInterceptor { &items .iter() .cloned() - .map(|item| (item, Vec::new())) + .map(|item| (item, Vec::new(), None)) .collect::>(), ) } @@ -341,8 +348,11 @@ impl Interceptor for WorkerInterceptor { projection_digest: projection.catalog_digest.clone(), logical_name: "internal.notify_wrapper".to_string(), }; - let mut system_items: Vec<(SystemItem, Vec)> = - Vec::with_capacity(drained.len()); + let mut system_items: Vec<( + SystemItem, + Vec, + Option, + )> = Vec::with_capacity(drained.len()); let mut items: Vec = Vec::with_capacity(drained.len()); for entry in &drained { let system_item = match build_system_item_with_provenance( @@ -360,7 +370,7 @@ impl Interceptor for WorkerInterceptor { } }; items.push(system_item.to_history_item()); - system_items.push((system_item, entry.extensions())); + system_items.push((system_item, entry.extensions(), entry.history_provenance())); } if let Err(error) = self.commit_system_items_with_extensions(&system_items) { self.pending_notifies.requeue_front(drained); diff --git a/crates/worker/src/ipc/notify_buffer.rs b/crates/worker/src/ipc/notify_buffer.rs index cbaedc5d..2212dfe1 100644 --- a/crates/worker/src/ipc/notify_buffer.rs +++ b/crates/worker/src/ipc/notify_buffer.rs @@ -25,7 +25,7 @@ use std::collections::VecDeque; use std::sync::{Arc, Mutex}; use protocol::WorkerEvent; -use session_store::{SessionExtension, SystemItem}; +use session_store::{LoggedSessionHistoryOrigin, SessionExtension, SystemItem}; use tracing::warn; use crate::prompt::catalog::{CatalogError, PromptCatalog}; @@ -45,6 +45,7 @@ pub enum PendingNotify { message: String, auto_run: bool, extensions: Vec, + history_provenance: Option, }, WorkerEvent { event: WorkerEvent, @@ -58,6 +59,15 @@ impl PendingNotify { PendingNotify::WorkerEvent { .. } => Vec::new(), } } + + pub(crate) fn history_provenance(&self) -> Option { + match self { + PendingNotify::Notify { + history_provenance, .. + } => history_provenance.clone(), + PendingNotify::WorkerEvent { .. } => None, + } + } } /// Shared, mutex-guarded buffer of pending entries. @@ -81,14 +91,22 @@ impl NotifyBuffer { message, auto_run, extensions: Vec::new(), + history_provenance: None, }); } - pub fn push_durable_notify(&self, message: String, extension: SessionExtension) { + pub fn push_durable_notify( + &self, + message: String, + auto_run: bool, + history_provenance: LoggedSessionHistoryOrigin, + extension: SessionExtension, + ) { self.push_entry(PendingNotify::Notify { message, - auto_run: true, + auto_run, extensions: vec![extension], + history_provenance: Some(history_provenance), }); } @@ -230,6 +248,7 @@ mod tests { message: "hello".into(), auto_run: false, extensions: Vec::new(), + history_provenance: None, }; let catalog = PromptCatalog::builtins_only().unwrap(); let item = build_system_item(&entry, &catalog).unwrap(); diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 3275aab0..d24658d0 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -79,11 +79,12 @@ const MAX_SUBMISSION_RECEIPTS: usize = 128; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub(crate) struct PendingSubmission { pub(crate) submission_request_id: String, + source_namespace: String, pub(crate) submission_id: String, payload_digest: String, accepted_at_ms: u64, activation_sequence: u64, - provenance: WorkerHistoryProvenance, + pub(crate) provenance: WorkerHistoryProvenance, #[serde(default)] was_queued: bool, pub(crate) input: Vec, @@ -92,6 +93,7 @@ pub(crate) struct PendingSubmission { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] struct SubmissionReceipt { submission_request_id: String, + source_namespace: String, submission_id: String, payload_digest: String, disposition: protocol::SubmissionDisposition, @@ -100,17 +102,21 @@ struct SubmissionReceipt { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub(crate) struct PendingNotification { pub(crate) notification_request_id: String, + source_namespace: String, pub(crate) message: String, payload_digest: String, + pub(crate) auto_run: bool, accepted_at_ms: u64, activation_sequence: u64, - provenance: WorkerHistoryProvenance, + pub(crate) provenance: WorkerHistoryProvenance, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] struct NotificationReceipt { notification_request_id: String, + source_namespace: String, payload_digest: String, + auto_run: bool, } #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] @@ -130,14 +136,24 @@ pub(crate) struct PendingActivationState { impl PendingActivationState { pub(crate) fn snapshot(&self) -> protocol::PendingSubmissionsSnapshot { - let head_id = match (self.pending.front(), self.pending_notifications.front()) { + let pending_notification = self + .pending_notifications + .iter() + .find(|notification| notification.auto_run); + let head_id = match (self.pending.front(), pending_notification) { (Some(submission), Some(notification)) if notification.activation_sequence < submission.activation_sequence => { - Some(notification.notification_request_id.clone()) + Some(notification_head_id( + ¬ification.source_namespace, + ¬ification.notification_request_id, + )) } (Some(submission), _) => Some(submission.submission_id.clone()), - (None, Some(notification)) => Some(notification.notification_request_id.clone()), + (None, Some(notification)) => Some(notification_head_id( + ¬ification.source_namespace, + ¬ification.notification_request_id, + )), (None, None) => None, }; protocol::PendingSubmissionsSnapshot { @@ -172,6 +188,65 @@ impl PendingActivationState { } } +fn notification_payload_digest(message: &str, auto_run: bool) -> String { + use sha2::Digest as _; + let mut hasher = sha2::Sha256::new(); + hasher.update(if auto_run { + &b"auto\0"[..] + } else { + &b"deferred\0"[..] + }); + hasher.update(message.as_bytes()); + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +pub(crate) fn authenticated_input_provenance( + source: &protocol::AuthenticatedInputSource, +) -> WorkerHistoryProvenance { + match source { + protocol::AuthenticatedInputSource::Account { account_id } => { + WorkerHistoryProvenance::HumanInput { + account_id: account_id.clone(), + } + } + protocol::AuthenticatedInputSource::Worker { + runtime_id, + worker_id, + } => WorkerHistoryProvenance::WorkerInput { + actor: session_store::LoggedWorkerSubject { + workspace_id: None, + runtime_id: Some(runtime_id.clone()), + worker_id: worker_id.clone(), + }, + }, + protocol::AuthenticatedInputSource::Backend { operation_id } => { + WorkerHistoryProvenance::BackendInstruction { + operation_id: Some(operation_id.clone()), + } + } + } +} + +fn notification_head_id(source_namespace: &str, request_id: &str) -> String { + use sha2::Digest as _; + let mut hasher = sha2::Sha256::new(); + hasher.update(source_namespace.as_bytes()); + hasher.update(b"\0"); + hasher.update(request_id.as_bytes()); + format!( + "notification:{}", + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ) +} + fn submission_payload_len(input: &[Segment]) -> u64 { serde_json::to_vec(input) .map(|bytes| u64::try_from(bytes.len()).unwrap_or(u64::MAX)) @@ -186,6 +261,15 @@ fn submission_payload_digest(input: &[Segment]) -> String { .collect() } +fn submission_uploaded_file_refs( + input: &[Segment], +) -> impl Iterator { + input.iter().filter_map(|segment| match segment { + Segment::UploadedFile { file } => Some(file), + _ => None, + }) +} + fn submission_artifact_ref_count(input: &[Segment]) -> usize { input .iter() @@ -1191,11 +1275,57 @@ where Ok(()) } + fn pin_submission_files( + &self, + pending: &PendingSubmission, + ) -> Result<(), PendingSubmissionError> { + let session_id = self.writer.state.location().session_id; + for reference in submission_uploaded_file_refs(&pending.input) { + self.writer + .store + .pin_uploaded_file(session_id, reference, &pending.submission_id)?; + } + Ok(()) + } + + fn release_submission_files( + &self, + pending: &PendingSubmission, + ) -> Result<(), PendingSubmissionError> { + let session_id = self.writer.state.location().session_id; + for reference in submission_uploaded_file_refs(&pending.input) { + self.writer.store.release_uploaded_file_pin( + session_id, + &reference.artifact_id, + &pending.submission_id, + )?; + } + Ok(()) + } + + #[cfg(test)] pub(crate) fn accept( &self, submission_request_id: String, input: Vec, activate_now: bool, + ) -> Result { + self.accept_from_source( + submission_request_id, + input, + self.direct_client_namespace(), + WorkerHistoryProvenance::LegacyUnknown, + activate_now, + ) + } + + pub(crate) fn accept_from_source( + &self, + submission_request_id: String, + input: Vec, + source_namespace: String, + provenance: WorkerHistoryProvenance, + activate_now: bool, ) -> Result { if submission_request_id.trim().is_empty() { return Err(PendingSubmissionError::EmptyRequestId); @@ -1218,11 +1348,10 @@ where .lock() .expect("pending activation state poisoned"); let original = current.clone(); - if let Some(receipt) = current - .receipts - .iter() - .find(|receipt| receipt.submission_request_id == submission_request_id) - { + if let Some(receipt) = current.receipts.iter().find(|receipt| { + receipt.submission_request_id == submission_request_id + && receipt.source_namespace == source_namespace + }) { if receipt.payload_digest != payload_digest { return Err(PendingSubmissionError::IdempotencyConflict); } @@ -1237,11 +1366,12 @@ where let submission_id = uuid::Uuid::now_v7().to_string(); let pending = PendingSubmission { submission_request_id: submission_request_id.clone(), + source_namespace: source_namespace.clone(), submission_id: submission_id.clone(), payload_digest: payload_digest.clone(), accepted_at_ms: segment_log::now_millis(), activation_sequence: current.next_activation_sequence, - provenance: WorkerHistoryProvenance::LegacyUnknown, + provenance, was_queued: !activate_now, input, }; @@ -1253,6 +1383,7 @@ where }; current.remember_receipt(SubmissionReceipt { submission_request_id: submission_request_id.clone(), + source_namespace, submission_id: submission_id.clone(), payload_digest, disposition, @@ -1299,7 +1430,14 @@ where return Err(PendingSubmissionError::ArtifactLimit); } current.pending.push_back(pending.clone()); + } + if !activate_now { + if let Err(error) = self.pin_submission_files(&pending) { + *current = original; + return Err(error); + } if let Err(error) = self.persist_locked(¤t) { + let _ = self.release_submission_files(&pending); *current = original; return Err(error); } @@ -1312,10 +1450,29 @@ where }) } + #[cfg(test)] pub(crate) fn accept_notification( &self, notification_request_id: String, message: String, + auto_run: bool, + ) -> Result { + self.accept_notification_from_source( + notification_request_id, + message, + self.direct_client_namespace(), + WorkerHistoryProvenance::LegacyUnknown, + auto_run, + ) + } + + pub(crate) fn accept_notification_from_source( + &self, + notification_request_id: String, + message: String, + source_namespace: String, + provenance: WorkerHistoryProvenance, + auto_run: bool, ) -> Result { if notification_request_id.trim().is_empty() { return Err(PendingSubmissionError::EmptyRequestId); @@ -1323,7 +1480,7 @@ where if notification_request_id.len() > MAX_ACTIVATION_REQUEST_ID_BYTES { return Err(PendingSubmissionError::RequestIdLimit); } - let payload_digest = submission_payload_digest(&[Segment::text(message.clone())]); + let payload_digest = notification_payload_digest(&message, auto_run); let _append_guard = self .writer .state @@ -1334,12 +1491,11 @@ where .state .lock() .expect("pending activation state poisoned"); - if let Some(receipt) = state - .notification_receipts - .iter() - .find(|receipt| receipt.notification_request_id == notification_request_id) - { - if receipt.payload_digest != payload_digest { + if let Some(receipt) = state.notification_receipts.iter().find(|receipt| { + receipt.notification_request_id == notification_request_id + && receipt.source_namespace == source_namespace + }) { + if receipt.payload_digest != payload_digest || receipt.auto_run != auto_run { return Err(PendingSubmissionError::IdempotencyConflict); } return Ok(false); @@ -1373,17 +1529,19 @@ where state.next_activation_sequence = state.next_activation_sequence.saturating_add(1); state.pending_notifications.push_back(PendingNotification { notification_request_id: notification_request_id.clone(), + source_namespace: source_namespace.clone(), message, payload_digest: payload_digest.clone(), + auto_run, accepted_at_ms: segment_log::now_millis(), activation_sequence, - provenance: WorkerHistoryProvenance::BackendInstruction { - operation_id: Some(notification_request_id.clone()), - }, + provenance, }); state.remember_notification_receipt(NotificationReceipt { notification_request_id, + source_namespace, payload_digest, + auto_run, }); state.revision = state.revision.saturating_add(1); if let Err(error) = self.persist_locked(&state) { @@ -1393,6 +1551,59 @@ where Ok(true) } + pub(crate) fn activating_passive_notification_id(&self) -> Option { + self.state + .lock() + .expect("pending activation state poisoned") + .activating_notification + .as_ref() + .filter(|notification| !notification.auto_run) + .map(|notification| notification.notification_request_id.clone()) + } + + pub(crate) fn next_passive_notification_identity(&self) -> Option<(String, String)> { + self.state + .lock() + .expect("pending activation state poisoned") + .pending_notifications + .iter() + .find(|notification| !notification.auto_run) + .map(|notification| { + ( + notification.source_namespace.clone(), + notification.notification_request_id.clone(), + ) + }) + } + + pub(crate) fn prepare_notification( + &self, + source_namespace: &str, + notification_request_id: &str, + ) -> Option { + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if state.activating_notification.is_some() { + return None; + } + let index = state + .pending_notifications + .iter() + .position(|notification| { + notification.notification_request_id == notification_request_id + && notification.source_namespace == source_namespace + })?; + let notification = state + .pending_notifications + .remove(index) + .expect("located pending notification must exist"); + state.activating_notification = Some(notification.clone()); + state.revision = state.revision.saturating_add(1); + Some(notification) + } + pub(crate) fn prepare_next_activation( &self, fence: Option<(u64, &str)>, @@ -1414,17 +1625,20 @@ where return Ok(None); } let submission_sequence = state.pending.front().map(|item| item.activation_sequence); - let notification_sequence = state + let notification_index = state .pending_notifications - .front() + .iter() + .position(|item| item.auto_run); + let notification_sequence = notification_index + .and_then(|index| state.pending_notifications.get(index)) .map(|item| item.activation_sequence); if notification_sequence.is_some() && (submission_sequence.is_none() || notification_sequence < submission_sequence) { let notification = state .pending_notifications - .pop_front() - .expect("notification sequence came from queue head"); + .remove(notification_index.expect("notification sequence came from an item")) + .expect("notification sequence came from an existing item"); state.activating_notification = Some(notification.clone()); state.revision = state.revision.saturating_add(1); return Ok(Some(PendingActivation::Notification(notification))); @@ -1442,6 +1656,12 @@ where } pub(crate) fn abort_activation(&self, pending: PendingSubmission) { + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); let mut state = self .state .lock() @@ -1452,13 +1672,16 @@ where } state.activating = None; if pending.was_queued { - state.pending.push_front(pending); + state.pending.push_front(pending.clone()); } else { state .receipts .retain(|receipt| receipt.submission_id != pending.submission_id); } state.revision = state.revision.saturating_add(1); + if let Err(error) = self.persist_locked(&state) { + tracing::error!(error = %error, "failed to persist aborted pending activation"); + } } pub(crate) fn activation_extension(&self) -> SessionExtension { @@ -1530,6 +1753,10 @@ where } } + pub(crate) fn direct_client_namespace(&self) -> String { + format!("direct:{}", self.writer.state.location().session_id) + } + pub(crate) fn snapshot(&self) -> protocol::PendingSubmissionsSnapshot { self.state .lock() @@ -1561,12 +1788,16 @@ where else { return Err(PendingSubmissionError::NotFound(submission_id.to_owned())); }; - state.pending.remove(index); + let removed = state + .pending + .remove(index) + .expect("located pending submission must exist"); state.revision = state.revision.saturating_add(1); if let Err(error) = self.persist_locked(&state) { *state = original; return Err(error); } + self.release_submission_files(&removed)?; Ok(state.snapshot()) } @@ -1586,13 +1817,16 @@ where .expect("pending activation state poisoned"); Self::validate_fence(&state, expected_revision, None)?; let original = state.clone(); - state.pending.clear(); + let removed = state.pending.drain(..).collect::>(); state.pending_notifications.clear(); state.revision = state.revision.saturating_add(1); if let Err(error) = self.persist_locked(&state) { *state = original; return Err(error); } + for pending in &removed { + self.release_submission_files(pending)?; + } Ok(state.snapshot()) } } @@ -1627,9 +1861,11 @@ pub trait SystemItemCommitter: Send + Sync { &self, item: SystemItem, extensions: Vec, + history_provenance: Option, ) -> Result, StoreError> { let metadata = new_history_metadata( - WorkerHistoryProvenance::BackendInstruction { operation_id: None }, + history_provenance + .unwrap_or(WorkerHistoryProvenance::BackendInstruction { operation_id: None }), None, ); let history_item = item.to_history_item(); @@ -2666,9 +2902,11 @@ impl Worker { .expect("pending activation state poisoned") .clone(); if !pending_state.pending.is_empty() + || !pending_state.pending_notifications.is_empty() || pending_state.activating.is_some() || pending_state.activating_notification.is_some() || !pending_state.receipts.is_empty() + || !pending_state.notification_receipts.is_empty() { let checkpoint = LogEntry::Extension { ts: segment_log::now_millis(), @@ -3501,8 +3739,13 @@ impl Worker { where St: Clone + 'static, { - self.run_with_input_extensions_and_commit_hook(input, input_extensions, || {}) - .await + self.run_with_input_extensions_and_commit_hook( + input, + input_extensions, + WorkerHistoryProvenance::LegacyUnknown, + || {}, + ) + .await } /// Run user input and invoke `on_input_committed` only after the annotated @@ -3513,6 +3756,7 @@ impl Worker { &mut self, input: Vec, mut input_extensions: Vec, + input_provenance: WorkerHistoryProvenance, on_input_committed: F, ) -> Result where @@ -3560,8 +3804,12 @@ impl Worker { trigger: protocol::InvokeKind::UserSend, })?; - let projected_input = - self.projected_input_history(&input, flow_projection.as_ref(), &projected_entry_ids); + let projected_input = self.projected_input_history( + &input, + flow_projection.as_ref(), + &projected_entry_ids, + &input_provenance, + ); // Persist original typed segments together with the exact ordered // model-visible item+origin projection before any entry becomes live. @@ -3867,6 +4115,7 @@ impl Worker { input: &[Segment], flow_projection: Option<&PreparedFlowProjection>, entry_ids: &[SessionHistoryEntryId], + provenance: &WorkerHistoryProvenance, ) -> Vec> { if let Some(flow) = flow_projection { return input @@ -3887,10 +4136,7 @@ impl Worker { other => history_entry_with_id( Item::user_message(Segment::flatten_to_text(std::slice::from_ref(other))), entry_id.clone(), - // Current public submit transport does not carry a - // trusted account/Worker subject envelope. Fail closed - // instead of promoting role=user to HumanInput. - WorkerHistoryProvenance::LegacyUnknown, + provenance.clone(), ), }) .collect(); @@ -3902,7 +4148,7 @@ impl Worker { .first() .expect("projected Worker input always has one entry id") .clone(), - WorkerHistoryProvenance::LegacyUnknown, + provenance.clone(), )] } @@ -7938,8 +8184,15 @@ mod build_summary_prompt_tests { serde_json::to_value(&state).unwrap(), ); let projected_ids = vec![SessionHistoryEntryId::new(), SessionHistoryEntryId::new()]; - let projected = - worker.projected_input_history(&segments, projection.as_ref(), &projected_ids); + let input_provenance = WorkerHistoryProvenance::HumanInput { + account_id: "account-1".into(), + }; + let projected = worker.projected_input_history( + &segments, + projection.as_ref(), + &projected_ids, + &input_provenance, + ); worker .commit_entry(LogEntry::AnnotatedUserInput { ts: segment_log::now_millis(), @@ -7966,6 +8219,7 @@ mod build_summary_prompt_tests { projected[0].annotation.origin, WorkerHistoryProvenance::FlowInstruction { .. } )); + assert_eq!(projected[1].annotation.origin, input_provenance); assert_eq!(state.instance.definition_revision, 3); assert_eq!(state.instance.current_state.as_str(), "implement"); assert_eq!(workspace_client.requests.lock().unwrap().len(), 1); @@ -8048,7 +8302,12 @@ mod build_summary_prompt_tests { .delete_uploaded_file(worker.session_id(), &file.artifact_id), Err(StoreError::ArtifactAlreadyCommitted) )); - let projected = worker.projected_input_history(&input, None, &[entry_id]); + let projected = worker.projected_input_history( + &input, + None, + &[entry_id], + &WorkerHistoryProvenance::LegacyUnknown, + ); let text = projected[0].item.as_text().unwrap(); assert!(text.contains("notes.md")); assert!(text.contains(&file.artifact_id)); @@ -8130,7 +8389,12 @@ mod build_summary_prompt_tests { if retained.source_entry_id == artifact.source_entry_id )); - let history = worker.projected_input_history(&input, None, &[entry_id]); + let history = worker.projected_input_history( + &input, + None, + &[entry_id], + &WorkerHistoryProvenance::LegacyUnknown, + ); assert!(!history[0].item.as_text().unwrap().contains("終端")); append_test_entry( &worker, @@ -8406,6 +8670,47 @@ mod build_summary_prompt_tests { assert_eq!(worker.history()[0].as_text().unwrap(), "first message"); } + #[tokio::test] + async fn rewind_preserves_notification_only_pending_activation_checkpoint() { + let (_dir, mut worker) = rewind_test_worker().await; + append_user_turn(&worker, 10, "first message"); + append_user_turn(&worker, 20, "second message"); + worker + .pending_submission_handle() + .accept_notification("notification-1".into(), "keep me".into(), true) + .unwrap(); + let (head_entries, targets) = worker.list_rewind_targets().unwrap(); + + worker + .rewind_to(targets.last().unwrap().id.clone(), head_entries) + .await + .unwrap(); + + let location = worker.segment_state.location(); + let entries = worker + .store + .read_all(location.session_id, location.segment_id) + .unwrap(); + let restored: PendingActivationState = entries + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN => { + serde_json::from_value(payload.clone()).ok() + } + _ => None, + }) + .unwrap(); + assert_eq!(restored.pending_notifications.len(), 1); + assert_eq!(restored.notification_receipts.len(), 1); + assert_eq!( + restored.pending_notifications[0].notification_request_id, + "notification-1" + ); + } + #[tokio::test] async fn annotated_history_rewind_commits_authoritative_prefix() { let (_dir, mut worker) = rewind_test_worker().await; @@ -9251,6 +9556,110 @@ mod build_summary_prompt_tests { ); } + #[test] + fn submission_retry_identity_is_scoped_to_authenticated_source_and_keeps_provenance() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let input = vec![Segment::text("same request")]; + let account_a = WorkerHistoryProvenance::HumanInput { + account_id: "account-a".into(), + }; + let account_b = WorkerHistoryProvenance::HumanInput { + account_id: "account-b".into(), + }; + let first = handle + .accept_from_source( + "request-1".into(), + input.clone(), + "account:account-a".into(), + account_a.clone(), + false, + ) + .unwrap(); + let replay = handle + .accept_from_source( + "request-1".into(), + input.clone(), + "account:account-a".into(), + account_a.clone(), + false, + ) + .unwrap(); + let other_source = handle + .accept_from_source( + "request-1".into(), + input, + "account:account-b".into(), + account_b.clone(), + false, + ) + .unwrap(); + + assert_eq!(replay.submission_id, first.submission_id); + assert_ne!(other_source.submission_id, first.submission_id); + let state = handle.state.lock().unwrap(); + assert_eq!(state.pending[0].provenance, account_a); + assert_eq!(state.pending[1].provenance, account_b); + } + + #[test] + fn queued_submission_pins_uploaded_file_until_cancelled() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let session_id = handle.writer.state.session_id(); + let reference = handle + .writer + .store + .write_uploaded_file( + session_id, + "queued.txt", + "text/plain", + b"queued artifact", + session_store::UploadedFileLimits { + max_file_bytes: 1024, + max_session_bytes: 2048, + }, + ) + .unwrap(); + let accepted = handle + .accept( + "artifact-request".into(), + vec![Segment::UploadedFile { + file: reference.clone(), + }], + false, + ) + .unwrap(); + + assert_eq!( + handle + .writer + .store + .delete_uncommitted_uploaded_files(session_id) + .unwrap(), + 0 + ); + assert!( + handle + .writer + .store + .read_uploaded_file_by_id(session_id, &reference.artifact_id) + .is_ok() + ); + + handle + .cancel(&accepted.submission_id, handle.snapshot().revision) + .unwrap(); + assert_eq!( + handle + .writer + .store + .delete_uncommitted_uploaded_files(session_id) + .unwrap(), + 1 + ); + } + #[test] fn pending_submission_queue_is_durable_idempotent_and_bounded() { let temp = tempfile::tempdir().unwrap(); @@ -9339,22 +9748,96 @@ mod build_summary_prompt_tests { assert_eq!(cleared.notification_count, 0); } + #[test] + fn durable_notification_commits_authenticated_history_provenance() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let provenance = WorkerHistoryProvenance::HumanInput { + account_id: "account-1".into(), + }; + let committed = handle + .writer + .commit_system_item_with_extensions( + SystemItem::Notification { + message: "notice".into(), + body: "notice".into(), + prompt_provenance: None, + }, + Vec::new(), + Some(provenance.clone()), + ) + .unwrap(); + assert_eq!(committed.annotation.origin, provenance); + } + + #[test] + fn notification_retry_identity_is_scoped_to_authenticated_source() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let account_a = WorkerHistoryProvenance::HumanInput { + account_id: "account-a".into(), + }; + let account_b = WorkerHistoryProvenance::HumanInput { + account_id: "account-b".into(), + }; + assert!( + handle + .accept_notification_from_source( + "request-1".into(), + "notice".into(), + "account:account-a".into(), + account_a.clone(), + false, + ) + .unwrap() + ); + assert!( + !handle + .accept_notification_from_source( + "request-1".into(), + "notice".into(), + "account:account-a".into(), + account_a.clone(), + false, + ) + .unwrap() + ); + assert!( + handle + .accept_notification_from_source( + "request-1".into(), + "notice".into(), + "account:account-b".into(), + account_b.clone(), + false, + ) + .unwrap() + ); + let state = handle.state.lock().unwrap(); + assert_eq!(state.pending_notifications[0].provenance, account_a); + assert_eq!(state.pending_notifications[1].provenance, account_b); + } + #[test] fn notification_and_submit_share_activation_order_and_notification_dedupes() { let temp = tempfile::tempdir().unwrap(); let handle = PendingSubmissionHandle::for_test(temp.path()); assert!( handle - .accept_notification("notification-1".into(), "notice".into()) + .accept_notification("notification-1".into(), "notice".into(), true) .unwrap() ); assert!( !handle - .accept_notification("notification-1".into(), "notice".into()) + .accept_notification("notification-1".into(), "notice".into(), true) .unwrap() ); assert!(matches!( - handle.accept_notification("notification-1".into(), "different".into()), + handle.accept_notification("notification-1".into(), "different".into(), true), + Err(PendingSubmissionError::IdempotencyConflict) + )); + assert!(matches!( + handle.accept_notification("notification-1".into(), "notice".into(), false), Err(PendingSubmissionError::IdempotencyConflict) )); handle @@ -9382,9 +9865,10 @@ mod build_summary_prompt_tests { let mut session = WorkerSession::new(session_store::new_session_id(), Vec::new()); let state = PendingActivationState { revision: 4, - next_activation_sequence: 2, + next_activation_sequence: 3, activating: Some(PendingSubmission { submission_request_id: "request-1".into(), + source_namespace: "direct:test".into(), submission_id: "submission-1".into(), payload_digest: submission_payload_digest(&[Segment::text("first")]), accepted_at_ms: 1, @@ -9393,9 +9877,21 @@ mod build_summary_prompt_tests { was_queued: false, input: vec![Segment::text("first")], }), - activating_notification: None, + activating_notification: Some(PendingNotification { + notification_request_id: "notification-1".into(), + source_namespace: "account:account-1".into(), + message: "deferred notice".into(), + payload_digest: notification_payload_digest("deferred notice", false), + auto_run: false, + accepted_at_ms: 3, + activation_sequence: 2, + provenance: WorkerHistoryProvenance::HumanInput { + account_id: "account-1".into(), + }, + }), pending: VecDeque::from([PendingSubmission { submission_request_id: "request-2".into(), + source_namespace: "direct:test".into(), submission_id: "submission-2".into(), payload_digest: submission_payload_digest(&[Segment::text("second")]), accepted_at_ms: 2, @@ -9406,7 +9902,12 @@ mod build_summary_prompt_tests { }]), pending_notifications: VecDeque::new(), receipts: VecDeque::new(), - notification_receipts: VecDeque::new(), + notification_receipts: VecDeque::from([NotificationReceipt { + notification_request_id: "notification-1".into(), + source_namespace: "account:account-1".into(), + payload_digest: notification_payload_digest("deferred notice", false), + auto_run: false, + }]), }; session.restore_pending_activations(&[( SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.into(), @@ -9420,6 +9921,14 @@ mod build_summary_prompt_tests { assert_eq!(state.pending.len(), 2); assert_eq!(state.pending[0].submission_id, "submission-1"); assert_eq!(state.pending[1].submission_id, "submission-2"); + assert!(state.activating_notification.is_none()); + assert_eq!(state.pending_notifications.len(), 1); + assert!(!state.pending_notifications[0].auto_run); + assert!(matches!( + state.pending_notifications[0].provenance, + WorkerHistoryProvenance::HumanInput { ref account_id } if account_id == "account-1" + )); + assert_eq!(state.notification_receipts.len(), 1); } fn minimal_manifest() -> WorkerManifest { diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 3f6438fe..319f773d 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -1804,15 +1804,18 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() { let client_for_assert = client.clone(); let worker = make_worker(client).await; let handle = spawn_controller(worker).await; + let notification_request_id = protocol::new_submission_request_id(); - handle - .send(Method::Notify { - notification_request_id: protocol::new_submission_request_id(), - message: "progress snapshot".into(), - auto_run: false, - }) - .await - .unwrap(); + for _ in 0..2 { + handle + .send(Method::Notify { + notification_request_id: notification_request_id.clone(), + message: "progress snapshot".into(), + auto_run: false, + }) + .await + .unwrap(); + } tokio::time::sleep(std::time::Duration::from_millis(100)).await; assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); @@ -2049,6 +2052,66 @@ async fn notify_while_running_does_not_emit_already_running_error() { wait_for_status(&handle, WorkerStatus::Idle).await; } +#[tokio::test] +async fn weak_notify_while_running_is_deduped_and_survives_until_next_submit() { + let client = MockClient::sequential(vec![ + MockResponse::Hang(Vec::new()), + MockResponse::Complete(simple_text_events()), + ]); + let client_for_assert = client.clone(); + let worker = make_worker(client).await; + let handle = spawn_controller(worker).await; + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "first", + )) + .await + .unwrap(); + wait_for_status(&handle, WorkerStatus::Running).await; + + let notification_request_id = protocol::new_submission_request_id(); + for _ in 0..2 { + handle + .send(Method::Notify { + notification_request_id: notification_request_id.clone(), + message: "durable weak notice".into(), + auto_run: false, + }) + .await + .unwrap(); + } + handle.send(Method::Cancel).await.unwrap(); + wait_for_status(&handle, WorkerStatus::Idle).await; + + let mut rx = handle.subscribe(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "second", + )) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + if matches!(rx.recv().await, Ok(Event::TurnEnd { .. })) { + break; + } + } + }) + .await + .expect("second submit completes"); + + let requests = client_for_assert.captured_requests(); + let notice_count = requests[1] + .items + .iter() + .filter_map(|item| item.as_text()) + .filter(|text| text.contains("durable weak notice")) + .count(); + assert_eq!(notice_count, 1); +} + #[tokio::test] async fn status_json_reflects_worker_name() { let client = MockClient::new(simple_text_events()); diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index f86a1d00..f7750e74 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -4987,7 +4987,7 @@ mod tests { "missing test context", ); }; - let submission_id = input.submission_id.clone(); + let submission_request_id = input.submission_request_id.clone(); let content = input.content; std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(10)); @@ -5004,11 +5004,13 @@ mod tests { status: protocol::WorkerStatus::Idle, }); }); - if let Some(submission_id) = submission_id { - worker_runtime::execution::WorkerExecutionResult::accepted_input_committed( + if let Some(submission_request_id) = submission_request_id { + worker_runtime::execution::WorkerExecutionResult::accepted_submission( worker_runtime::execution::WorkerExecutionOperation::Input, WorkerExecutionRunState::Busy, - submission_id, + submission_request_id, + uuid::Uuid::now_v7().to_string(), + protocol::SubmissionDisposition::Started, ) } else { worker_runtime::execution::WorkerExecutionResult::accepted( diff --git a/crates/workspace-server/src/runtime_subscription_tests.rs b/crates/workspace-server/src/runtime_subscription_tests.rs index a42c0c02..6c14c56a 100644 --- a/crates/workspace-server/src/runtime_subscription_tests.rs +++ b/crates/workspace-server/src/runtime_subscription_tests.rs @@ -32,11 +32,13 @@ impl WorkerExecutionBackend for TestExecutionBackend { _handle: &WorkerExecutionHandle, input: worker_runtime::interaction::WorkerInput, ) -> WorkerExecutionResult { - if let Some(submission_id) = input.submission_id { - WorkerExecutionResult::accepted_input_committed( + if let Some(submission_request_id) = input.submission_request_id { + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, WorkerExecutionRunState::Busy, - submission_id, + submission_request_id, + uuid::Uuid::now_v7().to_string(), + protocol::SubmissionDisposition::Started, ) } else { WorkerExecutionResult::accepted( diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 3599f26d..d53b239d 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -8537,13 +8537,15 @@ async fn scoped_list_runtimes( async fn scoped_workspace_protocol_ws( State(api): State, + Extension(actor): Extension, AxumPath(workspace_id): AxumPath, ws: axum::extract::ws::WebSocketUpgrade, ) -> std::result::Result { validate_workspace_scope(&api, &workspace_id).map_err(|error| error.into_response())?; + let input_source = authenticated_browser_input_source(&actor); Ok(ws .on_upgrade(move |socket| { - crate::workspace_subscription::serve_workspace_subscription(api, socket) + crate::workspace_subscription::serve_workspace_subscription(api, socket, input_source) }) .into_response()) } @@ -11458,6 +11460,7 @@ async fn scoped_cancel_runtime_worker( async fn scoped_worker_protocol_ws( ws: WebSocketUpgrade, State(api): State, + Extension(actor): Extension, AxumPath(path): AxumPath, ) -> Response { if let Err(err) = validate_workspace_scope(&api, &path.workspace_id) { @@ -11465,6 +11468,7 @@ async fn scoped_worker_protocol_ws( } worker_protocol_ws( State(api), + Extension(actor), AxumPath((path.worker.runtime_id, path.worker.worker_id)), ws, ) @@ -13877,8 +13881,45 @@ async fn cancel_runtime_worker( Ok(Json(result)) } +fn authenticated_browser_input_source(actor: &RequestActor) -> protocol::AuthenticatedInputSource { + protocol::AuthenticatedInputSource::Account { + account_id: actor.account_id.clone(), + } +} + +pub(crate) fn authorize_browser_worker_method( + method: protocol::Method, + source: &protocol::AuthenticatedInputSource, +) -> std::result::Result { + match method { + protocol::Method::Submit { + submission_request_id, + input, + } => Ok(protocol::Method::SubmitTracked { + submission_request_id, + input, + source: source.clone(), + }), + protocol::Method::Notify { + notification_request_id, + message, + auto_run, + } => Ok(protocol::Method::NotifyTracked { + notification_request_id, + message, + auto_run, + source: source.clone(), + }), + protocol::Method::SubmitTracked { .. } | protocol::Method::NotifyTracked { .. } => { + Err("authenticated Worker input source is server-owned") + } + other => Ok(other), + } +} + async fn worker_protocol_ws( State(api): State, + Extension(actor): Extension, AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, ws: WebSocketUpgrade, ) -> impl IntoResponse { @@ -13902,7 +13943,8 @@ async fn worker_protocol_ws( .into_response(); } }; - ws.on_upgrade(move |socket| worker_protocol_ws_session(source, socket)) + let input_source = authenticated_browser_input_source(&actor); + ws.on_upgrade(move |socket| worker_protocol_ws_session(source, socket, input_source)) } pub(crate) struct WorkspaceWorkerProtocolConnection { @@ -14029,13 +14071,17 @@ async fn connect_embedded_worker_protocol( Ok(WorkspaceWorkerProtocolConnection { methods, events }) } -async fn worker_protocol_ws_session(source: RuntimeObservationSource, socket: WebSocket) { +async fn worker_protocol_ws_session( + source: RuntimeObservationSource, + socket: WebSocket, + input_source: protocol::AuthenticatedInputSource, +) { match source { RuntimeObservationSource::RemoteWs(config) => { - remote_worker_protocol_ws_session(config, socket).await; + remote_worker_protocol_ws_session(config, socket, input_source).await; } RuntimeObservationSource::Embedded(source) => { - embedded_worker_protocol_ws_session(source, socket).await; + embedded_worker_protocol_ws_session(source, socket, input_source).await; } } } @@ -14043,6 +14089,7 @@ async fn worker_protocol_ws_session(source: RuntimeObservationSource, socket: We async fn remote_worker_protocol_ws_session( config: RuntimeObservationSourceConfig, socket: WebSocket, + input_source: protocol::AuthenticatedInputSource, ) { let mut request = match config.endpoint.clone().into_client_request() { Ok(request) => request, @@ -14091,14 +14138,33 @@ async fn remote_worker_protocol_ws_session( inbound = client_stream.next() => { match inbound { Some(Ok(WsMessage::Text(text))) => { - if upstream_sink.send(TungsteniteMessage::Text(text.to_string().into())).await.is_err() { + let method = match protocol::stream::decode_method(text.as_ref()) { + Ok(method) => match authorize_browser_worker_method(method, &input_source) { + Ok(method) => method, + Err(message) => { + if let Ok(event) = protocol::stream::encode_event(&protocol_error_event(message)) { + let _ = client_sink.send(WsMessage::Text(event.into())).await; + } + break; + } + }, + Err(error) => { + if let Ok(event) = protocol::stream::encode_event(&protocol_error_event(error.to_string())) { + let _ = client_sink.send(WsMessage::Text(event.into())).await; + } + break; + } + }; + let Ok(method) = protocol::stream::encode_method(&method) else { break }; + if upstream_sink.send(TungsteniteMessage::Text(method.into())).await.is_err() { break; } } - Some(Ok(WsMessage::Binary(binary))) => { - if upstream_sink.send(TungsteniteMessage::Binary(binary.to_vec().into())).await.is_err() { - break; + Some(Ok(WsMessage::Binary(_))) => { + if let Ok(event) = protocol::stream::encode_event(&protocol_error_event("binary Worker methods are not accepted")) { + let _ = client_sink.send(WsMessage::Text(event.into())).await; } + break; } Some(Ok(WsMessage::Close(_))) | None => { let _ = upstream_sink.send(TungsteniteMessage::Close(None)).await; @@ -14154,6 +14220,7 @@ async fn remote_worker_protocol_ws_session( async fn embedded_worker_protocol_ws_session( source: crate::observation::EmbeddedRuntimeObservationSource, mut socket: WebSocket, + input_source: protocol::AuthenticatedInputSource, ) { let mut upstream = match RuntimeObservationClient::connect(&RuntimeObservationSource::Embedded( source.clone(), @@ -14173,24 +14240,32 @@ async fn embedded_worker_protocol_ws_session( inbound = socket.next() => { match inbound { Some(Ok(WsMessage::Text(text))) => match decode_method(&text) { - Ok(method) => match source.runtime.send_protocol_method(&source.worker_ref, method) { - Ok(events) => { - for event in events { + Ok(method) => match authorize_browser_worker_method(method, &input_source) { + Ok(method) => match source.runtime.send_protocol_method(&source.worker_ref, method) { + Ok(events) => { + for event in events { + if !send_protocol_event(&mut socket, &event).await { + return; + } + } + } + Err(error) => { + let event = protocol_error_event(error.to_string()); if !send_protocol_event(&mut socket, &event).await { return; } } - } - Err(error) => { - let event = protocol_error_event(error.to_string()); - if !send_protocol_event(&mut socket, &event).await { - return; - } + }, + Err(message) => { + let event = protocol_error_event(message); + let _ = send_protocol_event(&mut socket, &event).await; + return; } }, Err(error) => { - let event = - protocol_error_event(format!("malformed protocol method frame: {error}")); + let event = protocol_error_event(format!( + "malformed protocol method frame: {error}" + )); if !send_protocol_event(&mut socket, &event).await { return; } @@ -16580,6 +16655,29 @@ mod tests { &tail[..end] } + #[test] + fn browser_worker_methods_receive_server_owned_account_source() { + let source = protocol::AuthenticatedInputSource::Account { + account_id: "account-1".into(), + }; + let method = authorize_browser_worker_method( + protocol::Method::Submit { + submission_request_id: "request-1".into(), + input: vec![protocol::Segment::text("hello")], + }, + &source, + ) + .unwrap(); + assert!(matches!( + method, + protocol::Method::SubmitTracked { + source: protocol::AuthenticatedInputSource::Account { ref account_id }, + .. + } if account_id == "account-1" + )); + assert!(authorize_browser_worker_method(method, &source).is_err()); + } + #[test] fn merge_request_http_paths_observe_refs_through_runtime_provider_authority() { let source = include_str!("server.rs"); @@ -18825,7 +18923,7 @@ mod tests { .get(handle.worker_ref()) .cloned() .expect("execution context"); - let submission_id = input.submission_id.clone(); + let submission_request_id = input.submission_request_id.clone(); let content = input.content.clone(); std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(25)); @@ -18833,11 +18931,13 @@ mod tests { text: format!("server companion echoed: {content}"), }); }); - if let Some(submission_id) = submission_id { - worker_runtime::execution::WorkerExecutionResult::accepted_input_committed( + if let Some(submission_request_id) = submission_request_id { + worker_runtime::execution::WorkerExecutionResult::accepted_submission( worker_runtime::execution::WorkerExecutionOperation::Input, worker_runtime::execution::WorkerExecutionRunState::Idle, - submission_id, + submission_request_id, + uuid::Uuid::now_v7().to_string(), + protocol::SubmissionDisposition::Started, ) } else { worker_runtime::execution::WorkerExecutionResult::accepted( @@ -27223,6 +27323,16 @@ mod tests { (runtime, worker_ref, endpoint) } + fn test_browser_request_actor() -> RequestActor { + RequestActor { + user_id: "test-user".into(), + account_id: format!("account-{TEST_WORKSPACE_ID}"), + handle: "test".into(), + display_name: "Test".into(), + auth_method: ActorAuthMethod::BrowserSession, + } + } + async fn spawn_workspace_proxy( source: RuntimeObservationSourceConfig, ) -> (String, tempfile::TempDir) { @@ -27241,11 +27351,8 @@ mod tests { .unwrap(); let app_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let app_addr = app_listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(app_listener, build_inner_router(api)) - .await - .unwrap() - }); + let app = build_inner_router(api).layer(Extension(test_browser_request_actor())); + tokio::spawn(async move { axum::serve(app_listener, app).await.unwrap() }); ( format!("ws://{app_addr}/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws"), dir, @@ -27257,7 +27364,8 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - let app = build_inner_router(test_api(dir.path()).await); + let app = build_inner_router(test_api(dir.path()).await) + .layer(Extension(test_browser_request_actor())); let server = tokio::spawn(async move { let _ = axum::serve(listener, app).await; }); @@ -27305,7 +27413,7 @@ mod tests { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - let app = build_inner_router(api); + let app = build_inner_router(api).layer(Extension(test_browser_request_actor())); let server = tokio::spawn(async move { let _ = axum::serve(listener, app).await; }); diff --git a/crates/workspace-server/src/workspace_subscription.rs b/crates/workspace-server/src/workspace_subscription.rs index fe26f7f9..d72017f1 100644 --- a/crates/workspace-server/src/workspace_subscription.rs +++ b/crates/workspace-server/src/workspace_subscription.rs @@ -11,7 +11,9 @@ use tokio::sync::mpsc; use worker_runtime::identity::RuntimeWorkerRef; use crate::runtime_subscription::{BrokerSubscriptionEvent, RuntimeSubscriptionBroker}; -use crate::server::{WorkspaceApi, connect_workspace_worker_protocol}; +use crate::server::{ + WorkspaceApi, authorize_browser_worker_method, connect_workspace_worker_protocol, +}; use crate::store::WorkspaceResourceKind; const OUTBOUND_CAPACITY: usize = 256; @@ -21,7 +23,11 @@ struct ActiveSubscription { methods: Option>, } -pub(crate) async fn serve_workspace_subscription(api: WorkspaceApi, socket: WebSocket) { +pub(crate) async fn serve_workspace_subscription( + api: WorkspaceApi, + socket: WebSocket, + input_source: protocol::AuthenticatedInputSource, +) { let broker = api.runtime_subscription_broker().clone(); let (mut socket_sender, mut socket_receiver) = socket.split(); let (control_outbound, mut control_receiver) = mpsc::channel::(OUTBOUND_CAPACITY); @@ -153,7 +159,12 @@ pub(crate) async fn serve_workspace_subscription(api: WorkspaceApi, socket: WebS else { break; }; - if methods.send(message.method).await.is_err() { + let Ok(method) = + authorize_browser_worker_method(message.method, &input_source) + else { + break; + }; + if methods.send(method).await.is_err() { break; } } From 5e9f7a7dc3d1169118412376f16b72850aab63f2 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 02:17:26 +0900 Subject: [PATCH 05/39] feat: add versioned runtime connection ping --- crates/worker-runtime/src/http_server.rs | 137 ++- crates/workspace-api/src/lib.rs | 62 +- crates/workspace-server/src/hosts.rs | 224 ++++- crates/workspace-server/src/server.rs | 873 +++++++----------- web/workspace/deno.json | 2 +- .../src/lib/generated/workspace-api.ts | 24 + .../lib/workspace/api/runtime-connection.ts | 142 +++ .../console/worker-console.ui.test.ts | 11 +- .../src/lib/workspace/styles/settings.css | 3 + .../settings/runtimes/+page.svelte | 56 +- .../tests/runtime-connection.test.ts | 103 +++ 11 files changed, 1053 insertions(+), 584 deletions(-) create mode 100644 web/workspace/src/lib/workspace/api/runtime-connection.ts create mode 100644 web/workspace/tests/runtime-connection.test.ts diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 2217e973..d1e93598 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -33,7 +33,7 @@ use axum::extract::rejection::{JsonRejection, QueryRejection}; #[cfg(feature = "ws-server")] use axum::extract::ws::{Message as WsMessage, WebSocket, WebSocketUpgrade}; use axum::extract::{DefaultBodyLimit, Extension, Path, Query, State}; -use axum::http::{Method, Request, StatusCode, header}; +use axum::http::{HeaderMap, Method, Request, StatusCode, header}; use axum::middleware::{self, Next}; use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post}; @@ -66,6 +66,11 @@ use workdir::{ }; const DEFAULT_RUNTIME_HTTP_PORT: u16 = 38800; +pub const RUNTIME_HTTP_PROTOCOL_MIN_VERSION: u32 = 1; +pub const RUNTIME_HTTP_PROTOCOL_MAX_VERSION: u32 = 1; +pub const RUNTIME_HTTP_PROTOCOL_VERSION: u32 = RUNTIME_HTTP_PROTOCOL_MAX_VERSION; +pub const RUNTIME_PING_PERMISSION: &str = "runtime:ping"; +pub const RUNTIME_WORKSPACE_SCOPE_HEADER: &str = "x-yoi-workspace-id"; fn default_runtime_http_bind_addr() -> SocketAddr { SocketAddr::from(([127, 0, 0, 1], DEFAULT_RUNTIME_HTTP_PORT)) @@ -187,6 +192,7 @@ fn runtime_http_router_with_optional_auth( }; let router = Router::new() + .route("/v1/ping", get(get_runtime_ping)) .route("/v1/runtime", get(get_runtime)) .route( "/v1/config-bundles", @@ -340,6 +346,14 @@ enum RuntimeHttpWorkerStatusFilter { Stopped, } +/// `GET /v1/ping` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RuntimeHttpPingResponse { + pub runtime_id: String, + pub protocol_version: u32, +} + /// `GET /v1/workers` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct RuntimeHttpWorkersResponse { @@ -461,6 +475,48 @@ struct RuntimeWorkerEventsWsQuery { type RestResult = Result, RuntimeHttpRestError>; +async fn get_runtime_ping( + State(state): State, + Extension(auth): Extension, + headers: HeaderMap, +) -> RestResult { + let requested_workspace_id = headers + .get(RUNTIME_WORKSPACE_SCOPE_HEADER) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "runtime_ping_workspace_scope_required", + "Runtime ping requires the target Workspace scope", + ) + })?; + if requested_workspace_id != auth.workspace_id { + return Err(RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "runtime_ping_workspace_scope_mismatch", + "Runtime ping Workspace scope does not match the authenticated capability", + )); + } + let runtime_id = state + .auth + .as_ref() + .map(|config| config.runtime_id.trim()) + .filter(|runtime_id| !runtime_id.is_empty()) + .ok_or_else(|| { + RuntimeHttpRestError::new( + StatusCode::SERVICE_UNAVAILABLE, + "runtime_ping_identity_unavailable", + "Runtime ping identity is not configured", + ) + })?; + Ok(Json(RuntimeHttpPingResponse { + runtime_id: runtime_id.to_string(), + protocol_version: RUNTIME_HTTP_PROTOCOL_VERSION, + })) +} + async fn get_runtime( State(state): State, ) -> RestResult { @@ -1767,6 +1823,9 @@ fn auth_workspace_scope( } fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static str> { + if path == "/v1/ping" && *method == Method::GET { + return Some(RUNTIME_PING_PERMISSION); + } if path == "/v1/runtime" { return None; } @@ -2084,6 +2143,82 @@ mod tests { WorkdirPath, WorkdirSessionCapabilities, }; + #[tokio::test] + async fn ping_requires_scoped_permission_and_returns_versioned_identity() { + let runtime = Runtime::new_memory(); + let (auth, signer) = auth_config_and_signer(); + let app = runtime_http_router_with_auth(runtime, None, auth); + let token = + token_for_workspace_with_permissions(&signer, "workspace-a", [RUNTIME_PING_PERMISSION]); + let request = Request::builder() + .method(Method::GET) + .uri("/v1/ping") + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-a") + .body(Body::empty()) + .unwrap(); + + let response = app.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap(), + RuntimeHttpPingResponse { + runtime_id: "runtime-test".to_string(), + protocol_version: RUNTIME_HTTP_PROTOCOL_VERSION, + } + ); + + let wrong_scope_token = + token_for_workspace_with_permissions(&signer, "workspace-a", [RUNTIME_PING_PERMISSION]); + let wrong_scope_request = Request::builder() + .method(Method::GET) + .uri("/v1/ping") + .header(header::AUTHORIZATION, format!("Bearer {wrong_scope_token}")) + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-b") + .body(Body::empty()) + .unwrap(); + assert_eq!( + app.oneshot(wrong_scope_request).await.unwrap().status(), + StatusCode::FORBIDDEN + ); + } + + #[tokio::test] + async fn ping_rejects_token_without_ping_permission() { + let runtime = Runtime::new_memory(); + let (auth, signer) = auth_config_and_signer(); + let app = runtime_http_router_with_auth(runtime, None, auth); + let missing_credential = Request::builder() + .method(Method::GET) + .uri("/v1/ping") + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-a") + .body(Body::empty()) + .unwrap(); + assert_eq!( + app.clone() + .oneshot(missing_credential) + .await + .unwrap() + .status(), + StatusCode::UNAUTHORIZED + ); + + let token = token_for_workspace_with_permissions(&signer, "workspace-a", ["workers:read"]); + let request = Request::builder() + .method(Method::GET) + .uri("/v1/ping") + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-a") + .body(Body::empty()) + .unwrap(); + + assert_eq!( + app.oneshot(request).await.unwrap().status(), + StatusCode::FORBIDDEN + ); + } + #[test] fn attachment_routes_require_worker_input_permission() { assert_eq!( diff --git a/crates/workspace-api/src/lib.rs b/crates/workspace-api/src/lib.rs index b77db77a..8c8dd902 100644 --- a/crates/workspace-api/src/lib.rs +++ b/crates/workspace-api/src/lib.rs @@ -1205,16 +1205,39 @@ pub struct CreateRemoteRuntimeRequest { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum RuntimeConnectionTestStatus { + Compatible, + Failed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum RuntimeConnectionTestFailureKind { + Authentication, + Authorization, + NetworkUnreachable, + Timeout, + TlsOrTransport, + MalformedResponse, + ProtocolVersionMismatch, + RuntimeIdentityMismatch, + Configuration, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] pub struct RuntimeConnectionTestResponse { pub workspace_id: String, pub runtime_id: String, pub checked_at: String, - pub state: String, - pub protocol_version: Option, - pub compatibility_basis: String, - #[serde(default)] - pub capabilities: Vec, - pub health_result: String, + pub status: RuntimeConnectionTestStatus, + pub failure_kind: Option, + pub expected_protocol_version: u32, + pub actual_protocol_version: Option, #[serde(default)] pub diagnostics: Vec, } @@ -2371,6 +2394,9 @@ pub fn catalog_typescript() -> String { RepositoryListResponse::decl(&config), RepositoryDetailResponse::decl(&config), RepositoryLogResponse::decl(&config), + RuntimeConnectionTestStatus::decl(&config), + RuntimeConnectionTestFailureKind::decl(&config), + RuntimeConnectionTestResponse::decl(&config), ] .map(|declaration| format!("export {declaration}")); @@ -3060,6 +3086,27 @@ mod tests { assert!(serde_json::from_value::(stale).is_err()); } + #[test] + fn runtime_connection_test_response_is_closed_and_typed() { + let compatible = serde_json::json!({ + "workspace_id": "workspace-test", + "runtime_id": "runtime-test", + "checked_at": "2026-09-01T12:00:00Z", + "status": "compatible", + "failure_kind": null, + "expected_protocol_version": 1, + "actual_protocol_version": 1, + "diagnostics": [] + }); + let parsed: RuntimeConnectionTestResponse = + serde_json::from_value(compatible.clone()).unwrap(); + assert_eq!(serde_json::to_value(parsed).unwrap(), compatible); + + let mut unknown = compatible; + unknown["capabilities"] = serde_json::json!(["shell"]); + assert!(serde_json::from_value::(unknown).is_err()); + } + #[cfg(feature = "typescript")] #[test] fn generated_catalog_typescript_keeps_public_wrappers_and_nullability() { @@ -3080,6 +3127,9 @@ mod tests { assert!(output.contains( "export type WorkspaceProfileSourceProvenance = \"project_profile_source_tree\"" )); + assert!(output.contains("export type RuntimeConnectionTestResponse =")); + assert!(output.contains("status: RuntimeConnectionTestStatus")); + assert!(output.contains("failure_kind: RuntimeConnectionTestFailureKind | null")); assert!(!output.contains("repository_key: string, display_name")); } diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 7f530a8d..3a9720f7 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -9,7 +9,9 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ + error::Error as _, future::Future, + io::Read as _, path::PathBuf, pin::Pin, sync::{Arc, RwLock}, @@ -38,13 +40,15 @@ use worker_runtime::error::RuntimeError as EmbeddedRuntimeError; use worker_runtime::execution::WorkerExecutionRunState; use worker_runtime::fs_store::FsRuntimeStoreOptions; use worker_runtime::http_server::{ + RUNTIME_PING_PERMISSION, RUNTIME_WORKSPACE_SCOPE_HEADER, RuntimeHttpConfigBundleAvailabilityResponse, RuntimeHttpConfigBundleSyncRequest, - RuntimeHttpErrorResponse, RuntimeHttpRepositoryAccessResponse, RuntimeHttpSummaryResponse, - RuntimeHttpUploadedFileDeleteResponse, RuntimeHttpUploadedFileResponse, - RuntimeHttpWorkerCompletionsRequest, RuntimeHttpWorkerCompletionsResponse, - RuntimeHttpWorkerDeleteResponse, RuntimeHttpWorkerInputResponse, - RuntimeHttpWorkerLifecycleRequest, RuntimeHttpWorkerLifecycleResponse, - RuntimeHttpWorkerResponse, RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse, + RuntimeHttpErrorResponse, RuntimeHttpPingResponse, RuntimeHttpRepositoryAccessResponse, + RuntimeHttpSummaryResponse, RuntimeHttpUploadedFileDeleteResponse, + RuntimeHttpUploadedFileResponse, RuntimeHttpWorkerCompletionsRequest, + RuntimeHttpWorkerCompletionsResponse, RuntimeHttpWorkerDeleteResponse, + RuntimeHttpWorkerInputResponse, RuntimeHttpWorkerLifecycleRequest, + RuntimeHttpWorkerLifecycleResponse, RuntimeHttpWorkerResponse, + RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse, RuntimeHttpWorkingDirectoriesResponse, RuntimeHttpWorkingDirectoryResponse, RuntimeHttpWorkspacePromptProjectionRequest, RuntimeHttpWorkspacePromptProjectionResponse, }; @@ -64,6 +68,7 @@ pub(crate) const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime"; const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host"; const REMOTE_HOST_KIND: &str = "remote-worker-runtime-host"; const MAX_DIAGNOSTICS: usize = 16; +const MAX_RUNTIME_PING_RESPONSE_BYTES: usize = 8 * 1024; const MAX_HOST_SCAN: usize = 256; const MAX_IDENTIFIER_LEN: usize = 120; const ID_DIGEST_HEX_LEN: usize = 16; @@ -760,11 +765,50 @@ fn default_worker_input_kind() -> WorkerInputKind { WorkerInputKind::User } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimePingFailureKind { + Authentication, + Authorization, + NetworkUnreachable, + Timeout, + TlsOrTransport, + MalformedResponse, + Configuration, + Unsupported, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimePingFailure { + pub kind: RuntimePingFailureKind, + pub diagnostic: RuntimeDiagnostic, +} + +impl RuntimePingFailure { + fn new( + kind: RuntimePingFailureKind, + code: impl Into, + message: impl Into, + ) -> Self { + Self { + kind, + diagnostic: diagnostic(code, DiagnosticSeverity::Error, message.into()), + } + } +} + pub trait WorkspaceWorkerRuntime: Send + Sync { fn runtime_id(&self) -> &str; fn runtime_summary(&self, limit: usize) -> RuntimeSummary; + fn ping(&self) -> Result { + Err(RuntimePingFailure::new( + RuntimePingFailureKind::Unsupported, + "runtime_ping_unsupported", + "Runtime connection testing is unavailable for this Runtime provider", + )) + } + fn list_hosts(&self, limit: usize) -> RuntimeList; fn list_workers(&self, limit: usize) -> RuntimeList; @@ -1791,6 +1835,17 @@ impl RuntimeRegistry { }) } + pub fn ping(&self, runtime_id: &str) -> Result { + let runtime = self.runtime(runtime_id).map_err(|_| { + RuntimePingFailure::new( + RuntimePingFailureKind::Configuration, + "runtime_ping_registration_unavailable", + "Registered Runtime binding is unavailable", + ) + })?; + runtime.ping() + } + fn runtimes_snapshot(&self) -> Vec> { self.runtimes .read() @@ -2901,6 +2956,49 @@ pub struct RemoteWorkerRuntime { async_http: AsyncHttpClient, } +fn remote_runtime_ping_transport_failure(error: reqwest::Error) -> RuntimePingFailure { + if error.is_timeout() { + return RuntimePingFailure::new( + RuntimePingFailureKind::Timeout, + "runtime_ping_timeout", + "Runtime ping timed out", + ); + } + let mut source = error.source(); + let mut tls_error = false; + while let Some(current) = source { + let message = current.to_string().to_ascii_lowercase(); + if message.contains("tls") + || message.contains("certificate") + || message.contains("unknownissuer") + || message.contains("handshake") + { + tls_error = true; + break; + } + source = current.source(); + } + if tls_error { + return RuntimePingFailure::new( + RuntimePingFailureKind::TlsOrTransport, + "runtime_ping_tls_failed", + "Runtime TLS connection failed", + ); + } + if error.is_connect() { + return RuntimePingFailure::new( + RuntimePingFailureKind::NetworkUnreachable, + "runtime_ping_network_unreachable", + "Runtime could not be reached", + ); + } + RuntimePingFailure::new( + RuntimePingFailureKind::TlsOrTransport, + "runtime_ping_transport_failed", + "Runtime ping transport failed", + ) +} + fn all_remote_runtime_permissions() -> Vec { [ "workers:list", @@ -3049,14 +3147,18 @@ impl RemoteWorkerRuntime { self.send_json(path, self.http.delete(self.endpoint(path))) } - fn runtime_capability_token(&self, path: &str) -> Option { + fn runtime_capability_token_with_permissions( + &self, + path: &str, + permissions: Vec, + ) -> Option { let auth = self.auth.as_ref()?; let signer = CapabilityTokenSigner::new(&auth.server_id, &auth.server_private_key); let claims = capability_claims( &auth.server_id, &self.runtime_id, &self.workspace_id, - all_remote_runtime_permissions(), + permissions, 300, ) .map_err(|error| { @@ -3079,6 +3181,82 @@ impl RemoteWorkerRuntime { .ok() } + fn runtime_capability_token(&self, path: &str) -> Option { + self.runtime_capability_token_with_permissions(path, all_remote_runtime_permissions()) + } + + fn ping_http(&self) -> Result { + const PATH: &str = "/v1/ping"; + let workspace_id = self.workspace_id.clone(); + let bearer_token = self.bearer_token.clone(); + let capability_token = self.runtime_capability_token_with_permissions( + PATH, + vec![RUNTIME_PING_PERMISSION.to_string()], + ); + let request = self + .http + .get(self.endpoint(PATH)) + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, &workspace_id); + run_blocking_http(move || { + let request = match capability_token.as_deref().or(bearer_token.as_deref()) { + Some(token) => request.header(AUTHORIZATION, format!("Bearer {token}")), + None => request, + }; + let response = request + .send() + .map_err(remote_runtime_ping_transport_failure)?; + match response.status() { + StatusCode::UNAUTHORIZED => { + return Err(RuntimePingFailure::new( + RuntimePingFailureKind::Authentication, + "runtime_ping_authentication_failed", + "Runtime rejected the connection-test credential", + )); + } + StatusCode::FORBIDDEN => { + return Err(RuntimePingFailure::new( + RuntimePingFailureKind::Authorization, + "runtime_ping_authorization_failed", + "Runtime rejected the connection-test scope or permission", + )); + } + status if !status.is_success() => { + return Err(RuntimePingFailure::new( + RuntimePingFailureKind::TlsOrTransport, + "runtime_ping_http_failed", + "Runtime ping returned an unsuccessful HTTP response", + )); + } + _ => {} + } + let mut body = Vec::new(); + response + .take((MAX_RUNTIME_PING_RESPONSE_BYTES + 1) as u64) + .read_to_end(&mut body) + .map_err(|_| { + RuntimePingFailure::new( + RuntimePingFailureKind::TlsOrTransport, + "runtime_ping_response_read_failed", + "Runtime ping response could not be read", + ) + })?; + if body.len() > MAX_RUNTIME_PING_RESPONSE_BYTES { + return Err(RuntimePingFailure::new( + RuntimePingFailureKind::MalformedResponse, + "runtime_ping_response_too_large", + "Runtime ping response exceeded the allowed size", + )); + } + serde_json::from_slice::(&body).map_err(|_| { + RuntimePingFailure::new( + RuntimePingFailureKind::MalformedResponse, + "runtime_ping_malformed_response", + "Runtime ping returned an unrecognized response", + ) + }) + }) + } + fn send_json(&self, path: &str, request: RequestBuilder) -> Result where T: DeserializeOwned + Send + 'static, @@ -3266,6 +3444,10 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { } } + fn ping(&self) -> Result { + self.ping_http() + } + fn list_hosts(&self, limit: usize) -> RuntimeList { if limit == 0 { return RuntimeList::new(Vec::new(), Vec::new()); @@ -4562,7 +4744,7 @@ mod tests { use super::*; use serde_json::json; use std::collections::HashMap; - use std::io::{Read as _, Write as _}; + use std::io::Write as _; use std::net::TcpListener; use std::sync::{Arc, Mutex}; use std::thread; @@ -5707,6 +5889,30 @@ mod tests { assert_eq!(runtime.runtime_id(), "remote:async-init"); } + #[test] + fn remote_runtime_ping_classifies_unreachable_without_endpoint_leak() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + let runtime = RemoteWorkerRuntime::new( + RemoteRuntimeConfig::new( + "remote:unreachable", + "Remote Unreachable", + endpoint.clone(), + Some("secret-token".to_string()), + ), + "workspace-test".to_string(), + "http://127.0.0.1:8787".to_string(), + ) + .unwrap(); + + let failure = runtime.ping().unwrap_err(); + assert_eq!(failure.kind, RuntimePingFailureKind::NetworkUnreachable); + assert_eq!(failure.diagnostic.code, "runtime_ping_network_unreachable"); + assert!(!failure.diagnostic.message.contains(&endpoint)); + assert!(!format!("{failure:?}").contains("secret-token")); + } + #[test] fn remote_runtime_registry_routes_commands_without_browser_secret_leaks() { let worker_id = EmbeddedWorkerId::from_legacy_u64(1).to_string(); diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 3599f26d..7c6fb8e8 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -53,6 +53,10 @@ use workdir::workspace::{ }; use workdir::{CommandHandle, WorkdirSessionHandle}; use worker::feature::builtin::{WorkerObservationSubject, WorkerObservationSubjectRef}; +use worker_runtime::http_server::{ + RUNTIME_HTTP_PROTOCOL_MAX_VERSION, RUNTIME_HTTP_PROTOCOL_MIN_VERSION, + RUNTIME_HTTP_PROTOCOL_VERSION, +}; use worker_runtime::resource::{BackendResourceError, BackendResourceFetchRequest}; use worker_runtime::worker_backend::{ProfileRuntimeWorkerFactory, WorkerRuntimeExecutionBackend}; use workspace_api::{ @@ -72,7 +76,8 @@ use workspace_api::{ PasskeyRegistrationOptionsResponse, ProfileSettingsResponse, PutRepositorySshHostTrustRequest, RepositoryAccessProjection, RepositoryDetailResponse, RepositoryListResponse, RepositoryLogResponse, RepositorySshCredential, RepositorySshHostTrust, RequestActor, - RotateRepositorySshCredentialRequest, RuntimeConnectionTestResponse, RuntimeManagementSummary, + RotateRepositorySshCredentialRequest, RuntimeConnectionTestFailureKind, + RuntimeConnectionTestResponse, RuntimeConnectionTestStatus, RuntimeManagementSummary, TICKET_ORCHESTRATION_PLANS_QUERY_PATH, TICKET_RELATIONS_QUERY_PATH, UpdateWorkspaceMetadataRequest, WhoamiResponse, WorkerLaunchOptionsResponse, WorkerLaunchProfileCandidate, WorkerLaunchRuntimeOption, WorkerLaunchWorkerSummary, @@ -107,14 +112,14 @@ use crate::config_source::ConfigCommitRequest; use crate::hosts::{ ConfigBundleCheckResult, ConfigBundleSyncResult, DiagnosticSeverity, EMBEDDED_RUNTIME_ID, EmbeddedWorkerRuntime, HostSummary, RemoteRuntimeConfig, RemoteWorkerRuntime, - RuntimeDiagnostic, RuntimeRegistry, RuntimeRegistryError, RuntimeRegistryUnregisterResult, - TicketWorkerRole, WorkerCapabilitySummary, WorkerCompletionsRequest, WorkerCompletionsResult, - WorkerControlOperation, WorkerCreateBinding, WorkerImplementationSummary, WorkerInputKind, - WorkerInputRequest, WorkerInputResult, WorkerLifecycleRequest, WorkerLifecycleResult, - WorkerOperationState, WorkerRestoreResult, WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, - WorkerSpawnRequest, WorkerSpawnResult, WorkerSpawnWorkingDirectoryRequest, WorkerSummary, - WorkerTicketAssignmentRequest, WorkerWorkspaceSummary, worker_spawn_create_fingerprint, - workspace_worker_summary, + RuntimeDiagnostic, RuntimePingFailureKind, RuntimeRegistry, RuntimeRegistryError, + RuntimeRegistryUnregisterResult, TicketWorkerRole, WorkerCapabilitySummary, + WorkerCompletionsRequest, WorkerCompletionsResult, WorkerControlOperation, WorkerCreateBinding, + WorkerImplementationSummary, WorkerInputKind, WorkerInputRequest, WorkerInputResult, + WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState, WorkerRestoreResult, + WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult, + WorkerSpawnWorkingDirectoryRequest, WorkerSummary, WorkerTicketAssignmentRequest, + WorkerWorkspaceSummary, worker_spawn_create_fingerprint, workspace_worker_summary, }; use crate::identity::WorkspaceIdentity; use crate::memory_backend::execute_memory_backend_operation_with_authority; @@ -164,11 +169,7 @@ use worker_runtime::catalog::{ WorkingDirectoryRepository, WorkingDirectoryRequest, WorkspaceApiRef, }; use worker_runtime::config_bundle::ConfigBundle; -use worker_runtime::http_server::{ - MAX_WORKER_FILE_UPLOAD_BYTES, RuntimeHttpConfigBundleAvailabilityResponse, - RuntimeHttpConfigBundlesResponse, RuntimeHttpSummaryResponse, RuntimeHttpWorkerResponse, - RuntimeHttpWorkersResponse, -}; +use worker_runtime::http_server::MAX_WORKER_FILE_UPLOAD_BYTES; use worker_runtime::identity::{RuntimeWorkerRef, WorkerId}; const EMBEDDED_WORKER_RUNTIME_ID: &str = "embedded-worker-runtime"; @@ -12518,14 +12519,39 @@ async fn test_runtime_connection( State(api): State, AxumPath(runtime_id): AxumPath, ) -> ApiResult> { + let runtime_id = runtime_id.trim().to_string(); + if runtime_id.is_empty() { + return Err(Error::InvalidRuntimeIdentifier { + kind: "runtime".to_string(), + value: runtime_id, + } + .into()); + } let runtime_config = load_backend_runtimes_config_for_settings(&api)?; - let remote = runtime_config + runtime_config .runtimes .remote .iter() .find(|remote| remote.id == runtime_id) .ok_or_else(|| Error::UnknownRuntime(runtime_id.clone()))?; - Ok(Json(test_remote_runtime_config(&api, remote).await)) + + let checked_at = Utc::now().to_rfc3339(); + let runtime = api.runtime.clone(); + let ping_runtime_id = runtime_id.clone(); + let ping = tokio::task::spawn_blocking(move || runtime.ping(&ping_runtime_id)) + .await + .map_err(|_| Error::RuntimeOperationFailed { + runtime_id: runtime_id.clone(), + code: "runtime_connection_test_unavailable".to_string(), + message: "Runtime connection test could not be completed".to_string(), + })?; + + Ok(Json(runtime_connection_test_response( + api.workspace_id(), + &runtime_id, + checked_at, + ping, + ))) } async fn get_worker_launch_options( @@ -14581,438 +14607,111 @@ fn remote_runtime_config_from_file( }) } -async fn test_remote_runtime_config( - api: &WorkspaceApi, - remote: &RemoteRuntimeConfigFile, -) -> RuntimeConnectionTestResponse { - let checked_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - if remote - .token_ref - .as_deref() - .is_some_and(|value| !value.trim().is_empty()) - { - return RuntimeConnectionTestResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: remote.id.clone(), - checked_at, - state: "rejected".to_string(), - protocol_version: None, - compatibility_basis: "not_checked_token_ref_unsupported".to_string(), - capabilities: Vec::new(), - health_result: "not_checked".to_string(), - diagnostics: vec![settings_diagnostic( - "remote_runtime_token_ref_unsupported", - DiagnosticSeverity::Error, - "Remote Runtime test cannot use token_ref in v0; no token or secret value was exposed to the Browser.", - ) - .into()], - }; - } - - let client = match reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(5)) - .build() - { - Ok(client) => client, - Err(_) => { - return remote_runtime_test_failed( - api, - remote, - checked_at, - "remote_runtime_test_client_unavailable", - "Remote Runtime test client could not be initialized.", - ); - } - }; - - let mut observation = RuntimeCompatibilityObservation::default(); - let summary_url = match remote_probe_url(remote, "/v1/runtime") { - Ok(url) => url, - Err(diagnostic) => { - return remote_runtime_test_failed( - api, - remote, - checked_at, - diagnostic.code, - diagnostic.message, - ); - } - }; - - let summary_payload = - match probe_remote_json(&client, summary_url, "runtime.summary", "Runtime summary").await { - Ok(payload) => payload, - Err(diagnostic) => { - return remote_runtime_test_failed( - api, - remote, - checked_at, - diagnostic.code, - diagnostic.message, - ); - } - }; - let protocol_version = summary_payload - .get("protocol_version") - .and_then(|value| value.as_str()) - .map(ToOwned::to_owned); - let summary = match serde_json::from_value::(summary_payload) { - Ok(summary) => summary, - Err(_) => { - return remote_runtime_test_failed( - api, - remote, - checked_at, - "remote_runtime_malformed_summary", - "Remote Runtime summary responded, but the payload was not recognized.", - ); - } - }; - observation.available( - "runtime.summary", - "Connected: /v1/runtime responded with a recognized worker-runtime summary.", - ); - - let workers_url = match remote_probe_url(remote, "/v1/workers") { - Ok(url) => url, - Err(diagnostic) => { - observation.incompatible("workers.list", diagnostic); - String::new() - } - }; - let workers = if workers_url.is_empty() { - None - } else { - match probe_remote_json(&client, workers_url, "workers.list", "Worker list").await { - Ok(payload) => match serde_json::from_value::(payload) { - Ok(workers) => { - observation.available( - "workers.list", - "Verified: /v1/workers responded with a recognized worker list.", - ); - Some(workers) - } - Err(_) => { - observation.incompatible( - "workers.list", - settings_diagnostic( - "remote_runtime_workers_malformed", - DiagnosticSeverity::Error, - "Remote Runtime worker list responded, but the payload was not recognized.", - ), - ); - None - } - }, - Err(diagnostic) => { - observation.incompatible("workers.list", diagnostic); - None - } - } - }; - - if let Some(worker) = workers.as_ref().and_then(|workers| workers.workers.first()) { - let path = format!( - "/v1/workers/{}", - encode_path_segment(&worker.worker_id.to_string()) - ); - match remote_probe_url(remote, &path) { - Ok(url) => match probe_remote_json(&client, url, "workers.detail", "Worker detail").await { - Ok(payload) => match serde_json::from_value::(payload) { - Ok(_) => observation.available( - "workers.detail", - "Verified: worker detail responded for an existing worker reported by the remote Runtime.", - ), - Err(_) => observation.incompatible( - "workers.detail", - settings_diagnostic( - "remote_runtime_worker_detail_malformed", - DiagnosticSeverity::Error, - "Remote Runtime worker detail responded, but the payload was not recognized.", - ), - ), - }, - Err(diagnostic) => observation.incompatible("workers.detail", diagnostic), - }, - Err(diagnostic) => observation.incompatible("workers.detail", diagnostic), - } - } else { - observation.unknown( - "workers.detail", - "No connection problem found. Worker detail was not checked because the remote Runtime reported no workers during the lightweight probe.", - ); - } - - observation.available( - "workers.events_ws.construct", - "Verified: worker event websocket URL can be constructed from the configured HTTP(S) Runtime endpoint. The lightweight test does not open a websocket stream.", - ); - - let bundles_url = match remote_probe_url(remote, "/v1/config-bundles") { - Ok(url) => url, - Err(diagnostic) => { - observation.incompatible("config_bundles.list", diagnostic); - String::new() - } - }; - let bundles = if bundles_url.is_empty() { - None - } else { - match probe_remote_json( - &client, - bundles_url, - "config_bundles.list", - "Config-bundle list", - ) - .await - { - Ok(payload) => { - match serde_json::from_value::(payload) { - Ok(bundles) => { - observation.available( - "config_bundles.list", - "Verified: /v1/config-bundles responded with a recognized config-bundle list.", - ); - Some(bundles) - } - Err(_) => { - observation.incompatible( - "config_bundles.list", - settings_diagnostic( - "remote_runtime_config_bundles_malformed", - DiagnosticSeverity::Error, - "Remote Runtime config-bundle list responded, but the payload was not recognized.", - ), - ); - None - } - } - } - Err(diagnostic) => { - observation.incompatible("config_bundles.list", diagnostic); - None - } - } - }; - - if let Some(bundle) = bundles.as_ref().and_then(|bundles| bundles.bundles.first()) { - let path = format!( - "/v1/config-bundles/{}/availability?digest={}", - encode_path_segment(&bundle.id), - encode_path_segment(&bundle.digest) - ); - match remote_probe_url(remote, &path) { - Ok(url) => match probe_remote_json( - &client, - url, - "config_bundles.availability", - "Config-bundle availability", - ) - .await - { - Ok(payload) => { - match serde_json::from_value::(payload) - { - Ok(_) => observation.available( - "config_bundles.availability", - "Verified: config-bundle availability was confirmed for an advertised bundle.", - ), - Err(_) => observation.incompatible( - "config_bundles.availability", - settings_diagnostic( - "remote_runtime_config_bundle_availability_malformed", - DiagnosticSeverity::Error, - "Remote Runtime config-bundle availability responded, but the payload was not recognized.", - ), - ), - } - } - Err(diagnostic) => { - observation.incompatible("config_bundles.availability", diagnostic) - } - }, - Err(diagnostic) => observation.incompatible("config_bundles.availability", diagnostic), - } - } else { - observation.unknown( - "config_bundles.availability", - "No connection problem found. Config-bundle availability was not checked because the remote Runtime advertised no bundles during the lightweight probe.", - ); - } - - if summary.runtime.worker_creation_available { - observation.available( - "workers.spawn", - "Verified: /v1/runtime reports worker creation is enabled by a Runtime execution backend. The lightweight test does not create a worker.", - ); - } else { - observation.incompatible( - "workers.spawn", - settings_diagnostic( - "remote_runtime_worker_creation_unavailable", - DiagnosticSeverity::Error, - "Connected to the Runtime, but worker creation is unavailable because this Runtime process has no execution backend attached.", - ), - ); - } - observation.unknown( - "workers.input_dispatch", - "No connection problem found. Worker input dispatch was not checked because this lightweight test does not send model-visible input as a side effect.", - ); - observation.unknown( - "config_bundles.sync", - "No connection problem found. Config-bundle sync was not checked because this lightweight test does not upload bundles as a side effect.", - ); - - RuntimeConnectionTestResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: remote.id.clone(), - checked_at, - state: observation.state().to_string(), - protocol_version, - compatibility_basis: "Connected to /v1/runtime and verified non-side-effecting worker-runtime HTTP endpoints. No incompatible operation was found; warning items below are unproven optional or side-effecting checks, not connection failures.".to_string(), - capabilities: observation.capabilities, - health_result: format!( - "connected=true; runtime_status={:?}; available={}; incompatible={}; warnings={}", - summary.runtime.status, - observation.available_count, - observation.incompatible_count, - observation.unknown_count - ), - diagnostics: observation - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - } -} - -fn remote_runtime_test_failed( - api: &WorkspaceApi, - remote: &RemoteRuntimeConfigFile, +fn runtime_connection_test_response( + workspace_id: &str, + runtime_id: &str, checked_at: String, - code: impl Into, - message: impl Into, + ping: std::result::Result< + worker_runtime::http_server::RuntimeHttpPingResponse, + crate::hosts::RuntimePingFailure, + >, +) -> RuntimeConnectionTestResponse { + match ping { + Ok(ping) if ping.runtime_id != runtime_id => runtime_connection_test_failure( + workspace_id, + runtime_id, + checked_at, + RuntimeConnectionTestFailureKind::RuntimeIdentityMismatch, + None, + RuntimeDiagnostic::new( + "runtime_ping_identity_mismatch", + "error", + "Runtime ping identity does not match the registered Runtime", + ), + ), + Ok(ping) + if !(RUNTIME_HTTP_PROTOCOL_MIN_VERSION..=RUNTIME_HTTP_PROTOCOL_MAX_VERSION) + .contains(&ping.protocol_version) => + { + let code = if ping.protocol_version > RUNTIME_HTTP_PROTOCOL_MAX_VERSION { + "runtime_ping_protocol_newer" + } else { + "runtime_ping_protocol_older" + }; + runtime_connection_test_failure( + workspace_id, + runtime_id, + checked_at, + RuntimeConnectionTestFailureKind::ProtocolVersionMismatch, + Some(ping.protocol_version), + RuntimeDiagnostic::new( + code, + "error", + "Runtime protocol version is incompatible with this Server", + ), + ) + } + Ok(ping) => RuntimeConnectionTestResponse { + workspace_id: workspace_id.to_string(), + runtime_id: runtime_id.to_string(), + checked_at, + status: RuntimeConnectionTestStatus::Compatible, + failure_kind: None, + expected_protocol_version: RUNTIME_HTTP_PROTOCOL_VERSION, + actual_protocol_version: Some(ping.protocol_version), + diagnostics: Vec::new(), + }, + Err(failure) => runtime_connection_test_failure( + workspace_id, + runtime_id, + checked_at, + match failure.kind { + RuntimePingFailureKind::Authentication => { + RuntimeConnectionTestFailureKind::Authentication + } + RuntimePingFailureKind::Authorization => { + RuntimeConnectionTestFailureKind::Authorization + } + RuntimePingFailureKind::NetworkUnreachable => { + RuntimeConnectionTestFailureKind::NetworkUnreachable + } + RuntimePingFailureKind::Timeout => RuntimeConnectionTestFailureKind::Timeout, + RuntimePingFailureKind::TlsOrTransport => { + RuntimeConnectionTestFailureKind::TlsOrTransport + } + RuntimePingFailureKind::MalformedResponse => { + RuntimeConnectionTestFailureKind::MalformedResponse + } + RuntimePingFailureKind::Configuration | RuntimePingFailureKind::Unsupported => { + RuntimeConnectionTestFailureKind::Configuration + } + }, + None, + failure.diagnostic, + ), + } +} + +fn runtime_connection_test_failure( + workspace_id: &str, + runtime_id: &str, + checked_at: String, + failure_kind: RuntimeConnectionTestFailureKind, + actual_protocol_version: Option, + diagnostic: RuntimeDiagnostic, ) -> RuntimeConnectionTestResponse { RuntimeConnectionTestResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: remote.id.clone(), + workspace_id: workspace_id.to_string(), + runtime_id: runtime_id.to_string(), checked_at, - state: "failed".to_string(), - protocol_version: None, - compatibility_basis: "worker-runtime lightweight HTTP compatibility probes".to_string(), - capabilities: Vec::new(), - health_result: "failed".to_string(), - diagnostics: vec![settings_diagnostic(code, DiagnosticSeverity::Error, message).into()], + status: RuntimeConnectionTestStatus::Failed, + failure_kind: Some(failure_kind), + expected_protocol_version: RUNTIME_HTTP_PROTOCOL_VERSION, + actual_protocol_version, + diagnostics: vec![diagnostic.into()], } } -#[derive(Default)] -struct RuntimeCompatibilityObservation { - capabilities: Vec, - diagnostics: Vec, - available_count: usize, - incompatible_count: usize, - unknown_count: usize, -} - -impl RuntimeCompatibilityObservation { - fn available(&mut self, operation: &str, message: impl Into) { - self.available_count += 1; - self.capabilities.push(format!("{operation}:available")); - self.diagnostics.push(settings_diagnostic( - format!("{operation}.available"), - DiagnosticSeverity::Info, - message, - )); - } - - fn unknown(&mut self, operation: &str, message: impl Into) { - self.unknown_count += 1; - self.capabilities.push(format!("{operation}:unknown")); - self.diagnostics.push(settings_diagnostic( - format!("{operation}.unknown"), - DiagnosticSeverity::Warning, - message, - )); - } - - fn incompatible(&mut self, operation: &str, diagnostic: RuntimeDiagnostic) { - self.incompatible_count += 1; - self.capabilities.push(format!("{operation}:incompatible")); - self.diagnostics.push(diagnostic); - } - - fn state(&self) -> &'static str { - if self.incompatible_count > 0 { - "incompatible" - } else { - "compatible" - } - } -} - -fn remote_probe_url( - remote: &RemoteRuntimeConfigFile, - path: &str, -) -> std::result::Result { - let endpoint = remote.endpoint.trim(); - if !(endpoint.starts_with("http://") || endpoint.starts_with("https://")) { - return Err(settings_diagnostic( - "remote_runtime_endpoint_invalid", - DiagnosticSeverity::Error, - "Configured remote Runtime endpoint is not an absolute HTTP(S) URL.", - )); - } - Ok(format!("{}{}", endpoint.trim_end_matches('/'), path)) -} - -async fn probe_remote_json( - client: &reqwest::Client, - url: String, - operation: &'static str, - label: &'static str, -) -> std::result::Result { - let response = client.get(url).send().await.map_err(|error| { - let (code, message) = if error.is_timeout() { - ( - format!("{operation}.timeout"), - format!("Remote Runtime probe for {label} timed out."), - ) - } else if error.is_connect() { - ( - format!("{operation}.connect_failed"), - format!("Remote Runtime probe for {label} could not connect."), - ) - } else { - ( - format!("{operation}.request_failed"), - format!("Remote Runtime probe for {label} failed before a response was received."), - ) - }; - settings_diagnostic(code, DiagnosticSeverity::Error, message) - })?; - - if !response.status().is_success() { - return Err(settings_diagnostic( - format!("{operation}.http_status"), - DiagnosticSeverity::Error, - format!( - "Remote Runtime probe for {label} returned HTTP status {}.", - response.status().as_u16() - ), - )); - } - - response.json::().await.map_err(|_| { - settings_diagnostic( - format!("{operation}.malformed_json"), - DiagnosticSeverity::Error, - format!("Remote Runtime probe for {label} returned an unrecognized JSON payload."), - ) - }) -} - fn worker_launch_options_response(api: &WorkspaceApi) -> ApiResult { let runtimes = api .runtime @@ -24337,6 +24036,71 @@ mod tests { axum::serve(listener, proxy).await } + async fn runtime_ping_stub( + status: StatusCode, + body: serde_json::Value, + ) -> (String, tokio::task::JoinHandle<()>) { + async fn ping( + State((status, body)): State<(StatusCode, serde_json::Value)>, + headers: HeaderMap, + ) -> (StatusCode, Json) { + assert_eq!( + headers + .get(worker_runtime::http_server::RUNTIME_WORKSPACE_SCOPE_HEADER) + .and_then(|value| value.to_str().ok()), + Some(TEST_WORKSPACE_ID) + ); + assert!( + headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.starts_with("Bearer ")) + ); + (status, Json(body)) + } + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ping stub"); + let base_url = format!("http://{}", listener.local_addr().expect("ping stub addr")); + let app = Router::new() + .route("/v1/ping", axum::routing::get(ping)) + .with_state((status, body)); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve ping stub"); + }); + (base_url, server) + } + + async fn test_app_with_remote_runtime( + workspace_root: impl Into, + runtime_id: &str, + endpoint: String, + ) -> Router { + let api = test_api(workspace_root).await; + api.runtime.register_or_replace( + RemoteWorkerRuntime::new( + RemoteRuntimeConfig { + runtime_id: runtime_id.to_string(), + workspace_id: Some(TEST_WORKSPACE_ID.to_string()), + display_name: "Probe Runtime".to_string(), + base_url: endpoint, + bearer_token: Some("test-connection-token".to_string()), + auth: None, + cached_worker_creation_available: true, + cached_os: "linux".to_string(), + cached_arch: "x86_64".to_string(), + cached_status: "active".to_string(), + timeout: std::time::Duration::from_secs(2), + }, + TEST_WORKSPACE_ID.to_string(), + "http://127.0.0.1:1".to_string(), + ) + .unwrap(), + ); + build_inner_router(api) + } + async fn test_app(workspace_root: impl Into) -> Router { build_inner_router(test_api(workspace_root).await) } @@ -25575,124 +25339,157 @@ mod tests { assert_eq!(persisted.runtimes.remote.len(), 1); } - #[tokio::test(flavor = "multi_thread")] - async fn runtime_connection_test_reports_compatible_with_unknown_warnings_without_endpoint_leak() - { - let (runtime, _worker_ref) = runtime_with_worker(); - let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let runtime_addr = runtime_listener.local_addr().unwrap(); - tokio::spawn({ - let runtime = runtime.clone(); - async move { - serve_runtime_http_with_injected_test_auth(runtime, runtime_listener) - .await - .unwrap() - } - }); - - let dir = tempfile::tempdir().unwrap(); - let endpoint = format!("http://{runtime_addr}"); + fn write_test_remote_runtime(root: &std::path::Path, runtime_id: &str, endpoint: String) { BackendRuntimesConfigFile { runtimes: WorkspaceBackendRuntimesConfig { remote: vec![RemoteRuntimeConfigFile { - id: "probe-runtime".to_string(), - endpoint: endpoint.clone(), + id: runtime_id.to_string(), + endpoint, display_name: Some("Probe Runtime".to_string()), token_ref: None, }], }, } - .write_to_path(dir.path().join(".test-config/runtimes.toml")) + .write_to_path(root.join(".test-config/runtimes.toml")) .unwrap(); - let app = test_app(dir.path()).await; + } - let response = post_json( + async fn run_runtime_connection_test( + body: serde_json::Value, + status: StatusCode, + ) -> serde_json::Value { + let (endpoint, _server) = runtime_ping_stub(status, body).await; + let dir = tempfile::tempdir().unwrap(); + write_test_remote_runtime(dir.path(), "probe-runtime", endpoint.clone()); + let app = test_app_with_remote_runtime(dir.path(), "probe-runtime", endpoint).await; + post_json( app, &format!("/api/w/{TEST_WORKSPACE_ID}/runtimes/probe-runtime/connection-tests"), serde_json::json!({}), ) - .await; - assert_eq!(response["state"], "compatible"); - let capabilities = response["capabilities"].as_array().unwrap(); - assert!( - capabilities - .iter() - .any(|value| value == "runtime.summary:available") - ); - assert!( - capabilities - .iter() - .any(|value| value == "workers.list:available") - ); - assert!( - capabilities - .iter() - .any(|value| value == "workers.spawn:available") - ); - assert!( - response["diagnostics"] - .as_array() - .unwrap() - .iter() - .any(|diagnostic| { diagnostic["code"] == "workers.spawn.available" }) - ); - let projected = serde_json::to_string(&response).unwrap(); - assert!(!projected.contains(&endpoint)); - assert!(!projected.contains(&runtime_addr.to_string())); - assert_eq!(response["protocol_version"], serde_json::Value::Null); + .await } #[tokio::test(flavor = "multi_thread")] - async fn runtime_connection_test_marks_missing_execution_backend_incompatible() { - let runtime = - worker_runtime::Runtime::with_options(worker_runtime::RuntimeOptions::default()); - let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let runtime_addr = runtime_listener.local_addr().unwrap(); - tokio::spawn(async move { - serve_runtime_http_with_injected_test_auth(runtime, runtime_listener) - .await - .unwrap() - }); - - let dir = tempfile::tempdir().unwrap(); - let endpoint = format!("http://{runtime_addr}"); - BackendRuntimesConfigFile { - runtimes: WorkspaceBackendRuntimesConfig { - remote: vec![RemoteRuntimeConfigFile { - id: "control-only-runtime".to_string(), - display_name: Some("Control-only Runtime".to_string()), - endpoint, - token_ref: None, - }], - }, - } - .write_to_path(dir.path().join(".test-config/runtimes.toml")) - .unwrap(); - let app = test_app(dir.path()).await; - - let response = post_json( - app, - &format!("/api/w/{TEST_WORKSPACE_ID}/runtimes/control-only-runtime/connection-tests"), - serde_json::json!({}), + async fn runtime_connection_test_reports_exact_compatible_protocol() { + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "probe-runtime", + "protocol_version": RUNTIME_HTTP_PROTOCOL_VERSION, + }), + StatusCode::OK, ) .await; - assert_eq!(response["state"], "incompatible"); - assert!( - response["capabilities"] - .as_array() - .unwrap() - .iter() - .any(|value| { value == "workers.spawn:incompatible" }) + + assert_eq!(response["status"], "compatible"); + assert_eq!(response["failure_kind"], serde_json::Value::Null); + assert_eq!( + response["expected_protocol_version"], + RUNTIME_HTTP_PROTOCOL_VERSION ); - assert!( - response["diagnostics"] - .as_array() - .unwrap() - .iter() - .any(|diagnostic| { - diagnostic["code"] == "remote_runtime_worker_creation_unavailable" - }) + assert_eq!( + response["actual_protocol_version"], + RUNTIME_HTTP_PROTOCOL_VERSION ); + assert_eq!(response["diagnostics"], serde_json::json!([])); + let projected = serde_json::to_string(&response).unwrap(); + assert!(!projected.contains("Bearer")); + assert!(!projected.contains("public_key")); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_rejects_newer_protocol() { + let newer = RUNTIME_HTTP_PROTOCOL_MAX_VERSION + 1; + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "probe-runtime", + "protocol_version": newer, + }), + StatusCode::OK, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "protocol_version_mismatch"); + assert_eq!(response["actual_protocol_version"], newer); + assert_eq!( + response["diagnostics"][0]["code"], + "runtime_ping_protocol_newer" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_rejects_older_protocol() { + let older = RUNTIME_HTTP_PROTOCOL_MIN_VERSION.saturating_sub(1); + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "probe-runtime", + "protocol_version": older, + }), + StatusCode::OK, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "protocol_version_mismatch"); + assert_eq!(response["actual_protocol_version"], older); + assert_eq!( + response["diagnostics"][0]["code"], + "runtime_ping_protocol_older" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_classifies_authentication_failure() { + let response = run_runtime_connection_test( + serde_json::json!({"error": "credential details must not escape"}), + StatusCode::UNAUTHORIZED, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "authentication"); + assert_eq!(response["actual_protocol_version"], serde_json::Value::Null); + let projected = serde_json::to_string(&response).unwrap(); + assert!(!projected.contains("credential details")); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_rejects_malformed_ping_response() { + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "probe-runtime", + "protocol_version": "not-a-number", + "unexpected": true, + }), + StatusCode::OK, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "malformed_response"); + assert_eq!( + response["diagnostics"][0]["code"], + "runtime_ping_malformed_response" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_rejects_runtime_identity_mismatch() { + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "different-runtime", + "protocol_version": RUNTIME_HTTP_PROTOCOL_VERSION, + }), + StatusCode::OK, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "runtime_identity_mismatch"); + assert_eq!(response["actual_protocol_version"], serde_json::Value::Null); + let projected = serde_json::to_string(&response).unwrap(); + assert!(!projected.contains("different-runtime")); } #[tokio::test] diff --git a/web/workspace/deno.json b/web/workspace/deno.json index a8b34b18..cf49316b 100644 --- a/web/workspace/deno.json +++ b/web/workspace/deno.json @@ -6,7 +6,7 @@ "dev": "deno run -A npm:vite@7.2.7 dev", "dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787", "check": "deno run -A npm:@sveltejs/kit@2.49.4 sync && deno run -A npm:svelte-check@4.3.4 --tsconfig ./tsconfig.json", - "test": "deno test --allow-read=src,test,tests --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts src/lib/workspace/auth/model.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/markdown.test.ts test/console/ansi.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-launch.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/ui.test.ts", + "test": "deno test --allow-read=src,test,tests --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts src/lib/workspace/auth/model.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/markdown.test.ts test/console/ansi.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-launch.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/ui.test.ts", "build": "deno run -A npm:vite@7.2.7 build", "preview": "deno run -A npm:vite@7.2.7 preview" }, diff --git a/web/workspace/src/lib/generated/workspace-api.ts b/web/workspace/src/lib/generated/workspace-api.ts index 36c6c324..5d96d4ea 100644 --- a/web/workspace/src/lib/generated/workspace-api.ts +++ b/web/workspace/src/lib/generated/workspace-api.ts @@ -220,3 +220,27 @@ export type RepositoryLogResponse = { items: Array; diagnostics: Array; }; + +export type RuntimeConnectionTestStatus = "compatible" | "failed"; + +export type RuntimeConnectionTestFailureKind = + | "authentication" + | "authorization" + | "network_unreachable" + | "timeout" + | "tls_or_transport" + | "malformed_response" + | "protocol_version_mismatch" + | "runtime_identity_mismatch" + | "configuration"; + +export type RuntimeConnectionTestResponse = { + workspace_id: string; + runtime_id: string; + checked_at: string; + status: RuntimeConnectionTestStatus; + failure_kind: RuntimeConnectionTestFailureKind | null; + expected_protocol_version: number; + actual_protocol_version: number | null; + diagnostics: Array; +}; diff --git a/web/workspace/src/lib/workspace/api/runtime-connection.ts b/web/workspace/src/lib/workspace/api/runtime-connection.ts new file mode 100644 index 00000000..6eb58965 --- /dev/null +++ b/web/workspace/src/lib/workspace/api/runtime-connection.ts @@ -0,0 +1,142 @@ +import type { + Diagnostic, + RuntimeConnectionTestFailureKind, + RuntimeConnectionTestResponse, +} from "$lib/generated/workspace-api"; + +const RESPONSE_KEYS = [ + "workspace_id", + "runtime_id", + "checked_at", + "status", + "failure_kind", + "expected_protocol_version", + "actual_protocol_version", + "diagnostics", +] as const; +const DIAGNOSTIC_KEYS = ["code", "severity", "message"] as const; +const FAILURE_KINDS = new Set([ + "authentication", + "authorization", + "network_unreachable", + "timeout", + "tls_or_transport", + "malformed_response", + "protocol_version_mismatch", + "runtime_identity_mismatch", + "configuration", +]); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function hasExactKeys( + record: Record, + expected: readonly string[], +): boolean { + const actual = Object.keys(record).sort(); + const wanted = [...expected].sort(); + return actual.length === wanted.length && + actual.every((key, index) => key === wanted[index]); +} + +function isBoundedString(value: unknown, max = 1024): value is string { + return typeof value === "string" && value.length > 0 && value.length <= max; +} + +function isProtocolVersion(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + +function parseDiagnostic(value: unknown): Diagnostic | null { + if (!isRecord(value) || !hasExactKeys(value, DIAGNOSTIC_KEYS)) return null; + if (!isBoundedString(value.code, 128) || !isBoundedString(value.message)) { + return null; + } + if ( + value.severity !== "info" && value.severity !== "warning" && + value.severity !== "error" + ) { + return null; + } + return { + code: value.code, + severity: value.severity, + message: value.message, + }; +} + +export function parseRuntimeConnectionTestResponse( + value: unknown, +): RuntimeConnectionTestResponse | null { + if (!isRecord(value) || !hasExactKeys(value, RESPONSE_KEYS)) return null; + if ( + !isBoundedString(value.workspace_id, 256) || + !isBoundedString(value.runtime_id, 256) || + !isBoundedString(value.checked_at, 128) || + Number.isNaN(Date.parse(value.checked_at)) || + (value.status !== "compatible" && value.status !== "failed") || + !isProtocolVersion(value.expected_protocol_version) || + (value.actual_protocol_version !== null && + !isProtocolVersion(value.actual_protocol_version)) || + !Array.isArray(value.diagnostics) || + value.diagnostics.length > 16 + ) { + return null; + } + const failureKind = value.failure_kind; + if ( + failureKind !== null && + !FAILURE_KINDS.has(failureKind as RuntimeConnectionTestFailureKind) + ) { + return null; + } + const diagnostics = value.diagnostics.map(parseDiagnostic); + if (diagnostics.some((diagnostic) => diagnostic === null)) return null; + if ( + (value.status === "compatible" && + (failureKind !== null || + value.actual_protocol_version !== value.expected_protocol_version || + diagnostics.length !== 0)) || + (value.status === "failed" && failureKind === null) + ) { + return null; + } + return { + workspace_id: value.workspace_id, + runtime_id: value.runtime_id, + checked_at: value.checked_at, + status: value.status, + failure_kind: failureKind as RuntimeConnectionTestFailureKind | null, + expected_protocol_version: value.expected_protocol_version, + actual_protocol_version: value.actual_protocol_version, + diagnostics: diagnostics as Diagnostic[], + }; +} + +export async function testRuntimeConnection( + workspaceId: string, + runtimeId: string, + fetchImpl: typeof fetch = fetch, +): Promise { + const response = await fetchImpl( + `/api/w/${encodeURIComponent(workspaceId)}/runtimes/${ + encodeURIComponent(runtimeId) + }/connection-tests`, + { method: "POST" }, + ); + if (!response.ok) { + throw new Error(`Connection test failed (${response.status})`); + } + const parsed = parseRuntimeConnectionTestResponse(await response.json()); + if (!parsed) { + throw new Error("Connection test returned an invalid response"); + } + if (parsed.workspace_id !== workspaceId || parsed.runtime_id !== runtimeId) { + throw new Error( + "Connection test response did not match the selected Runtime", + ); + } + return parsed; +} diff --git a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts index d9e8650c..e7effcb7 100644 --- a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts +++ b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts @@ -655,6 +655,9 @@ Deno.test("workspace Runtime inventory lives under Settings admin routes", async import.meta.url, ), ); + const runtimeConnectionApi = await Deno.readTextFile( + new URL("../api/runtime-connection.ts", import.meta.url), + ); const workdirsPage = await Deno.readTextFile( new URL( "./../../../routes/w/[workspaceId]/settings/runtimes/[runtimeId]/workdirs/+page.svelte", @@ -678,9 +681,11 @@ Deno.test("workspace Runtime inventory lives under Settings admin routes", async runtimesPage.includes("Add remote Runtime") && runtimesPage.includes("Open workdirs") && runtimesPage.includes("settings-runtime-table") && - runtimesPage.includes( - "/runtimes/${encodeURIComponent(runtime.runtime_id)}/connection-tests", - ) && + runtimesPage.includes("testRuntimeConnection") && + runtimesPage.includes("data.workspaceId") && + runtimesPage.includes("runtime.runtime_id") && + runtimeConnectionApi.includes("/runtimes/${") && + runtimeConnectionApi.includes("}/connection-tests") && runtimesPage.includes( "/settings/runtimes/${encodeURIComponent(runtime.runtime_id)}/workdirs", ), diff --git a/web/workspace/src/lib/workspace/styles/settings.css b/web/workspace/src/lib/workspace/styles/settings.css index 0a1d1b82..ded7886b 100644 --- a/web/workspace/src/lib/workspace/styles/settings.css +++ b/web/workspace/src/lib/workspace/styles/settings.css @@ -339,6 +339,9 @@ background: rgba(255, 255, 255, 0.04); padding: 0.75rem; } + .settings-test-result.failed { + border-inline-start: 3px solid var(--danger); + } .settings-page { display: grid; gap: var(--space-5); diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/+page.svelte index f1e3e25d..4e9380b0 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/+page.svelte @@ -1,20 +1,11 @@ + + + {data.runtimeDetail?.runtime.label ?? data.runtimeId} · Runtime Settings · Yoi Workspace + + + +
+
+
+ Runtimes +

{data.runtimeDetail?.runtime.label ?? data.runtimeId}

+

{data.runtimeId}

+
+ + Workdirs + +
+ + {#if data.runtimeDetailError} +

{data.runtimeDetailError}

+ {:else if !data.runtimeDetail} +

Loading Runtime…

+ {:else} + {@const detail = data.runtimeDetail} + {@const runtime = detail.runtime} + {@const trust = detail.trust_key} + {@const currentAction = trustAction(trust.status)} + +
+

Identity and binding

+
+
Runtime ID
{runtime.runtime_id}
+
Kind
{runtime.kind}
+
Endpoint
{detail.endpoint ?? 'Not configured'}
+
Status
{runtime.status}
+
Binding status
{trust.status}
+
Fingerprint
{trust.fingerprint ?? '—'}
+
Revision
{trust.revision?.toString() ?? '—'}
+
Created
{formatTimestamp(trust.created_at)}
+
Updated
{formatTimestamp(trust.updated_at)}
+
Revoked
{formatTimestamp(trust.revoked_at)}
+
+ {#if runtime.diagnostics.length > 0} +
    + {#each runtime.diagnostics as diagnostic} +
  • + {diagnostic.code} + {diagnostic.message} +
    • + {/each} +
+ {/if} +
+ + {#if data.workspace.permissions.manage_runtimes} +
+

Workspace trust

+ + {#if trust.public_key} +
+ + +
+ {#if revealPublicKey} + 
{trust.public_key}
+ {/if} + {:else if trust.status !== 'unconfigured'} +

The public key was not included in this authorized response.

+ {/if} + +
+ + + + {#if currentAction !== 'create'} + + + Enter {trust.fingerprint ?? 'the current fingerprint'} exactly. + {/if} + + {#if fieldError} +

{fieldError}

+ {/if} +
+ +
+
+ +
+
+ Revoke Workspace trust +

Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.

+
+ +
+ + {#if requestError} + + {/if} + {#if successMessage} +

{successMessage}

+ {/if} +
+ {/if} + +
+

Recent trust audit

+ {#if detail.recent_audit.length === 0} +

No trust changes are recorded.

+ {:else} +
+ + + + + + {#each detail.recent_audit as entry} + + + + + + + + {/each} + +
ActionRevisionFingerprintActorTime
{entry.action}{entry.revision.toString()}{entry.new_fingerprint ?? entry.old_fingerprint ?? '—'}{entry.actor_account_id}{formatTimestamp(entry.at)}
+
+ {/if} +
+ {/if} +
diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.ts b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.ts new file mode 100644 index 00000000..100e82f5 --- /dev/null +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.ts @@ -0,0 +1,31 @@ +import { loadJson, workspaceApiPath } from "$lib/workspace/api/http"; +import { parseWorkspaceRuntimeDetail } from "$lib/workspace/api/runtime-management"; +import type { PageLoad } from "./$types"; + +export const load: PageLoad = async ({ fetch, params }) => { + const detail = await loadJson( + fetch, + workspaceApiPath( + params.workspaceId, + `/runtimes/${encodeURIComponent(params.runtimeId)}`, + ), + undefined, + (value) => { + const response = parseWorkspaceRuntimeDetail(value); + if ( + response.workspace_id !== params.workspaceId || + response.runtime.runtime_id !== params.runtimeId + ) { + throw new Error("Runtime detail did not match the route"); + } + return response; + }, + ); + + return { + workspaceId: params.workspaceId, + runtimeId: params.runtimeId, + runtimeDetail: detail.data, + runtimeDetailError: detail.error, + }; +}; diff --git a/web/workspace/tests/runtime-management-source.test.ts b/web/workspace/tests/runtime-management-source.test.ts new file mode 100644 index 00000000..f455191e --- /dev/null +++ b/web/workspace/tests/runtime-management-source.test.ts @@ -0,0 +1,133 @@ +declare const Deno: { + test(name: string, fn: () => void | Promise): void; + readTextFile(path: URL): Promise; +}; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +Deno.test("Runtime Settings routes validate unknown JSON through the shared Runtime parser", async () => { + const [listLoader, detailLoader] = await Promise.all([ + Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/+page.ts", + import.meta.url, + ), + ), + Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.ts", + import.meta.url, + ), + ), + ]); + + assert( + listLoader.includes("parseWorkspaceRuntimeList(value)"), + "Runtime list loader should validate unknown JSON", + ); + assert( + detailLoader.includes("parseWorkspaceRuntimeDetail(value)"), + "Runtime detail loader should validate unknown JSON", + ); + for (const source of [listLoader, detailLoader]) { + assert( + !source.includes("loadJson<"), + "Runtime loaders must not cast response JSON to a handwritten DTO", + ); + } +}); + +Deno.test("Runtime list links to canonical detail and has no inline delete action", async () => { + const page = await Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/+page.svelte", + import.meta.url, + ), + ); + + assert( + page.includes( + "/settings/runtimes/${encodeURIComponent(runtime.runtime_id)}", + ), + "Runtime name should link to canonical detail", + ); + assert( + page.includes("testRuntime(runtime)"), + "connection Test should remain available", + ); + assert(page.includes("Add Runtime"), "Add Runtime should remain available"); + assert( + page.includes("data.workspace.permissions.manage_runtimes"), + "Add Runtime should be hidden from non-owners", + ); + assert( + !page.includes("deleteRuntime"), + "inline Runtime delete logic must be removed", + ); + assert( + !page.includes(">Delete"), + "inline Runtime delete control must be removed", + ); +}); + +Deno.test("Runtime detail keeps trust controls owner-only and conflict-safe", async () => { + const page = await Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte", + import.meta.url, + ), + ); + + const ownerGate = page.indexOf("data.workspace.permissions.manage_runtimes"); + const reveal = page.indexOf("Reveal public key"); + const mutation = page.indexOf('id="runtime-public-key-input"'); + assert(ownerGate >= 0, "Runtime trust controls should use manage_runtimes"); + assert( + ownerGate < reveal && ownerGate < mutation, + "owner gate should wrap key controls", + ); + + for ( + const token of [ + "Create Workspace trust", + "Replace trusted key", + "Reactivate with this key", + "Confirm current fingerprint", + "Revoke Workspace trust", + "Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.", + "RuntimeTrustConflictError", + "await reloadAuthority()", + "busyAction !== null", + "Workdirs", + "Recent trust audit", + ] + ) { + assert(page.includes(token), `Runtime detail should include ${token}`); + } +}); + +Deno.test("Runtime detail uses flat sections instead of nested cards", async () => { + const [page, css] = await Promise.all([ + Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte", + import.meta.url, + ), + ), + Deno.readTextFile( + new URL("../src/lib/workspace/styles/settings.css", import.meta.url), + ), + ]); + + assert( + !page.includes('class="card"') && !page.includes("settings-card"), + "Runtime detail should not add card nesting", + ); + assert( + css.includes(".runtime-detail-section") && + css.includes("border-top: 1px solid var(--line)"), + "Runtime detail hierarchy should use flat section separators", + ); +}); diff --git a/web/workspace/tests/runtime-management.test.ts b/web/workspace/tests/runtime-management.test.ts new file mode 100644 index 00000000..cc938db9 --- /dev/null +++ b/web/workspace/tests/runtime-management.test.ts @@ -0,0 +1,227 @@ +declare const Deno: { + test(name: string, fn: () => void | Promise): void; +}; + +import { + parseRuntimeTrustConflict, + parseWorkspaceRuntimeDetail, + parseWorkspaceRuntimeList, + putRuntimeTrustKey, + RuntimeTrustConflictError, +} from "../src/lib/workspace/api/runtime-management.ts"; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function assertThrows(operation: () => unknown, expected: string): void { + try { + operation(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.includes(expected)) return; + throw new Error( + `expected error containing ${expected}, received ${message}`, + ); + } + throw new Error("expected operation to throw"); +} + +function runtime() { + return { + management: { + built_in: false, + config_managed: true, + removable: false, + endpoint_configured: true, + token_ref_configured: false, + }, + runtime_id: "arcadia", + label: "Arcadia", + kind: "remote", + status: "started", + source: { + kind: "remote_http", + status: "active", + identity_authority: "server_runtime_configuration", + note: "Configured by Server authority", + }, + host_ids: ["host-a"], + worker_creation_available: true, + os: "linux", + arch: "x86_64", + diagnostics: [], + }; +} + +function detail() { + return { + workspace_id: "workspace-a", + runtime: runtime(), + endpoint: "https://runtime.example.test", + trust_key: { + status: "active", + public_key: "ssh-ed25519 AAAA-test", + fingerprint: "SHA256:current", + revision: 3, + created_at: "2026-09-01T12:00:00Z", + updated_at: "2026-09-01T13:00:00Z", + revoked_at: null, + }, + recent_audit: [{ + action: "created", + actor_account_id: "account-a", + old_fingerprint: null, + new_fingerprint: "SHA256:current", + revision: 3, + at: "2026-09-01T13:00:00Z", + }], + }; +} + +Deno.test("Runtime list and detail parsers return generated Runtime DTO shapes", () => { + const list = parseWorkspaceRuntimeList({ + workspace_id: "workspace-a", + limit: 200, + items: [runtime()], + source: "workspace-control-plane", + diagnostics: [], + }); + assert( + list.items[0]?.runtime_id === "arcadia", + "Runtime ID was not preserved", + ); + + const parsed = parseWorkspaceRuntimeDetail(detail()); + assert( + parsed.trust_key.revision === 3, + "revision was not preserved as a safe integer", + ); + assert( + parsed.recent_audit[0]?.revision === 3, + "audit revision was not normalized", + ); +}); + +Deno.test("Runtime validators reject unknown object keys and enum variants", () => { + assertThrows( + () => parseWorkspaceRuntimeDetail({ ...detail(), head_tree: "stale" }), + "head_tree is not part", + ); + + const futureSource = structuredClone(detail()); + futureSource.runtime.source.kind = "future_transport"; + assertThrows( + () => parseWorkspaceRuntimeDetail(futureSource), + "contains an unknown enum value", + ); + + assertThrows( + () => + parseRuntimeTrustConflict({ + error: "future_conflict", + message: "conflict", + current_revision: 4, + current_fingerprint: "SHA256:new", + }), + "contains an unknown enum value", + ); +}); + +Deno.test("Runtime validators reject unsafe revisions and bounded collection overflow", () => { + const unsafeRevision = structuredClone(detail()); + unsafeRevision.trust_key.revision = Number.MAX_SAFE_INTEGER + 1; + assertThrows( + () => parseWorkspaceRuntimeDetail(unsafeRevision), + "must be a safe integer", + ); + + const tooMuchAudit = structuredClone(detail()); + tooMuchAudit.recent_audit = Array.from( + { length: 21 }, + () => structuredClone(detail().recent_audit[0]), + ); + assertThrows( + () => parseWorkspaceRuntimeDetail(tooMuchAudit), + "must contain at most 20 items", + ); + + const tooManyItems = Array.from({ length: 201 }, () => runtime()); + assertThrows( + () => + parseWorkspaceRuntimeList({ + workspace_id: "workspace-a", + limit: 200, + items: tooManyItems, + source: "workspace-control-plane", + diagnostics: [], + }), + "must contain at most 200 items", + ); +}); + +Deno.test("Runtime detail rejects unbounded strings and incoherent trust state", () => { + const largeKey = structuredClone(detail()); + largeKey.trust_key.public_key = "x".repeat(16 * 1024 + 1); + assertThrows( + () => parseWorkspaceRuntimeDetail(largeKey), + "must be at most 16384 UTF-8 bytes", + ); + + const activeWithoutFingerprint = structuredClone(detail()) as Record< + string, + unknown + >; + (activeWithoutFingerprint.trust_key as Record).fingerprint = + null; + assertThrows( + () => parseWorkspaceRuntimeDetail(activeWithoutFingerprint), + "must include fingerprint", + ); +}); + +Deno.test("typed trust conflict is validated and preserves authoritative revision", async () => { + let sentBody: unknown = null; + const fetchImpl = ((_: RequestInfo | URL, init?: RequestInit) => { + sentBody = JSON.parse(String(init?.body)) as unknown; + return Promise.resolve( + new Response( + JSON.stringify({ + error: "stale_revision", + message: "Runtime trust changed", + current_revision: 4, + current_fingerprint: "SHA256:new", + }), + { status: 409, headers: { "content-type": "application/json" } }, + ), + ); + }) as typeof fetch; + + try { + await putRuntimeTrustKey( + "workspace-a", + "arcadia", + { public_key: "ssh-ed25519 AAAA-new", expected_revision: 3 }, + fetchImpl, + ); + throw new Error("expected mutation to reject"); + } catch (error) { + assert( + error instanceof RuntimeTrustConflictError, + "expected typed conflict", + ); + assert( + error.conflict.current_revision === 4, + "authoritative revision was lost", + ); + } + + assert( + JSON.stringify(sentBody) === + JSON.stringify({ + public_key: "ssh-ed25519 AAAA-new", + expected_revision: 3, + }), + "request should serialize the generated bigint revision as a safe JSON integer", + ); +}); From 2cd57a32b2f091559377ca1ae4ef6067db98ee68 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 05:17:27 +0900 Subject: [PATCH 24/39] fix: align runtime trust schema and built-in controls --- crates/workspace-server/src/latest_schema.sql | 2 +- crates/workspace-server/src/store.rs | 11 +++++++++++ .../settings/runtimes/[runtimeId]/+page.svelte | 2 +- web/workspace/tests/runtime-management-source.test.ts | 4 ++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index e9f86a18..720ff8ea 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -440,7 +440,7 @@ CREATE TABLE workspace_runtime_bindings ( base_url TEXT NOT NULL, public_key TEXT NOT NULL, public_key_fingerprint TEXT NOT NULL, - binding_revision INTEGER NOT NULL CHECK (binding_revision > 0), + binding_revision INTEGER NOT NULL DEFAULT 1 CHECK (binding_revision > 0), created_at TEXT NOT NULL, updated_at TEXT NOT NULL, revoked_at TEXT, diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 44de5aa5..0c7039b4 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -6470,6 +6470,17 @@ fn verify_workspace_runtime_binding_schema(conn: &Connection) -> Result<()> { "workspace_runtime_bindings schema does not match schema-52".to_string(), )); } + let revision_default = conn.query_row( + "SELECT dflt_value FROM pragma_table_info('workspace_runtime_bindings') WHERE name = 'binding_revision'", + [], + |row| row.get::<_, Option>(0), + )?; + if revision_default.as_deref() != Some("1") { + return Err(Error::Store( + "workspace_runtime_bindings binding_revision default does not match schema-52" + .to_string(), + )); + } let sql = conn.query_row( "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'workspace_runtime_bindings'", [], diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte index c7ad9c21..e05b9663 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte @@ -219,7 +219,7 @@ {/if} - {#if data.workspace.permissions.manage_runtimes} + {#if data.workspace.permissions.manage_runtimes && !runtime.management.built_in}

Workspace trust

diff --git a/web/workspace/tests/runtime-management-source.test.ts b/web/workspace/tests/runtime-management-source.test.ts index f455191e..034b4626 100644 --- a/web/workspace/tests/runtime-management-source.test.ts +++ b/web/workspace/tests/runtime-management-source.test.ts @@ -84,6 +84,10 @@ Deno.test("Runtime detail keeps trust controls owner-only and conflict-safe", as const reveal = page.indexOf("Reveal public key"); const mutation = page.indexOf('id="runtime-public-key-input"'); assert(ownerGate >= 0, "Runtime trust controls should use manage_runtimes"); + assert( + page.includes("!runtime.management.built_in"), + "Runtime trust controls should be hidden for the built-in Runtime", + ); assert( ownerGate < reveal && ownerGate < mutation, "owner gate should wrap key controls", From 5686bbc9fd49f561fad0fb2dd67207b80a7171ba Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 05:25:03 +0900 Subject: [PATCH 25/39] fix: preview runtime trust rotation fingerprint --- .../lib/workspace/api/runtime-management.ts | 35 ++++++++++++++ .../src/lib/workspace/styles/settings.css | 24 ++++++++++ .../runtimes/[runtimeId]/+page.svelte | 48 ++++++++++++++++++- .../tests/runtime-management-source.test.ts | 8 ++++ .../tests/runtime-management.test.ts | 12 +++++ 5 files changed, 126 insertions(+), 1 deletion(-) diff --git a/web/workspace/src/lib/workspace/api/runtime-management.ts b/web/workspace/src/lib/workspace/api/runtime-management.ts index 944a5238..cefb9785 100644 --- a/web/workspace/src/lib/workspace/api/runtime-management.ts +++ b/web/workspace/src/lib/workspace/api/runtime-management.ts @@ -677,6 +677,41 @@ async function finishMutation( return detail; } +export async function previewRuntimePublicKeyFingerprint( + publicKey: string, +): Promise { + const normalized = publicKey.trim(); + const prefix = "yoi-ed25519-pub:v1:"; + if (!normalized.startsWith(prefix)) { + throw new RuntimeTrustRequestError( + `Public key must start with ${prefix}`, + ); + } + const encoded = normalized.slice(prefix.length); + if (!/^[A-Za-z0-9_-]+$/.test(encoded)) { + throw new RuntimeTrustRequestError("Public key encoding is invalid"); + } + const padded = encoded.replaceAll("-", "+").replaceAll("_", "/") + + "=".repeat((4 - (encoded.length % 4)) % 4); + let decoded: string; + try { + decoded = atob(padded); + } catch { + throw new RuntimeTrustRequestError("Public key encoding is invalid"); + } + if (decoded.length !== 32) { + throw new RuntimeTrustRequestError("Public key must contain 32 bytes"); + } + const bytes = Uint8Array.from( + decoded, + (character) => character.charCodeAt(0), + ); + const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)); + const hex = Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")) + .join(""); + return `sha256:${hex}`; +} + export async function putRuntimeTrustKey( workspaceId: string, runtimeId: string, diff --git a/web/workspace/src/lib/workspace/styles/settings.css b/web/workspace/src/lib/workspace/styles/settings.css index e6f363b3..f085acea 100644 --- a/web/workspace/src/lib/workspace/styles/settings.css +++ b/web/workspace/src/lib/workspace/styles/settings.css @@ -470,6 +470,30 @@ color: var(--text-muted); } + .runtime-trust-comparison { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--space-3); + margin: 0; + padding: var(--space-3) 0; + border-block: 1px solid var(--line); + } + + .runtime-trust-comparison div { + min-width: 0; + } + + .runtime-trust-comparison dt { + color: var(--text-muted); + font-size: 0.72rem; + font-weight: 700; + } + + .runtime-trust-comparison dd { + margin: var(--space-1) 0 0; + overflow-wrap: anywhere; + } + .runtime-trust-form .field-error, .runtime-detail-page .section-state.error { color: var(--danger); diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte index e05b9663..13a36d5e 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte @@ -6,6 +6,7 @@ RuntimeTrustKeyStatus, } from '$lib/generated/workspace-api'; import { + previewRuntimePublicKeyFingerprint, putRuntimeTrustKey, revokeRuntimeTrustKey, RuntimeTrustConflictError, @@ -23,6 +24,29 @@ let fieldError = $state(null); let requestError = $state(null); let successMessage = $state(null); + let replacementFingerprint = $state(null); + let replacementFingerprintError = $state(null); + let fingerprintGeneration = 0; + + $effect(() => { + const key = publicKey.trim(); + const generation = ++fingerprintGeneration; + replacementFingerprint = null; + replacementFingerprintError = null; + if (!key) return; + void previewRuntimePublicKeyFingerprint(key).then( + (fingerprint) => { + if (generation === fingerprintGeneration) replacementFingerprint = fingerprint; + }, + (error) => { + if (generation === fingerprintGeneration) { + replacementFingerprintError = error instanceof Error + ? error.message + : String(error); + } + }, + ); + }); function trustAction(status: RuntimeTrustKeyStatus): TrustAction { if (status === 'unconfigured') return 'create'; @@ -69,6 +93,14 @@ fieldError = 'Public key must be at most 16 KiB of UTF-8 text.'; return; } + if (replacementFingerprintError) { + fieldError = replacementFingerprintError; + return; + } + if (!replacementFingerprint) { + fieldError = 'Wait for the replacement fingerprint preview before saving.'; + return; + } const trust = data.runtimeDetail.trust_key; const action = trustAction(trust.status); @@ -249,9 +281,23 @@ spellcheck="false" aria-describedby={fieldError ? 'runtime-public-key-error' : undefined} aria-invalid={fieldError ? 'true' : undefined} - placeholder="ssh-ed25519 …" + placeholder="yoi-ed25519-pub:v1:…" > +
+
+
Current fingerprint
+
{trust.fingerprint ?? 'Not configured'}
+
+
+
Replacement fingerprint
+
{replacementFingerprint ?? 'Enter a valid public key'}
+
+
+ {#if replacementFingerprintError} +

{replacementFingerprintError}

+ {/if} + {#if currentAction !== 'create'} = 0, "Runtime trust controls should use manage_runtimes"); + assert( + page.includes("Current fingerprint"), + "current fingerprint must be explicit", + ); + assert( + page.includes("Replacement fingerprint"), + "replacement fingerprint must be previewed before confirmation", + ); assert( page.includes("!runtime.management.built_in"), "Runtime trust controls should be hidden for the built-in Runtime", diff --git a/web/workspace/tests/runtime-management.test.ts b/web/workspace/tests/runtime-management.test.ts index cc938db9..daf80b71 100644 --- a/web/workspace/tests/runtime-management.test.ts +++ b/web/workspace/tests/runtime-management.test.ts @@ -6,6 +6,7 @@ import { parseRuntimeTrustConflict, parseWorkspaceRuntimeDetail, parseWorkspaceRuntimeList, + previewRuntimePublicKeyFingerprint, putRuntimeTrustKey, RuntimeTrustConflictError, } from "../src/lib/workspace/api/runtime-management.ts"; @@ -180,6 +181,17 @@ Deno.test("Runtime detail rejects unbounded strings and incoherent trust state", ); }); +Deno.test("Runtime public key preview matches the Server fingerprint contract", async () => { + const fingerprint = await previewRuntimePublicKeyFingerprint( + "yoi-ed25519-pub:v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ); + assert( + fingerprint === + "sha256:66687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f2925", + "fingerprint preview drifted from the Server SHA-256 contract", + ); +}); + Deno.test("typed trust conflict is validated and preserves authoritative revision", async () => { let sentBody: unknown = null; const fetchImpl = ((_: RequestInfo | URL, init?: RequestInit) => { From 5fd2ccf0840c7e6f723f8edc884416f177e671df Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 05:50:29 +0900 Subject: [PATCH 26/39] fix: gate runtime key reveal and revoke confirmation --- crates/client/src/workspace_product.rs | 16 ++- crates/workspace-api/src/lib.rs | 18 +++- crates/workspace-server/src/server.rs | 86 +++++++++++----- .../src/lib/generated/workspace-api.ts | 3 +- .../lib/workspace/api/runtime-management.ts | 56 ++++++++--- .../src/lib/workspace/styles/settings.css | 12 ++- .../runtimes/[runtimeId]/+page.svelte | 97 +++++++++++++++---- .../tests/runtime-management-source.test.ts | 2 + .../tests/runtime-management.test.ts | 34 ++++++- 9 files changed, 253 insertions(+), 71 deletions(-) diff --git a/crates/client/src/workspace_product.rs b/crates/client/src/workspace_product.rs index bdc97f1f..99d72538 100644 --- a/crates/client/src/workspace_product.rs +++ b/crates/client/src/workspace_product.rs @@ -13,9 +13,9 @@ use workspace_api::{ CreateWorkspaceWorkerRequest, ListResponse, MemoryDocumentResponse, MemoryStagingListResponse, ObjectiveCreateRequest, ObjectiveDetail, ObjectiveEditRequest, ObjectiveLinkTicketRequest, ObjectiveStateRequest, ObjectiveSummary, PutRuntimeTrustKeyRequest, - RevokeRuntimeTrustKeyRequest, TICKET_ORCHESTRATION_PLANS_QUERY_PATH, - TICKET_RELATIONS_QUERY_PATH, WorkerLaunchOptionsResponse, WorkspaceRuntimeDetail, - WorkspaceRuntimeResource, + RevokeRuntimeTrustKeyRequest, RuntimeTrustKeyRevealResponse, + TICKET_ORCHESTRATION_PLANS_QUERY_PATH, TICKET_RELATIONS_QUERY_PATH, + WorkerLaunchOptionsResponse, WorkspaceRuntimeDetail, WorkspaceRuntimeResource, }; use crate::{BackendApiClient, BackendWorkspaceClientError}; @@ -256,6 +256,16 @@ impl BackendWorkspaceProductClient { self.get_json(&format!("/runtimes/{}", encode_path_segment(runtime_id))) } + pub fn reveal_runtime_trust_key( + &self, + runtime_id: &str, + ) -> Result { + self.get_json(&format!( + "/runtimes/{}/trust-key", + encode_path_segment(runtime_id) + )) + } + pub fn put_runtime_trust_key( &self, runtime_id: &str, diff --git a/crates/workspace-api/src/lib.rs b/crates/workspace-api/src/lib.rs index 6cc62968..a2cf8f87 100644 --- a/crates/workspace-api/src/lib.rs +++ b/crates/workspace-api/src/lib.rs @@ -1220,8 +1220,6 @@ pub enum RuntimeTrustKeyStatus { pub struct RuntimeTrustKeyState { pub status: RuntimeTrustKeyStatus, #[serde(default, skip_serializing_if = "Option::is_none")] - pub public_key: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub fingerprint: Option, #[serde(default, skip_serializing_if = "Option::is_none")] #[cfg_attr(feature = "typescript", ts(type = "number | null"))] @@ -1272,6 +1270,13 @@ pub struct WorkspaceRuntimeDetail { pub recent_audit: Vec, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct RuntimeTrustKeyRevealResponse { + pub public_key: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(deny_unknown_fields)] @@ -2522,6 +2527,7 @@ pub fn catalog_typescript() -> String { RuntimeTrustAuditAction::decl(&config), RuntimeTrustAuditEntry::decl(&config), WorkspaceRuntimeDetail::decl(&config), + RuntimeTrustKeyRevealResponse::decl(&config), PutRuntimeTrustKeyRequest::decl(&config), RevokeRuntimeTrustKeyRequest::decl(&config), RuntimeTrustConflictKind::decl(&config), @@ -3250,7 +3256,6 @@ mod tests { "endpoint": "https://runtime.example", "trust_key": { "status": "active", - "public_key": "ssh-ed25519 AAAA runtime-test", "fingerprint": "SHA256:test", "revision": 2, "created_at": "2026-09-01T12:00:00Z", @@ -3271,6 +3276,13 @@ mod tests { let mut unknown = detail; unknown["trust_key"]["private_key"] = serde_json::json!("forbidden"); assert!(serde_json::from_value::(unknown).is_err()); + assert!( + serde_json::from_value::(serde_json::json!({ + "public_key": "yoi-ed25519-pub:v1:key", + "private_key": "forbidden" + })) + .is_err() + ); assert!( serde_json::from_value::(serde_json::json!({ "public_key": "key", diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 41492447..ba0fa922 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -78,10 +78,11 @@ use workspace_api::{ RequestActor, RevokeRuntimeTrustKeyRequest, RotateRepositorySshCredentialRequest, RuntimeConnectionTestFailureKind, RuntimeConnectionTestResponse, RuntimeConnectionTestStatus, RuntimeManagementSummary, RuntimeTrustAuditAction, RuntimeTrustAuditEntry, - RuntimeTrustConflictKind, RuntimeTrustConflictResponse, RuntimeTrustKeyState, - RuntimeTrustKeyStatus, TICKET_ORCHESTRATION_PLANS_QUERY_PATH, TICKET_RELATIONS_QUERY_PATH, - UpdateWorkspaceMetadataRequest, WhoamiResponse, WorkerLaunchOptionsResponse, - WorkerLaunchProfileCandidate, WorkerLaunchRuntimeOption, WorkerLaunchWorkerSummary, + RuntimeTrustConflictKind, RuntimeTrustConflictResponse, RuntimeTrustKeyRevealResponse, + RuntimeTrustKeyState, RuntimeTrustKeyStatus, TICKET_ORCHESTRATION_PLANS_QUERY_PATH, + TICKET_RELATIONS_QUERY_PATH, UpdateWorkspaceMetadataRequest, WhoamiResponse, + WorkerLaunchOptionsResponse, WorkerLaunchProfileCandidate, WorkerLaunchRuntimeOption, + WorkerLaunchWorkerSummary, WorkingDirectoryCreateRequest as BrowserWorkingDirectoryCreateRequest, WorkingDirectoryCreateResponse as BrowserWorkingDirectoryCreateResponse, WorkingDirectoryDetailResponse as BrowserWorkingDirectoryDetailResponse, @@ -2672,7 +2673,9 @@ fn build_inner_router(api: WorkspaceApi) -> Router { ) .route( "/api/w/{workspace_id}/runtimes/{runtime_id}/trust-key", - put(scoped_put_runtime_trust_key).delete(scoped_revoke_runtime_trust_key), + get(scoped_reveal_runtime_trust_key) + .put(scoped_put_runtime_trust_key) + .delete(scoped_revoke_runtime_trust_key), ) .route( "/api/w/{workspace_id}/runtimes/{runtime_id}/connection-tests", @@ -10884,20 +10887,44 @@ async fn scoped_create_remote_runtime( async fn scoped_get_runtime_detail( State(api): State, AxumPath(path): AxumPath, - Extension(actor): Extension, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let workspace = api - .store - .get_workspace(&path.workspace_id) - .await? - .ok_or(Error::WorkspaceIdMismatch)?; - let is_owner = workspace.owner_account_id == actor.account_id; Ok(Json( - workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id, is_owner).await?, + workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id).await?, )) } +async fn scoped_reveal_runtime_trust_key( + State(api): State, + AxumPath(path): AxumPath, + Extension(actor): Extension, +) -> ApiResult> { + validate_workspace_scope(&api, &path.workspace_id)?; + require_workspace_owner( + &api, + &path.workspace_id, + &actor, + "Runtime public key reveal", + ) + .await?; + if path.runtime_id == EMBEDDED_WORKER_RUNTIME_ID { + return Err(settings_bad_request( + "embedded_runtime_trust_managed_internally", + "the embedded Runtime trust key is managed by Server identity authority", + )); + } + let binding = api + .store + .get_workspace_runtime_binding(&path.workspace_id, &path.runtime_id) + .await? + .ok_or_else(|| Error::RuntimeBindingNotFound { + runtime_id: path.runtime_id.clone(), + })?; + Ok(Json(RuntimeTrustKeyRevealResponse { + public_key: binding.public_key, + })) +} + async fn scoped_put_runtime_trust_key( State(api): State, AxumPath(path): AxumPath, @@ -10993,7 +11020,7 @@ async fn scoped_put_runtime_trust_key( .register_remote_runtime(source); } Ok( - Json(workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id, true).await?) + Json(workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id).await?) .into_response(), ) } @@ -11046,7 +11073,7 @@ async fn scoped_revoke_runtime_trust_key( api.runtime_subscription_broker .unregister_runtime(&path.runtime_id); Ok( - Json(workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id, true).await?) + Json(workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id).await?) .into_response(), ) } @@ -14738,7 +14765,6 @@ async fn workspace_runtime_detail( api: &WorkspaceApi, workspace_id: &str, runtime_id: &str, - include_public_key: bool, ) -> ApiResult { let binding = api .store @@ -14800,7 +14826,6 @@ async fn workspace_runtime_detail( let trust_key = binding.as_ref().map_or( RuntimeTrustKeyState { status: RuntimeTrustKeyStatus::Unconfigured, - public_key: None, fingerprint: None, revision: None, created_at: None, @@ -14813,7 +14838,6 @@ async fn workspace_runtime_detail( } else { RuntimeTrustKeyStatus::Active }, - public_key: include_public_key.then(|| binding.public_key.clone()), fingerprint: Some(binding.public_key_fingerprint.clone()), revision: Some(binding.binding_revision), created_at: Some(binding.created_at.clone()), @@ -22529,7 +22553,18 @@ mod tests { .await .unwrap(); - let Json(owner_detail) = scoped_get_runtime_detail( + let Json(detail) = scoped_get_runtime_detail( + State(api.clone()), + AxumPath(ScopedRuntimePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "runtime-a".to_string(), + }), + ) + .await + .unwrap(); + assert_eq!(detail.trust_key.revision, Some(1)); + assert!(detail.trust_key.fingerprint.is_some()); + let Json(revealed) = scoped_reveal_runtime_trust_key( State(api.clone()), AxumPath(ScopedRuntimePath { workspace_id: TEST_WORKSPACE_ID.to_string(), @@ -22539,9 +22574,8 @@ mod tests { ) .await .unwrap(); - assert!(owner_detail.trust_key.public_key.is_some()); - assert_eq!(owner_detail.trust_key.revision, Some(1)); - let Json(reader_detail) = scoped_get_runtime_detail( + assert!(revealed.public_key.starts_with("yoi-ed25519-pub:v1:")); + let denied_reveal = scoped_reveal_runtime_trust_key( State(api.clone()), AxumPath(ScopedRuntimePath { workspace_id: TEST_WORKSPACE_ID.to_string(), @@ -22550,9 +22584,11 @@ mod tests { Extension(non_owner.clone()), ) .await - .unwrap(); - assert!(reader_detail.trust_key.public_key.is_none()); - assert!(reader_detail.trust_key.fingerprint.is_some()); + .unwrap_err(); + assert_eq!( + denied_reveal.into_response().status(), + StatusCode::FORBIDDEN + ); let response = scoped_put_runtime_trust_key( State(api.clone()), diff --git a/web/workspace/src/lib/generated/workspace-api.ts b/web/workspace/src/lib/generated/workspace-api.ts index f28915e7..09be8bf9 100644 --- a/web/workspace/src/lib/generated/workspace-api.ts +++ b/web/workspace/src/lib/generated/workspace-api.ts @@ -276,7 +276,6 @@ export type RuntimeTrustKeyStatus = "unconfigured" | "active" | "revoked"; export type RuntimeTrustKeyState = { status: RuntimeTrustKeyStatus; - public_key?: string | null; fingerprint?: string | null; revision?: number | null; created_at?: string | null; @@ -307,6 +306,8 @@ export type WorkspaceRuntimeDetail = { recent_audit: Array; }; +export type RuntimeTrustKeyRevealResponse = { public_key: string }; + export type PutRuntimeTrustKeyRequest = { public_key: string; expected_revision: number | null; diff --git a/web/workspace/src/lib/workspace/api/runtime-management.ts b/web/workspace/src/lib/workspace/api/runtime-management.ts index cefb9785..2b6a8ee3 100644 --- a/web/workspace/src/lib/workspace/api/runtime-management.ts +++ b/web/workspace/src/lib/workspace/api/runtime-management.ts @@ -11,6 +11,7 @@ import type { RuntimeTrustAuditEntry, RuntimeTrustConflictKind, RuntimeTrustConflictResponse, + RuntimeTrustKeyRevealResponse, RuntimeTrustKeyState, RuntimeTrustKeyStatus, WorkspaceRuntimeDetail, @@ -354,23 +355,11 @@ function trustKey(value: unknown, path: string): RuntimeTrustKeyState { exactKeys( item, ["status"], - [ - "public_key", - "fingerprint", - "revision", - "created_at", - "updated_at", - "revoked_at", - ], + ["fingerprint", "revision", "created_at", "updated_at", "revoked_at"], path, ); const result: RuntimeTrustKeyState = { status: enumValue(item.status, `${path}.status`, TRUST_STATUSES), - public_key: optionalNullableString( - item.public_key, - `${path}.public_key`, - LIMITS.publicKeyBytes, - ), fingerprint: optionalNullableString( item.fingerprint, `${path}.fingerprint`, @@ -538,6 +527,25 @@ export function parseWorkspaceRuntimeDetail( }; } +export function parseRuntimeTrustKeyRevealResponse( + value: unknown, +): RuntimeTrustKeyRevealResponse { + const response = object(value, "Runtime trust key reveal response"); + exactKeys( + response, + ["public_key"], + [], + "Runtime trust key reveal response", + ); + return { + public_key: boundedString( + response.public_key, + "Runtime trust key reveal response.public_key", + LIMITS.publicKeyBytes, + ), + }; +} + export function parseRuntimeTrustConflict( value: unknown, ): RuntimeTrustConflictResponse { @@ -677,6 +685,21 @@ async function finishMutation( return detail; } +export async function revealRuntimeTrustKey( + workspaceId: string, + runtimeId: string, +): Promise { + const response = await fetch( + workspaceApiPath( + workspaceId, + `/runtimes/${encodeURIComponent(runtimeId)}/trust-key`, + ), + ); + const payload = await readBoundedJson(response); + if (!response.ok) throw requestErrorFrom(payload, response.status); + return parseRuntimeTrustKeyRevealResponse(payload); +} + export async function previewRuntimePublicKeyFingerprint( publicKey: string, ): Promise { @@ -739,8 +762,15 @@ export async function revokeRuntimeTrustKey( workspaceId: string, runtimeId: string, request: RevokeRuntimeTrustKeyRequest, + currentFingerprint: string, + confirmation: string, fetchImpl: typeof fetch = fetch, ): Promise { + if (!currentFingerprint || confirmation.trim() !== currentFingerprint) { + throw new RuntimeTrustRequestError( + "Enter the current fingerprint exactly before revoking Workspace trust.", + ); + } const response = await fetchImpl( workspaceApiPath( workspaceId, diff --git a/web/workspace/src/lib/workspace/styles/settings.css b/web/workspace/src/lib/workspace/styles/settings.css index f085acea..03b4c3ad 100644 --- a/web/workspace/src/lib/workspace/styles/settings.css +++ b/web/workspace/src/lib/workspace/styles/settings.css @@ -426,7 +426,8 @@ .runtime-public-key, .runtime-trust-form textarea, - .runtime-trust-form input { + .runtime-trust-form input, + .runtime-revoke-row input { border: 1px solid var(--line); border-radius: 0.5rem; background: var(--bg-raised); @@ -450,14 +451,16 @@ max-width: 56rem; } - .runtime-trust-form label { + .runtime-trust-form label, + .runtime-revoke-row label { color: var(--text-muted); font-size: 0.78rem; font-weight: 700; } .runtime-trust-form textarea, - .runtime-trust-form input { + .runtime-trust-form input, + .runtime-revoke-row input { width: 100%; padding: 0.65rem 0.75rem; } @@ -466,7 +469,8 @@ resize: vertical; } - .runtime-trust-form small { + .runtime-trust-form small, + .runtime-revoke-row small { color: var(--text-muted); } diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte index 13a36d5e..2aa5d7be 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte @@ -8,6 +8,7 @@ import { previewRuntimePublicKeyFingerprint, putRuntimeTrustKey, + revealRuntimeTrustKey, revokeRuntimeTrustKey, RuntimeTrustConflictError, RuntimeTrustRequestError, @@ -17,10 +18,12 @@ type TrustAction = 'create' | 'replace' | 'reactivate'; let { data }: PageProps = $props(); - let revealPublicKey = $state(false); + let showPublicKey = $state(false); + let revealedPublicKey = $state(null); let publicKey = $state(''); let fingerprintConfirmation = $state(''); - let busyAction = $state<'save' | 'revoke' | 'copy' | null>(null); + let revokeFingerprintConfirmation = $state(''); + let busyAction = $state<'save' | 'revoke' | 'reveal' | 'copy' | null>(null); let fieldError = $state(null); let requestError = $state(null); let successMessage = $state(null); @@ -125,7 +128,9 @@ await putRuntimeTrustKey(data.workspaceId, data.runtimeId, request); publicKey = ''; fingerprintConfirmation = ''; - revealPublicKey = false; + revokeFingerprintConfirmation = ''; + showPublicKey = false; + revealedPublicKey = null; successMessage = action === 'create' ? 'Workspace trust was created.' : action === 'replace' @@ -154,6 +159,13 @@ requestError = 'Only active Workspace trust can be revoked.'; return; } + if ( + !trust.fingerprint || + revokeFingerprintConfirmation.trim() !== trust.fingerprint + ) { + fieldError = 'Enter the current fingerprint exactly before revoking Workspace trust.'; + return; + } fieldError = null; requestError = null; @@ -164,10 +176,18 @@ }; try { - await revokeRuntimeTrustKey(data.workspaceId, data.runtimeId, request); + await revokeRuntimeTrustKey( + data.workspaceId, + data.runtimeId, + request, + trust.fingerprint, + revokeFingerprintConfirmation, + ); publicKey = ''; fingerprintConfirmation = ''; - revealPublicKey = false; + revokeFingerprintConfirmation = ''; + showPublicKey = false; + revealedPublicKey = null; successMessage = 'Workspace trust was revoked.'; await reloadAuthority(); } catch (error) { @@ -182,16 +202,40 @@ } } + async function togglePublicKeyReveal(): Promise { + if (showPublicKey) { + showPublicKey = false; + revealedPublicKey = null; + return; + } + if (busyAction !== null) return; + busyAction = 'reveal'; + requestError = null; + successMessage = null; + try { + const response = await revealRuntimeTrustKey(data.workspaceId, data.runtimeId); + revealedPublicKey = response.public_key; + showPublicKey = true; + } catch (error) { + requestError = error instanceof Error ? error.message : 'Public key reveal failed.'; + } finally { + busyAction = null; + } + } + async function copyPublicKey(): Promise { - const key = data.runtimeDetail?.trust_key.public_key; - if (!key || busyAction !== null) return; + if (busyAction !== null) return; busyAction = 'copy'; requestError = null; + successMessage = null; try { - await navigator.clipboard.writeText(key); + const response = await revealRuntimeTrustKey(data.workspaceId, data.runtimeId); + await navigator.clipboard.writeText(response.public_key); successMessage = 'Public key copied.'; - } catch { - requestError = 'The browser could not copy the public key.'; + } catch (error) { + requestError = error instanceof Error + ? error.message + : 'The browser could not copy the public key.'; } finally { busyAction = null; } @@ -255,20 +299,23 @@

Workspace trust

- {#if trust.public_key} + {#if trust.status !== 'unconfigured'}
-
- {#if revealPublicKey} - 
{trust.public_key}
+ {#if showPublicKey && revealedPublicKey} + 
{revealedPublicKey}
{/if} - {:else if trust.status !== 'unconfigured'} -

The public key was not included in this authorized response.

{/if}
@@ -324,11 +371,25 @@
Revoke Workspace trust

Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.

+
diff --git a/web/workspace/tests/runtime-management-source.test.ts b/web/workspace/tests/runtime-management-source.test.ts index ac08da59..d1ef4af4 100644 --- a/web/workspace/tests/runtime-management-source.test.ts +++ b/web/workspace/tests/runtime-management-source.test.ts @@ -110,6 +110,8 @@ Deno.test("Runtime detail keeps trust controls owner-only and conflict-safe", as "Revoke Workspace trust", "Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.", "RuntimeTrustConflictError", + "revealRuntimeTrustKey", + "revokeFingerprintConfirmation.trim() !== trust.fingerprint", "await reloadAuthority()", "busyAction !== null", "Workdirs", diff --git a/web/workspace/tests/runtime-management.test.ts b/web/workspace/tests/runtime-management.test.ts index daf80b71..9e9daf5b 100644 --- a/web/workspace/tests/runtime-management.test.ts +++ b/web/workspace/tests/runtime-management.test.ts @@ -4,10 +4,12 @@ declare const Deno: { import { parseRuntimeTrustConflict, + parseRuntimeTrustKeyRevealResponse, parseWorkspaceRuntimeDetail, parseWorkspaceRuntimeList, previewRuntimePublicKeyFingerprint, putRuntimeTrustKey, + revokeRuntimeTrustKey, RuntimeTrustConflictError, } from "../src/lib/workspace/api/runtime-management.ts"; @@ -62,7 +64,6 @@ function detail() { endpoint: "https://runtime.example.test", trust_key: { status: "active", - public_key: "ssh-ed25519 AAAA-test", fingerprint: "SHA256:current", revision: 3, created_at: "2026-09-01T12:00:00Z", @@ -162,10 +163,11 @@ Deno.test("Runtime validators reject unsafe revisions and bounded collection ove }); Deno.test("Runtime detail rejects unbounded strings and incoherent trust state", () => { - const largeKey = structuredClone(detail()); - largeKey.trust_key.public_key = "x".repeat(16 * 1024 + 1); assertThrows( - () => parseWorkspaceRuntimeDetail(largeKey), + () => + parseRuntimeTrustKeyRevealResponse({ + public_key: "x".repeat(16 * 1024 + 1), + }), "must be at most 16384 UTF-8 bytes", ); @@ -181,6 +183,30 @@ Deno.test("Runtime detail rejects unbounded strings and incoherent trust state", ); }); +Deno.test("mismatched revoke fingerprint never sends a request", async () => { + let requests = 0; + const fetchImpl: typeof fetch = () => { + requests += 1; + return Promise.reject(new Error("request must not be sent")); + }; + let rejected = false; + try { + await revokeRuntimeTrustKey( + "workspace-a", + "runtime-a", + { expected_revision: 3 }, + "sha256:current", + "sha256:different", + fetchImpl, + ); + } catch (error) { + rejected = error instanceof Error && + error.message.includes("current fingerprint exactly"); + } + assert(rejected, "mismatched fingerprint should be rejected locally"); + assert(requests === 0, "mismatched fingerprint sent a revoke request"); +}); + Deno.test("Runtime public key preview matches the Server fingerprint contract", async () => { const fingerprint = await previewRuntimePublicKeyFingerprint( "yoi-ed25519-pub:v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", From 668a9062b3116784d98ac18d333689cfd78f3fc5 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 06:11:23 +0900 Subject: [PATCH 27/39] fix: fence runtime detail route state --- crates/workspace-server/src/server.rs | 16 +++++- .../lib/workspace/api/runtime-management.ts | 28 ++++++++++ .../runtimes/[runtimeId]/+page.svelte | 55 ++++++++++++++++--- .../tests/runtime-management-source.test.ts | 10 ++++ .../tests/runtime-management.test.ts | 22 ++++++++ 5 files changed, 122 insertions(+), 9 deletions(-) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index ba0fa922..41db9ae9 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -14684,7 +14684,7 @@ async fn workspace_runtime_resources_response( let runtimes = api.runtime.list_runtimes(limit); let bindings = api .store - .list_workspace_runtime_bindings(workspace_id, false) + .list_workspace_runtime_bindings(workspace_id, true) .await?; let mut items = runtimes .items @@ -22674,6 +22674,20 @@ mod tests { .unwrap(); assert_eq!(binding.binding_revision, 3); assert!(binding.revoked_at.is_some()); + let listed = workspace_runtime_resources_response(&api, TEST_WORKSPACE_ID) + .await + .unwrap(); + let listed_runtime = listed + .items + .iter() + .find(|resource| resource.runtime.runtime_id == "runtime-a") + .expect("revoked binding must remain listed"); + assert!(listed_runtime.management.config_managed); + let detail = workspace_runtime_detail(&api, TEST_WORKSPACE_ID, "runtime-a") + .await + .unwrap(); + assert_eq!(detail.trust_key.status, RuntimeTrustKeyStatus::Revoked); + assert!(detail.runtime.management.config_managed); assert!( !api.runtime_binding_expectations .read() diff --git a/web/workspace/src/lib/workspace/api/runtime-management.ts b/web/workspace/src/lib/workspace/api/runtime-management.ts index 2b6a8ee3..ac15d012 100644 --- a/web/workspace/src/lib/workspace/api/runtime-management.ts +++ b/web/workspace/src/lib/workspace/api/runtime-management.ts @@ -98,6 +98,34 @@ export class RuntimeTrustRequestError extends Error { } } +export type RuntimeTrustRouteOperation = Readonly<{ + runtimeId: string; + generation: number; +}>; + +export class RuntimeTrustRouteFence { + #runtimeId: string | null = null; + #generation = 0; + + enter(runtimeId: string): number { + if (this.#runtimeId !== runtimeId) { + this.#runtimeId = runtimeId; + this.#generation += 1; + } + return this.#generation; + } + + capture(runtimeId: string): RuntimeTrustRouteOperation { + return { runtimeId, generation: this.enter(runtimeId) }; + } + + isCurrent(operation: RuntimeTrustRouteOperation, runtimeId: string): boolean { + return operation.runtimeId === runtimeId && + operation.generation === this.#generation && + this.#runtimeId === runtimeId; + } +} + function fail(path: string, message: string): never { throw new RuntimeManagementValidationError(`${path} ${message}`); } diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte index 2aa5d7be..bceedc8f 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte @@ -11,7 +11,9 @@ revealRuntimeTrustKey, revokeRuntimeTrustKey, RuntimeTrustConflictError, + RuntimeTrustRouteFence, RuntimeTrustRequestError, + type RuntimeTrustRouteOperation, } from '$lib/workspace/api/runtime-management'; import type { PageProps } from './$types'; @@ -30,6 +32,26 @@ let replacementFingerprint = $state(null); let replacementFingerprintError = $state(null); let fingerprintGeneration = 0; + const routeFence = new RuntimeTrustRouteFence(); + let routeGeneration = 0; + + $effect(() => { + const nextGeneration = routeFence.enter(data.runtimeId); + if (nextGeneration === routeGeneration) return; + routeGeneration = nextGeneration; + fingerprintGeneration += 1; + showPublicKey = false; + revealedPublicKey = null; + publicKey = ''; + fingerprintConfirmation = ''; + revokeFingerprintConfirmation = ''; + busyAction = null; + fieldError = null; + requestError = null; + successMessage = null; + replacementFingerprint = null; + replacementFingerprintError = null; + }); $effect(() => { const key = publicKey.trim(); @@ -79,6 +101,10 @@ await invalidateAll(); } + function isCurrentRoute(operation: RuntimeTrustRouteOperation): boolean { + return routeFence.isCurrent(operation, data.runtimeId); + } + async function saveTrustKey(event: SubmitEvent): Promise { event.preventDefault(); if (busyAction !== null || !data.runtimeDetail) return; @@ -123,9 +149,11 @@ expected_revision: trust.revision ?? null, }; + const operation = routeFence.capture(data.runtimeId); busyAction = 'save'; try { - await putRuntimeTrustKey(data.workspaceId, data.runtimeId, request); + await putRuntimeTrustKey(data.workspaceId, operation.runtimeId, request); + if (!isCurrentRoute(operation)) return; publicKey = ''; fingerprintConfirmation = ''; revokeFingerprintConfirmation = ''; @@ -138,6 +166,7 @@ : 'Workspace trust was reactivated.'; await reloadAuthority(); } catch (error) { + if (!isCurrentRoute(operation)) return; fingerprintConfirmation = ''; if (error instanceof RuntimeTrustConflictError) { requestError = `${error.message} Authoritative Runtime trust has been reloaded.`; @@ -148,7 +177,7 @@ requestError = error instanceof Error ? error.message : 'Runtime trust update failed.'; } } finally { - busyAction = null; + if (isCurrentRoute(operation)) busyAction = null; } } @@ -170,6 +199,7 @@ fieldError = null; requestError = null; successMessage = null; + const operation = routeFence.capture(data.runtimeId); busyAction = 'revoke'; const request: RevokeRuntimeTrustKeyRequest = { expected_revision: trust.revision, @@ -178,11 +208,12 @@ try { await revokeRuntimeTrustKey( data.workspaceId, - data.runtimeId, + operation.runtimeId, request, trust.fingerprint, revokeFingerprintConfirmation, ); + if (!isCurrentRoute(operation)) return; publicKey = ''; fingerprintConfirmation = ''; revokeFingerprintConfirmation = ''; @@ -191,6 +222,7 @@ successMessage = 'Workspace trust was revoked.'; await reloadAuthority(); } catch (error) { + if (!isCurrentRoute(operation)) return; if (error instanceof RuntimeTrustConflictError) { requestError = `${error.message} Authoritative Runtime trust has been reloaded.`; await reloadAuthority(); @@ -198,7 +230,7 @@ requestError = error instanceof Error ? error.message : 'Runtime trust revoke failed.'; } } finally { - busyAction = null; + if (isCurrentRoute(operation)) busyAction = null; } } @@ -209,35 +241,42 @@ return; } if (busyAction !== null) return; + const operation = routeFence.capture(data.runtimeId); busyAction = 'reveal'; requestError = null; successMessage = null; try { - const response = await revealRuntimeTrustKey(data.workspaceId, data.runtimeId); + const response = await revealRuntimeTrustKey(data.workspaceId, operation.runtimeId); + if (!isCurrentRoute(operation)) return; revealedPublicKey = response.public_key; showPublicKey = true; } catch (error) { + if (!isCurrentRoute(operation)) return; requestError = error instanceof Error ? error.message : 'Public key reveal failed.'; } finally { - busyAction = null; + if (isCurrentRoute(operation)) busyAction = null; } } async function copyPublicKey(): Promise { if (busyAction !== null) return; + const operation = routeFence.capture(data.runtimeId); busyAction = 'copy'; requestError = null; successMessage = null; try { - const response = await revealRuntimeTrustKey(data.workspaceId, data.runtimeId); + const response = await revealRuntimeTrustKey(data.workspaceId, operation.runtimeId); + if (!isCurrentRoute(operation)) return; await navigator.clipboard.writeText(response.public_key); + if (!isCurrentRoute(operation)) return; successMessage = 'Public key copied.'; } catch (error) { + if (!isCurrentRoute(operation)) return; requestError = error instanceof Error ? error.message : 'The browser could not copy the public key.'; } finally { - busyAction = null; + if (isCurrentRoute(operation)) busyAction = null; } } diff --git a/web/workspace/tests/runtime-management-source.test.ts b/web/workspace/tests/runtime-management-source.test.ts index d1ef4af4..1aaee2c1 100644 --- a/web/workspace/tests/runtime-management-source.test.ts +++ b/web/workspace/tests/runtime-management-source.test.ts @@ -110,6 +110,16 @@ Deno.test("Runtime detail keeps trust controls owner-only and conflict-safe", as "Revoke Workspace trust", "Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.", "RuntimeTrustConflictError", + "RuntimeTrustRouteFence", + "routeFence.enter(data.runtimeId)", + "showPublicKey = false", + "revealedPublicKey = null", + "publicKey = ''", + "fingerprintConfirmation = ''", + "revokeFingerprintConfirmation = ''", + "requestError = null", + "successMessage = null", + "isCurrentRoute(operation)", "revealRuntimeTrustKey", "revokeFingerprintConfirmation.trim() !== trust.fingerprint", "await reloadAuthority()", diff --git a/web/workspace/tests/runtime-management.test.ts b/web/workspace/tests/runtime-management.test.ts index 9e9daf5b..3d5c83b6 100644 --- a/web/workspace/tests/runtime-management.test.ts +++ b/web/workspace/tests/runtime-management.test.ts @@ -11,6 +11,7 @@ import { putRuntimeTrustKey, revokeRuntimeTrustKey, RuntimeTrustConflictError, + RuntimeTrustRouteFence, } from "../src/lib/workspace/api/runtime-management.ts"; function assert(condition: unknown, message: string): asserts condition { @@ -207,6 +208,27 @@ Deno.test("mismatched revoke fingerprint never sends a request", async () => { assert(requests === 0, "mismatched fingerprint sent a revoke request"); }); +Deno.test("Runtime route fence rejects a delayed reveal from the prior Runtime", async () => { + const fence = new RuntimeTrustRouteFence(); + fence.enter("runtime-a"); + const operation = fence.capture("runtime-a"); + let renderedKey: string | null = null; + let resolveReveal!: (key: string) => void; + const delayedReveal = new Promise((resolve) => { + resolveReveal = resolve; + }).then((key) => { + if (fence.isCurrent(operation, "runtime-b")) renderedKey = key; + }); + + fence.enter("runtime-b"); + resolveReveal("runtime-a-public-key"); + await delayedReveal; + assert( + renderedKey === null, + "Runtime A key rendered after navigating to Runtime B", + ); +}); + Deno.test("Runtime public key preview matches the Server fingerprint contract", async () => { const fingerprint = await previewRuntimePublicKeyFingerprint( "yoi-ed25519-pub:v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", From e8b9adcde43028bae4bae1cb5ee067c018e7693a Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 06:54:29 +0900 Subject: [PATCH 28/39] feat: add revisioned worker execution state --- crates/client/src/client.rs | 8 +- crates/client/src/transport/in_process.rs | 8 +- crates/client/src/transport/unix_socket.rs | 11 +- crates/client/src/transport/websocket.rs | 8 +- crates/protocol/src/lib.rs | 274 ++++++-- crates/protocol/src/typescript.rs | 14 +- crates/standalone/src/host.rs | 12 +- crates/tui/src/app.rs | 70 +- crates/tui/src/command.rs | 9 +- crates/tui/src/console/mod.rs | 43 +- crates/worker-runtime/src/execution.rs | 42 +- crates/worker-runtime/src/http_server.rs | 40 +- crates/worker-runtime/src/runtime.rs | 216 +++--- crates/worker-runtime/src/worker_backend.rs | 415 ++++++----- crates/worker/src/controller.rs | 647 +++++++++++++++--- crates/worker/src/discovery.rs | 16 +- crates/worker/src/internal_worker.rs | 70 +- crates/worker/src/runtime/dir.rs | 5 +- crates/worker/src/shared_state.rs | 141 ++-- crates/worker/src/spawn/comm_tools.rs | 12 +- crates/worker/src/worker.rs | 206 +++++- crates/worker/tests/compact_events_test.rs | 208 +++++- crates/worker/tests/controller_test.rs | 200 ++++-- crates/workspace-server/src/hosts.rs | 13 +- .../src/runtime_subscription_tests.rs | 22 +- crates/workspace-server/src/server.rs | 20 +- web/workspace/src/lib/generated/protocol.ts | 42 +- .../src/lib/workspace/console/model.test.ts | 37 +- .../src/lib/workspace/console/model.ts | 27 +- .../lib/workspace/console/run-status.test.ts | 7 +- .../src/lib/workspace/console/run-status.ts | 4 +- .../workers/[workerId]/console/+page.svelte | 71 +- 32 files changed, 2168 insertions(+), 750 deletions(-) diff --git a/crates/client/src/client.rs b/crates/client/src/client.rs index b760ce00..bdd73375 100644 --- a/crates/client/src/client.rs +++ b/crates/client/src/client.rs @@ -112,8 +112,8 @@ mod tests { async fn encodes_methods_and_decodes_events_above_transport() { let mut socket = TestSocket::default(); socket.incoming.push_back( - encode_event(&Event::Status { - status: WorkerStatus::Idle, + encode_event(&Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }) .expect("encode event"), ); @@ -132,9 +132,7 @@ mod tests { )); assert!(matches!( client.next_event().await, - Ok(Some(Event::Status { - status: WorkerStatus::Idle - })) + Ok(Some(Event::WorkerState { .. })) )); } } diff --git a/crates/client/src/transport/in_process.rs b/crates/client/src/transport/in_process.rs index 3111808a..ed50be54 100644 --- a/crates/client/src/transport/in_process.rs +++ b/crates/client/src/transport/in_process.rs @@ -101,8 +101,8 @@ mod tests { )); peer.send( - encode_event(&Event::Status { - status: WorkerStatus::Idle, + encode_event(&Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }) .expect("encode event"), ) @@ -110,9 +110,7 @@ mod tests { .expect("send event"); assert!(matches!( client.next_event().await, - Ok(Some(Event::Status { - status: WorkerStatus::Idle - })) + Ok(Some(Event::WorkerState { .. })) )); } } diff --git a/crates/client/src/transport/unix_socket.rs b/crates/client/src/transport/unix_socket.rs index 089ed83e..8b57bb2c 100644 --- a/crates/client/src/transport/unix_socket.rs +++ b/crates/client/src/transport/unix_socket.rs @@ -113,8 +113,8 @@ mod tests { let listener = UnixListener::bind(&socket_path).unwrap(); let server = tokio::spawn(async move { let (mut stream, _) = listener.accept().await.unwrap(); - let event = encode_event(&Event::Status { - status: WorkerStatus::Idle, + let event = encode_event(&Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }) .unwrap(); stream.write_all(event.as_bytes()).await.unwrap(); @@ -126,12 +126,7 @@ mod tests { .await .expect("client should receive event while alive") .expect("transport should succeed"); - assert!(matches!( - event, - Some(Event::Status { - status: WorkerStatus::Idle - }) - )); + assert!(matches!(event, Some(Event::WorkerState { .. }))); server.await.unwrap(); } diff --git a/crates/client/src/transport/websocket.rs b/crates/client/src/transport/websocket.rs index b4c1ed84..883af12e 100644 --- a/crates/client/src/transport/websocket.rs +++ b/crates/client/src/transport/websocket.rs @@ -116,8 +116,8 @@ mod tests { Message::Text(ref text) if matches!(decode_method(text), Ok(Method::Submit { .. })) )); - let event = encode_event(&Event::Status { - status: WorkerStatus::Idle, + let event = encode_event(&Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }) .unwrap(); socket.send(Message::Text(event.into())).await.unwrap(); @@ -134,9 +134,7 @@ mod tests { .expect("send method"); assert!(matches!( client.next_event().await, - Ok(Some(Event::Status { - status: WorkerStatus::Idle - })) + Ok(Some(Event::WorkerState { .. })) )); server.await.unwrap(); } diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index fd5190c7..ec96e226 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -85,6 +85,142 @@ impl AuthenticatedInputSource { } } +/// Immutable identity and revision fence for one state-changing Worker command. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkerCommandEnvelope { + /// Caller-owned sequence. A controller accepts command ids in strictly + /// increasing order for one execution generation. + pub command_id: u64, + pub expected_execution_generation: u64, + pub expected_worker_state_revision: u64, +} + +impl WorkerCommandEnvelope { + pub fn for_snapshot(command_id: u64, snapshot: &WorkerStateSnapshot) -> Self { + Self { + command_id, + expected_execution_generation: snapshot.execution_generation, + expected_worker_state_revision: snapshot.revision, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkerCommandKind { + Resume, + Cancel, + Pause, + Compact, + Shutdown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkerCommandDisposition { + Accepted, + StaleExecutionGeneration, + StaleWorkerStateRevision, + StaleCommandId, + InvalidState, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkerCommandAcknowledgement { + pub command_id: u64, + pub command: WorkerCommandKind, + pub disposition: WorkerCommandDisposition, + /// The complete authoritative state observed after command admission. + pub state: WorkerStateSnapshot, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(tag = "kind", content = "state", rename_all = "snake_case")] +pub enum WorkerState { + Idle, + Busy(WorkerBusyState), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(tag = "kind", content = "state", rename_all = "snake_case")] +pub enum WorkerBusyState { + Run(WorkerRunState), + Maintenance(WorkerMaintenanceState), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkerRunState { + Running, + Pausing, + Paused, + Cancelling, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkerMaintenanceState { + Compacting, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkerStateSnapshot { + pub execution_generation: u64, + pub revision: u64, + /// Highest lifecycle command id observed by this controller generation. + pub last_command_id: u64, + pub state: WorkerState, +} + +impl WorkerStateSnapshot { + pub fn initial(execution_generation: u64) -> Self { + Self { + execution_generation, + revision: 0, + last_command_id: 0, + state: WorkerState::Idle, + } + } + + /// Compatibility projection for Runtime catalog lifecycle. This value is + /// never command-admission authority and cannot produce `Stopped`. + pub fn catalog_status(&self) -> WorkerStatus { + match self.state { + WorkerState::Idle => WorkerStatus::Idle, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused, + WorkerState::Busy(WorkerBusyState::Run(_)) + | WorkerState::Busy(WorkerBusyState::Maintenance(_)) => WorkerStatus::Running, + } + } +} + +impl From for WorkerStateSnapshot { + fn from(status: WorkerStatus) -> Self { + let state = match status { + WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle, + WorkerStatus::Running => { + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)) + } + WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)), + }; + Self { + execution_generation: 1, + revision: 0, + last_command_id: 0, + state, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(tag = "method", content = "params", rename_all = "snake_case")] @@ -149,20 +285,28 @@ pub enum Method { expected_revision: u64, expected_head_id: String, }, - Resume, - Cancel, + Resume { + command: WorkerCommandEnvelope, + }, + Cancel { + command: WorkerCommandEnvelope, + }, /// Stop the in-flight turn and transition to `Paused`. /// /// Unlike `Cancel` (which discards and returns to `Idle`), a paused /// Worker can resume the interrupted work via `Resume`, or accept a /// fresh `Submit` (orphan `tool_use` items are closed with a /// synthetic tool result before the new user message is appended). - Pause, + Pause { + command: WorkerCommandEnvelope, + }, /// Request an explicit compaction while the Worker is otherwise idle. /// /// This is a typed control method: clients must not send `compact` as a /// `Method::Submit` user message. - Compact, + Compact { + command: WorkerCommandEnvelope, + }, /// Ask the Worker to list valid rewind targets from its authoritative session log. ListRewindTargets, /// Truncate the current session back to the selected rewind target and @@ -171,7 +315,9 @@ pub enum Method { target: RewindTargetId, expected_head_entries: usize, }, - Shutdown, + Shutdown { + command: WorkerCommandEnvelope, + }, /// Request a list of completion candidates from the Worker. /// /// Reply is sent on the same socket as `Event::Completions` (not @@ -938,8 +1084,9 @@ pub enum Event { Snapshot { session: SessionSnapshot, greeting: Greeting, - #[serde(default)] - status: WorkerStatus, + /// Full revisioned live execution state. `Stopped` remains Runtime + /// catalog authority and is deliberately not represented here. + state: WorkerStateSnapshot, /// Unfinished model output that has already streamed in the current /// run but is not yet represented by committed snapshot entries. #[serde(default, skip_serializing_if = "InFlightSnapshot::is_empty")] @@ -976,8 +1123,11 @@ pub enum Event { }, /// Current Worker controller status. Broadcast on every controller-level /// transition and included in `History` snapshots for late attach. - Status { - status: WorkerStatus, + WorkerState { + snapshot: WorkerStateSnapshot, + }, + CommandAcknowledged { + acknowledgement: WorkerCommandAcknowledgement, }, /// Bounded, provider-owned command telemetry for the live Console. This is /// intentionally not a history entry and is reconstructed from @@ -1612,28 +1762,39 @@ mod tests { } #[test] - fn method_without_params() { - let json = r#"{"method":"resume"}"#; - let method: Method = serde_json::from_str(json).unwrap(); - assert!(matches!(method, Method::Resume)); + fn lifecycle_method_without_command_fails_closed() { + let error = serde_json::from_str::(r#"{"method":"resume"}"#).unwrap_err(); + assert!(error.to_string().contains("params")); } #[test] - fn method_pause_roundtrip() { - let json = r#"{"method":"pause"}"#; - let method: Method = serde_json::from_str(json).unwrap(); - assert!(matches!(method, Method::Pause)); - let serialized = serde_json::to_string(&method).unwrap(); - assert_eq!(serialized, json); - } - - #[test] - fn method_compact_roundtrip() { - let json = r#"{"method":"compact"}"#; - let method: Method = serde_json::from_str(json).unwrap(); - assert!(matches!(method, Method::Compact)); - let serialized = serde_json::to_string(&method).unwrap(); - assert_eq!(serialized, json); + fn lifecycle_methods_roundtrip_with_fences() { + for method in [ + Method::Pause { + command: WorkerCommandEnvelope { + command_id: 11, + expected_execution_generation: 4, + expected_worker_state_revision: 8, + }, + }, + Method::Compact { + command: WorkerCommandEnvelope { + command_id: 12, + expected_execution_generation: 4, + expected_worker_state_revision: 9, + }, + }, + ] { + let json = serde_json::to_string(&method).unwrap(); + let decoded: Method = serde_json::from_str(&json).unwrap(); + match decoded { + Method::Pause { command } | Method::Compact { command } => { + assert_eq!(command.expected_execution_generation, 4); + assert!(command.command_id >= 11); + } + other => panic!("unexpected lifecycle method: {other:?}"), + } + } } #[test] @@ -1902,7 +2063,7 @@ mod tests { context_window: 200_000, context_tokens: 42_000, }, - status: WorkerStatus::Paused, + state: WorkerStatus::Paused.into(), in_flight: InFlightSnapshot::default(), internal_workers: Vec::new(), }; @@ -1919,12 +2080,13 @@ mod tests { assert_eq!(parsed["data"]["greeting"]["tools"][0], "Read"); assert_eq!(parsed["data"]["greeting"]["context_window"], 200_000); assert_eq!(parsed["data"]["greeting"]["context_tokens"], 42_000); - assert_eq!(parsed["data"]["status"], "paused"); + assert_eq!(parsed["data"]["state"]["state"]["kind"], "busy"); + assert_eq!(parsed["data"]["state"]["state"]["state"]["state"], "paused"); } #[test] fn event_snapshot_in_flight_roundtrip_and_default() { - let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"status":"running"}}"#; + let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"state":{"execution_generation":1,"revision":1,"last_command_id":0,"state":{"kind":"busy","state":{"kind":"run","state":"running"}}}}}"#; let decoded: Event = serde_json::from_str(inbound).unwrap(); match decoded { Event::Snapshot { in_flight, .. } => assert!(in_flight.is_empty()), @@ -1946,7 +2108,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Running, + state: WorkerStatus::Running.into(), in_flight: InFlightSnapshot { blocks: vec![ InFlightBlock::Text { @@ -2034,20 +2196,32 @@ mod tests { } #[test] - fn event_status_format() { - let event = Event::Status { - status: WorkerStatus::Running, + fn event_worker_state_format() { + let event = Event::WorkerState { + snapshot: WorkerStateSnapshot { + execution_generation: 7, + revision: 3, + last_command_id: 9, + state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), + }, }; let json = serde_json::to_string(&event).unwrap(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["event"], "status"); - assert_eq!(parsed["data"]["status"], "running"); + assert_eq!(parsed["event"], "worker_state"); + assert_eq!(parsed["data"]["snapshot"]["execution_generation"], 7); + assert_eq!(parsed["data"]["snapshot"]["revision"], 3); + assert_eq!(parsed["data"]["snapshot"]["state"]["kind"], "busy"); let decoded: Event = serde_json::from_str(&json).unwrap(); assert!(matches!( decoded, - Event::Status { - status: WorkerStatus::Running + Event::WorkerState { + snapshot: WorkerStateSnapshot { + execution_generation: 7, + revision: 3, + state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), + .. + } } )); } @@ -2088,19 +2262,10 @@ mod tests { } #[test] - fn event_snapshot_without_status_defaults_to_idle() { + fn event_snapshot_without_worker_state_fails_closed() { let json = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#; - let decoded: Event = serde_json::from_str(json).unwrap(); - match decoded { - Event::Snapshot { - status, greeting, .. - } => { - assert_eq!(status, WorkerStatus::Idle); - assert_eq!(greeting.context_window, 0); - assert_eq!(greeting.context_tokens, 0); - } - other => panic!("expected Snapshot, got {other:?}"), - } + let error = serde_json::from_str::(json).unwrap_err(); + assert!(error.to_string().contains("state")); } #[test] @@ -2513,7 +2678,12 @@ mod tests { "scope_summary": "scope", "tools": [] }, - "status": "idle" + "state": { + "execution_generation": 1, + "revision": 0, + "last_command_id": 0, + "state": { "kind": "idle" } + } } })) .unwrap(); diff --git a/crates/protocol/src/typescript.rs b/crates/protocol/src/typescript.rs index bf63c8d6..ec5588f9 100644 --- a/crates/protocol/src/typescript.rs +++ b/crates/protocol/src/typescript.rs @@ -12,7 +12,10 @@ use crate::{ RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart, SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition, - TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, WorkerStatus, + TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerBusyState, + WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope, + WorkerCommandKind, WorkerEvent, WorkerMaintenanceState, WorkerRunState, WorkerState, + WorkerStateSnapshot, WorkerStatus, subscription::{ EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame, SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest, @@ -46,6 +49,15 @@ pub fn generated_protocol_types() -> String { push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); diff --git a/crates/standalone/src/host.rs b/crates/standalone/src/host.rs index c4c5df73..1131bccd 100644 --- a/crates/standalone/src/host.rs +++ b/crates/standalone/src/host.rs @@ -318,7 +318,11 @@ impl StandaloneHost { } pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> { - let _ = self.handle.send(Method::Shutdown).await; + let command = protocol::WorkerCommandEnvelope::for_snapshot( + u64::MAX, + &self.handle.shared_state.snapshot(), + ); + let _ = self.handle.send(Method::Shutdown { command }).await; let Some(shutdown) = self.shutdown.take() else { self.retain_lease(); return Err(StandaloneShutdownError::ConfirmationLost); @@ -500,7 +504,11 @@ fn active_pointer( } async fn stop_started_worker(started: BootstrappedWorker) { - let _ = started.handle.send(Method::Shutdown).await; + let command = protocol::WorkerCommandEnvelope::for_snapshot( + u64::MAX, + &started.handle.shared_state.snapshot(), + ); + let _ = started.handle.send(Method::Shutdown { command }).await; let _ = tokio::time::timeout(Duration::from_secs(2), started.shutdown).await; } diff --git a/crates/tui/src/app.rs b/crates/tui/src/app.rs index d23430c2..a76d32b5 100644 --- a/crates/tui/src/app.rs +++ b/crates/tui/src/app.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use protocol::{ AlertLevel, AlertSource, CompletionEntry, CompletionKind, ErrorCode, Event, InFlightBlock, InFlightSnapshot, InFlightToolCallState, InternalWorkerRef, InternalWorkerSnapshot, Method, - RewindTarget, RunResult, Segment, WorkerStatus, + RewindTarget, RunResult, Segment, WorkerCommandEnvelope, WorkerStateSnapshot, WorkerStatus, }; use crate::block::{ @@ -225,8 +225,10 @@ pub struct WorkerViewTab { pub struct App { pub worker_name: String, pub connected: bool, - /// Last controller status reported by the Worker. Drives the status line - /// and Ctrl-key routing; do not infer this solely from replayed history. + /// Latest authoritative revisioned live execution state. + pub worker_state: WorkerStateSnapshot, + next_command_id: u64, + /// Derived Runtime-catalog compatibility projection used by existing UI. pub worker_status: WorkerStatus, /// True while the Worker is in `WorkerStatus::Running`. pub running: bool, @@ -337,6 +339,8 @@ impl App { Self { worker_name, connected: false, + worker_state: WorkerStateSnapshot::initial(1), + next_command_id: 1, worker_status: WorkerStatus::Idle, running: false, paused: false, @@ -745,7 +749,8 @@ impl App { if self.paused { self.input_history.cancel_browse(); self.input.clear(); - return Some(Method::Resume); + let command = self.next_command_envelope(); + return Some(Method::Resume { command }); } return None; } @@ -1114,6 +1119,15 @@ impl App { } } + pub fn next_command_envelope(&mut self) -> WorkerCommandEnvelope { + let command_id = self + .next_command_id + .max(self.worker_state.last_command_id.saturating_add(1)); + let command = WorkerCommandEnvelope::for_snapshot(command_id, &self.worker_state); + self.next_command_id = command_id.saturating_add(1); + command + } + pub fn handle_worker_event(&mut self, event: Event) -> Option { if self.rewind_refresh_fence && event_is_stale_after_rewind(&event) { return None; @@ -1443,7 +1457,7 @@ impl App { Event::Snapshot { session, greeting, - status, + state, in_flight, internal_workers, } => { @@ -1451,7 +1465,8 @@ impl App { self.pending_submissions = session.pending_submissions.clone(); self.restore_snapshot(&session, greeting, in_flight); self.replace_internal_worker_snapshots(internal_workers); - self.set_worker_status(status); + self.worker_state = state.clone(); + self.set_worker_status(state.catalog_status()); } Event::InternalWorker { worker, @@ -1461,9 +1476,14 @@ impl App { Event::InternalWorkerRemoved { worker, revision } => { self.remove_internal_worker(worker, revision) } - Event::Status { status } => { + Event::WorkerState { snapshot } => { self.rewind_refresh_fence = false; - self.set_worker_status(status); + self.worker_state = snapshot.clone(); + self.set_worker_status(snapshot.catalog_status()); + } + Event::CommandAcknowledged { acknowledgement } => { + self.worker_state = acknowledgement.state.clone(); + self.set_worker_status(acknowledgement.state.catalog_status()); } // Command telemetry is an operational Web Console surface. The // TUI continues to render the final Bash ToolResult from history. @@ -2026,12 +2046,18 @@ impl App { self.input_mode = CommandInputMode::Composer; self.command_completion_selected = None; } - if let Some(Method::ListRewindTargets) = result.method.as_ref() { + let mut method = result.method; + if let Some(Method::Compact { .. }) = method { + method = Some(Method::Compact { + command: self.next_command_envelope(), + }); + } + if let Some(Method::ListRewindTargets) = method.as_ref() { self.completion = None; self.rewind_picker = None; self.rewind_request_pending = true; } - result.method + method } fn push_command_diagnostic(&mut self, message: impl Into) { @@ -2761,8 +2787,8 @@ mod rewind_refresh_tests { }); assert!(!blocks_contain(&app, "stale tail after rewind")); - app.handle_worker_event(Event::Status { - status: WorkerStatus::Idle, + app.handle_worker_event(Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }); app.handle_worker_event(Event::TextDelta { text: "new live tail after status".into(), @@ -3478,7 +3504,7 @@ mod completion_flow_tests { let mut app = App::new("test".into()); app.set_worker_status(WorkerStatus::Paused); - assert!(matches!(app.submit_input(), Some(Method::Resume))); + assert!(matches!(app.submit_input(), Some(Method::Resume { .. }))); assert_eq!(app.queued_input_count(), 0); } @@ -3533,7 +3559,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: public_session(vec![session_start_value]), - status: WorkerStatus::Running, + state: WorkerStatus::Running.into(), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -3551,8 +3577,8 @@ mod completion_flow_tests { code: ErrorCode::ProviderError, message: "provider unavailable".into(), }); - app.handle_worker_event(Event::Status { - status: WorkerStatus::Idle, + app.handle_worker_event(Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }); let live_errors = app @@ -3577,7 +3603,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: public_session(vec![serde_json::to_value(run_errored).unwrap()]), - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -3641,7 +3667,7 @@ mod completion_flow_tests { pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - status: WorkerStatus::Running, + state: WorkerStatus::Running.into(), in_flight: InFlightSnapshot { blocks: vec![ InFlightBlock::Thinking { @@ -3968,7 +3994,7 @@ mod completion_flow_tests { pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -4020,7 +4046,7 @@ mod completion_flow_tests { pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: vec![InternalWorkerSnapshot { worker: InternalWorkerRef { @@ -4194,7 +4220,7 @@ mod completion_flow_tests { entries: Vec::new(), }, greeting, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -4393,7 +4419,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: public_session(assistant_item_entries), - status: WorkerStatus::Running, + state: WorkerStatus::Running.into(), in_flight: Default::default(), internal_workers: Vec::new(), }); diff --git a/crates/tui/src/command.rs b/crates/tui/src/command.rs index 44004a30..ba24299e 100644 --- a/crates/tui/src/command.rs +++ b/crates/tui/src/command.rs @@ -409,7 +409,12 @@ fn compact_command(invocation: CommandInvocation<'_>) -> CommandExecution { let _ = invocation.environment; let _ = invocation.args.raw(); CommandExecution { - method: Some(Method::Compact), + method: Some(Method::Compact { + command: protocol::WorkerCommandEnvelope::for_snapshot( + 0, + &protocol::WorkerStateSnapshot::initial(1), + ), + }), diagnostics: vec![CommandDiagnostic::new("compact requested")], exit_command_mode: true, clear_input: true, @@ -483,7 +488,7 @@ mod tests { fn compact_command_returns_compact_method_not_run() { let registry = CommandRegistry::builtins(); let result = registry.dispatch("compact", &env()); - assert!(matches!(result.method, Some(Method::Compact))); + assert!(matches!(result.method, Some(Method::Compact { .. }))); assert!(result.exit_command_mode); assert!(result.clear_input); assert!(result.diagnostics[0].message.contains("compact requested")); diff --git a/crates/tui/src/console/mod.rs b/crates/tui/src/console/mod.rs index a7ff703e..af041066 100644 --- a/crates/tui/src/console/mod.rs +++ b/crates/tui/src/console/mod.rs @@ -572,7 +572,7 @@ async fn run_e2e_rewind_fixture( pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), greeting: Greeting { worker_name: worker_name.clone(), cwd: workspace_root.display().to_string(), @@ -1438,13 +1438,15 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option { WorkerStatus::Running | WorkerStatus::Paused ) { app.shutdown_confirm = None; - return Some(Method::Cancel); + let command = app.next_command_envelope(); + return Some(Method::Cancel { command }); } if let Some(pressed_at) = app.shutdown_confirm && pressed_at.elapsed() < CONFIRM_TIMEOUT { app.shutdown_confirm = None; - return Some(Method::Shutdown); + let command = app.next_command_envelope(); + return Some(Method::Shutdown { command }); } app.shutdown_confirm = Some(std::time::Instant::now()); app.flash_actionbar_notice( @@ -1460,7 +1462,8 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option { /// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running). fn handle_pause_or_quit(app: &mut App) -> Option { if app.worker_status == WorkerStatus::Running { - return Some(Method::Pause); + let command = app.next_command_envelope(); + return Some(Method::Pause { command }); } if let Some(t) = app.quit_confirm && t.elapsed() < CONFIRM_TIMEOUT @@ -2090,7 +2093,7 @@ mod tests { &mut app, KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL), ), - Some(Method::Pause) + Some(Method::Pause { .. }) )); assert_eq!(app.queued_input_count(), 1); @@ -2100,7 +2103,7 @@ mod tests { &mut app, KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), ), - Some(Method::Cancel) + Some(Method::Cancel { .. }) )); assert_eq!(app.queued_input_count(), 1); } @@ -2114,7 +2117,7 @@ mod tests { &mut app, KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), ); - assert!(matches!(cancel, Some(Method::Cancel))); + assert!(matches!(cancel, Some(Method::Cancel { .. }))); } #[test] @@ -2136,7 +2139,7 @@ mod tests { assert!(matches!( handle_key(&mut app, ctrl_x()), - Some(Method::Shutdown) + Some(Method::Shutdown { .. }) )); assert!(app.shutdown_confirm.is_none()); } @@ -2466,7 +2469,7 @@ mod tests { } let method = handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); - assert!(matches!(method, Some(protocol::Method::Compact))); + assert!(matches!(method, Some(protocol::Method::Compact { .. }))); assert!(!app.is_command_mode()); assert_eq!(input_text(&app), ""); assert_eq!(app.queued_input_count(), 0); @@ -2573,7 +2576,7 @@ mod tests { pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: vec![], }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -2606,7 +2609,7 @@ mod tests { pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: vec![], }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -2743,8 +2746,8 @@ mod tests { kind: protocol::InternalWorkerKind::SubWorker, }, revision: 1, - event: Box::new(Event::Status { - status: WorkerStatus::Running, + event: Box::new(Event::WorkerState { + snapshot: WorkerStatus::Running.into(), }), }); enter_command_mode(&mut app); @@ -2859,8 +2862,8 @@ mod tests { kind: protocol::InternalWorkerKind::SubWorker, }, revision: 1, - event: Box::new(Event::Status { - status: WorkerStatus::Running, + event: Box::new(Event::WorkerState { + snapshot: WorkerStatus::Running.into(), }), }); @@ -2885,8 +2888,8 @@ mod tests { kind: protocol::InternalWorkerKind::SubWorker, }, revision: 1, - event: Box::new(Event::Status { - status: WorkerStatus::Running, + event: Box::new(Event::WorkerState { + snapshot: WorkerStatus::Running.into(), }), }); handle_key(&mut app, key(KeyCode::Tab)); @@ -2902,7 +2905,7 @@ mod tests { ); assert!(first.is_none()); - assert!(matches!(second, Some(Method::Shutdown))); + assert!(matches!(second, Some(Method::Shutdown { .. }))); assert_eq!(app.worker_status, WorkerStatus::Idle); } @@ -2924,8 +2927,8 @@ mod tests { kind: protocol::InternalWorkerKind::SubWorker, }, revision: 1, - event: Box::new(Event::Status { - status: WorkerStatus::Running, + event: Box::new(Event::WorkerState { + snapshot: WorkerStatus::Running.into(), }), }); diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index c85e9f29..3c75dc2a 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -15,18 +15,6 @@ use std::fmt; use std::sync::Arc; use workdir::WorkdirSessionHandle; -/// Current execution-side run state for a Worker. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WorkerExecutionRunState { - #[default] - Stopped, - Idle, - Busy, - Rejected, - Errored, -} - /// Execution operation that produced a result. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -55,7 +43,8 @@ pub struct WorkerSubmissionAck { pub struct WorkerExecutionResult { pub operation: WorkerExecutionOperation, pub outcome: WorkerExecutionOutcome, - pub run_state: WorkerExecutionRunState, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub message: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -74,22 +63,23 @@ pub enum WorkerExecutionOutcome { } impl WorkerExecutionResult { - pub fn accepted( - operation: WorkerExecutionOperation, - run_state: WorkerExecutionRunState, - ) -> Self { + pub fn accepted(operation: WorkerExecutionOperation) -> Self { Self { operation, outcome: WorkerExecutionOutcome::Accepted, - run_state, + worker_state: None, message: None, submission: None, } } + pub fn with_worker_state(mut self, worker_state: protocol::WorkerStateSnapshot) -> Self { + self.worker_state = Some(worker_state); + self + } + pub fn accepted_submission( operation: WorkerExecutionOperation, - run_state: WorkerExecutionRunState, submission_request_id: impl Into, submission_id: impl Into, disposition: protocol::SubmissionDisposition, @@ -97,7 +87,7 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Accepted, - run_state, + worker_state: None, message: None, submission: Some(WorkerSubmissionAck { submission_request_id: submission_request_id.into(), @@ -111,7 +101,7 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Busy, - run_state: WorkerExecutionRunState::Busy, + worker_state: None, message: Some(message.into()), submission: None, } @@ -121,7 +111,7 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Rejected, - run_state: WorkerExecutionRunState::Stopped, + worker_state: None, message: Some(message.into()), submission: None, } @@ -131,7 +121,7 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Errored, - run_state: WorkerExecutionRunState::Errored, + worker_state: None, message: Some(message.into()), submission: None, } @@ -141,7 +131,7 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Unsupported, - run_state: WorkerExecutionRunState::Stopped, + worker_state: None, message: Some(message.into()), submission: None, } @@ -280,7 +270,6 @@ pub struct WorkerExecutionRestoreRequest { pub enum WorkerExecutionSpawnResult { Connected { handle: WorkerExecutionHandle, - run_state: WorkerExecutionRunState, working_directory: Option, }, Rejected(WorkerExecutionResult), @@ -290,12 +279,10 @@ pub enum WorkerExecutionSpawnResult { impl WorkerExecutionSpawnResult { pub fn connected( handle: WorkerExecutionHandle, - run_state: WorkerExecutionRunState, working_directory: Option, ) -> Self { Self::Connected { handle, - run_state, working_directory, } } @@ -623,7 +610,6 @@ mod tests { fn submission_ack_survives_json_round_trip() { let result = WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, "request-1", "submission-1", protocol::SubmissionDisposition::Started, diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 6bcc5adf..7762ac86 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -2206,8 +2206,8 @@ mod tests { }; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, - WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState, - WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest, + WorkerExecutionSpawnResult, }; use crate::management::RuntimeOptions; use axum::body::to_bytes; @@ -2979,7 +2979,6 @@ mod tests { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -2993,7 +2992,6 @@ mod tests { ) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request.previous_working_directory, } } @@ -3006,24 +3004,17 @@ mod tests { if let Some(submission_id) = input.submission_request_id { WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, submission_id.clone(), submission_id, protocol::SubmissionDisposition::Started, ) } else { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Input) } } fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Stop, - WorkerExecutionRunState::Stopped, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) } } @@ -3295,8 +3286,7 @@ mod ws_tests { }; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, - WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionSpawnRequest, - WorkerExecutionSpawnResult, + WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, }; use crate::management::RuntimeOptions; use futures::{SinkExt, StreamExt}; @@ -3316,7 +3306,6 @@ mod ws_tests { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -3332,16 +3321,12 @@ mod ws_tests { if let Some(submission_id) = input.submission_request_id { WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, submission_id.clone(), submission_id, protocol::SubmissionDisposition::Started, ) } else { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Input) } } @@ -3350,10 +3335,7 @@ mod ws_tests { _handle: &WorkerExecutionHandle, _method: protocol::Method, ) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::ProtocolMethod, - WorkerExecutionRunState::Idle, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::ProtocolMethod) } } @@ -3564,16 +3546,16 @@ mod ws_tests { runtime .observe_worker_event( &other.worker_ref, - protocol::Event::Status { - status: protocol::WorkerStatus::Running, + protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }, ) .unwrap(); runtime .observe_worker_event( &worker_ref, - protocol::Event::Status { - status: protocol::WorkerStatus::Running, + protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }, ) .unwrap(); diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 9701aae1..808788f4 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -13,8 +13,8 @@ use crate::error::RuntimeError; use crate::execution::WorkerExecutionRestoreRequest; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionHandle, - WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionRunState, - WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionSpawnRequest, + WorkerExecutionSpawnResult, }; #[cfg(feature = "fs-store")] use crate::fs_store::{ @@ -725,12 +725,11 @@ impl Runtime { }; let spawn_result = backend.spawn_worker(spawn_request); - let (handle, run_state, working_directory) = match spawn_result { + let (handle, working_directory) = match spawn_result { WorkerExecutionSpawnResult::Connected { handle, - run_state, working_directory, - } => (handle, run_state, working_directory), + } => (handle, working_directory), WorkerExecutionSpawnResult::Rejected(result) | WorkerExecutionSpawnResult::Errored(result) => { self.rollback_failed_create(&worker_ref)?; @@ -785,11 +784,10 @@ impl Runtime { result, }); } - let initial_run_state = dispatch_result.run_state; let detail = self.commit_created_worker( &worker_ref, handle, - initial_run_state, + WorkerStatus::Running, working_directory, dispatch_result, )?; @@ -799,9 +797,9 @@ impl Runtime { self.commit_created_worker( &worker_ref, handle, - run_state, + WorkerStatus::Idle, working_directory, - WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn, run_state), + WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn), ) } } @@ -1086,13 +1084,12 @@ impl Runtime { match backend.restore_worker(request) { WorkerExecutionSpawnResult::Connected { handle, - run_state, working_directory, } => { self.commit_restored_worker_execution( worker_ref, handle, - run_state, + WorkerStatus::Idle, working_directory, )?; self.worker_detail(worker_ref) @@ -1222,7 +1219,19 @@ impl Runtime { let mut state = self.lock()?; state.ensure_running()?; let worker = state.worker_mut(worker_ref)?; - worker.status = worker_status_from_run_state(dispatch_result.run_state); + if let Some(snapshot) = dispatch_result.worker_state.as_ref() { + worker.status = match snapshot.catalog_status() { + protocol::WorkerStatus::Idle => WorkerStatus::Idle, + protocol::WorkerStatus::Running => WorkerStatus::Running, + protocol::WorkerStatus::Paused => WorkerStatus::Paused, + protocol::WorkerStatus::Stopped => WorkerStatus::Stopped, + }; + } else if matches!( + submission.as_ref().map(|ack| ack.disposition), + Some(protocol::SubmissionDisposition::Started) + ) { + worker.status = WorkerStatus::Running; + } let status = worker.status; #[cfg(feature = "ws-server")] if let Some(payload) = input_protocol_event(&input) { @@ -1431,7 +1440,7 @@ impl Runtime { let entries = self.worker_completions(worker_ref, kind, &prefix)?; return Ok(vec![Event::Completions { kind, entries }]); } - if matches!(&method, Method::Shutdown) { + if matches!(&method, Method::Shutdown { .. }) { self.stop_worker(worker_ref, Some("worker protocol shutdown".to_string()))?; return Ok(Vec::new()); } @@ -1481,7 +1490,7 @@ impl Runtime { &self, worker_ref: &WorkerRef, handle: WorkerExecutionHandle, - run_state: WorkerExecutionRunState, + status: WorkerStatus, working_directory: Option, _result: WorkerExecutionResult, ) -> Result { @@ -1490,7 +1499,7 @@ impl Runtime { let worker = state.worker_mut(worker_ref)?; worker.execution_handle = Some(handle); worker.execution_bound = true; - worker.status = worker_status_from_run_state(run_state); + worker.status = status; worker.restore_intent = restore_intent_for_status(worker.status); worker.working_directory = working_directory; worker.detail() @@ -1518,16 +1527,28 @@ impl Runtime { worker_ref: &WorkerRef, result: WorkerExecutionResult, ) -> Result<(), RuntimeError> { - let mut state = self.lock()?; - if result.is_accepted() { - let status = worker_status_from_run_state(result.run_state); - let worker = state.worker_mut(worker_ref)?; - worker.status = status; - worker.restore_intent = restore_intent_for_status(status); - state.publish_worker_upsert(worker_ref.worker_id)?; - state.persist_runtime_snapshot()?; - state.persist_worker(&worker_ref.worker_id)?; + // Accepted dispatch without a state snapshot is transport evidence only; + // the revisioned protocol stream remains live authority. Test/detached + // backends may return an exact full snapshot as their acknowledgement. + if !result.is_accepted() { + return Ok(()); } + let Some(snapshot) = result.worker_state else { + return Ok(()); + }; + let status = match snapshot.catalog_status() { + protocol::WorkerStatus::Idle => WorkerStatus::Idle, + protocol::WorkerStatus::Running => WorkerStatus::Running, + protocol::WorkerStatus::Paused => WorkerStatus::Paused, + protocol::WorkerStatus::Stopped => WorkerStatus::Stopped, + }; + let mut state = self.lock()?; + let worker = state.worker_mut(worker_ref)?; + worker.status = status; + worker.restore_intent = restore_intent_for_status(status); + state.publish_worker_upsert(worker_ref.worker_id)?; + state.persist_runtime_snapshot()?; + state.persist_worker(&worker_ref.worker_id)?; Ok(()) } @@ -1730,7 +1751,7 @@ impl Runtime { context_window: 0, context_tokens: 0, }, - status: protocol::WorkerStatus::Idle, + state: protocol::WorkerStateSnapshot::initial(1), in_flight: protocol::InFlightSnapshot { blocks: Vec::new(), commands: Vec::new(), @@ -1968,12 +1989,11 @@ impl Runtime { match backend.restore_worker(request) { WorkerExecutionSpawnResult::Connected { handle, - run_state, working_directory, } => self.commit_restored_worker_execution( &candidate.worker_ref, handle, - run_state, + WorkerStatus::Idle, working_directory, )?, WorkerExecutionSpawnResult::Rejected(result) @@ -1990,7 +2010,7 @@ impl Runtime { &self, worker_ref: &WorkerRef, handle: WorkerExecutionHandle, - run_state: WorkerExecutionRunState, + status: WorkerStatus, working_directory: Option, ) -> Result<(), RuntimeError> { let mut state = self.lock()?; @@ -1999,7 +2019,7 @@ impl Runtime { let worker = state.worker_mut(worker_ref)?; worker.execution_handle = Some(handle); worker.execution_bound = true; - worker.status = worker_status_from_run_state(run_state); + worker.status = status; worker.restore_intent = restore_intent_for_status(worker.status); worker.working_directory = working_directory; } @@ -2867,7 +2887,7 @@ impl RuntimeState { ) { match event { protocol::Event::Snapshot { - status, + state, internal_workers, .. } => { @@ -2875,7 +2895,7 @@ impl RuntimeState { statuses.insert( worker.session_id.clone(), InternalWorkerActivity { - status: *status, + status: state.catalog_status(), parent_session_id: worker.parent_session_id.clone(), }, ); @@ -2888,26 +2908,17 @@ impl RuntimeState { event, .. } => Self::project_internal_worker_event(statuses, nested_worker, event), - protocol::Event::Status { status } => { - statuses.insert( - worker.session_id.clone(), - InternalWorkerActivity { - status: *status, - parent_session_id: worker.parent_session_id.clone(), + protocol::Event::WorkerState { snapshot } + | protocol::Event::CommandAcknowledged { + acknowledgement: + protocol::WorkerCommandAcknowledgement { + state: snapshot, .. }, - ); - } - protocol::Event::RunEnd { result } => { - let status = match result { - protocol::RunResult::Paused => protocol::WorkerStatus::Paused, - protocol::RunResult::Finished - | protocol::RunResult::LimitReached - | protocol::RunResult::RolledBack => protocol::WorkerStatus::Idle, - }; + } => { statuses.insert( worker.session_id.clone(), InternalWorkerActivity { - status, + status: snapshot.catalog_status(), parent_session_id: worker.parent_session_id.clone(), }, ); @@ -2963,28 +2974,21 @@ impl RuntimeState { return false; }; let next_status = match event { - protocol::Event::Status { - status: protocol::WorkerStatus::Running, - } => Some(WorkerStatus::Running), - protocol::Event::Status { - status: protocol::WorkerStatus::Idle, - } => Some(WorkerStatus::Idle), - protocol::Event::Status { - status: protocol::WorkerStatus::Paused, - } => Some(WorkerStatus::Paused), - protocol::Event::Snapshot { status, .. } => match status { - protocol::WorkerStatus::Running => Some(WorkerStatus::Running), - protocol::WorkerStatus::Idle => Some(WorkerStatus::Idle), - protocol::WorkerStatus::Paused => Some(WorkerStatus::Paused), - protocol::WorkerStatus::Stopped => Some(WorkerStatus::Stopped), - }, - protocol::Event::RunEnd { result } => match result { - protocol::RunResult::Finished | protocol::RunResult::RolledBack => { - Some(WorkerStatus::Idle) - } - protocol::RunResult::Paused => Some(WorkerStatus::Paused), - protocol::RunResult::LimitReached => Some(WorkerStatus::Idle), - }, + protocol::Event::WorkerState { snapshot } + | protocol::Event::Snapshot { + state: snapshot, .. + } + | protocol::Event::CommandAcknowledged { + acknowledgement: + protocol::WorkerCommandAcknowledgement { + state: snapshot, .. + }, + } => Some(match snapshot.catalog_status() { + protocol::WorkerStatus::Idle => WorkerStatus::Idle, + protocol::WorkerStatus::Running => WorkerStatus::Running, + protocol::WorkerStatus::Paused => WorkerStatus::Paused, + protocol::WorkerStatus::Stopped => WorkerStatus::Stopped, + }), _ => None, }; if let Some(next_status) = next_status { @@ -3081,16 +3085,6 @@ fn restore_intent_for_status(status: WorkerStatus) -> WorkerRestoreIntent { } } -fn worker_status_from_run_state(run_state: WorkerExecutionRunState) -> WorkerStatus { - match run_state { - WorkerExecutionRunState::Idle => WorkerStatus::Idle, - WorkerExecutionRunState::Busy => WorkerStatus::Running, - WorkerExecutionRunState::Stopped - | WorkerExecutionRunState::Rejected - | WorkerExecutionRunState::Errored => WorkerStatus::Stopped, - } -} - fn repository_resource_error(error: BackendResourceError) -> RuntimeError { let (code, message) = match error { BackendResourceError::Expired => ( @@ -3304,7 +3298,7 @@ mod tests { }; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionHandle, - WorkerExecutionRestoreRequest, WorkerExecutionRunState, + WorkerExecutionRestoreRequest, }; use crate::working_directory::WorkingDirectoryDiagnostic; use async_trait::async_trait; @@ -3313,6 +3307,14 @@ mod tests { use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; + fn test_command() -> protocol::WorkerCommandEnvelope { + protocol::WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 1, + expected_worker_state_revision: 0, + } + } + #[test] fn repository_resource_failures_keep_typed_credential_diagnostics() { let cases = [ @@ -3359,7 +3361,9 @@ mod tests { protocol::Event::InternalWorker { worker, revision: 1, - event: Box::new(protocol::Event::Status { status }), + event: Box::new(protocol::Event::WorkerState { + snapshot: status.into(), + }), } } @@ -3452,7 +3456,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: protocol::WorkerStatus::Idle, + state: protocol::WorkerStatus::Idle.into(), in_flight: protocol::InFlightSnapshot::default(), internal_workers: Vec::new(), }; @@ -3967,7 +3971,6 @@ mod tests { .insert(request.worker_ref.worker_id.clone(), request.context); WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -3997,7 +4000,6 @@ mod tests { .insert(request.worker_ref.worker_id.clone(), request.context); WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -4020,7 +4022,6 @@ mod tests { .unwrap_or_else(|| { WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, "request-test", "test-submission", protocol::SubmissionDisposition::Started, @@ -4038,17 +4039,11 @@ mod tests { } fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Stop, - WorkerExecutionRunState::Stopped, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) } fn cancel_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Cancel, - WorkerExecutionRunState::Stopped, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Cancel) } #[cfg(feature = "ws-server")] @@ -4374,7 +4369,9 @@ mod tests { .send_protocol_method_scoped( &scope("workspace-a", "server-a"), &workspace_b.worker_ref, - Method::Shutdown, + Method::Shutdown { + command: test_command(), + }, ) .unwrap_err(); assert!(matches!( @@ -4722,11 +4719,10 @@ mod tests { } #[test] - fn create_worker_uses_committed_input_ack_run_state() { + fn create_worker_uses_started_submission_ack_for_initial_running_status() { let (runtime, backend) = runtime_and_backend(); backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, "request-test", "test-submission", protocol::SubmissionDisposition::Started, @@ -4736,7 +4732,7 @@ mod tests { let detail = runtime.create_worker(request).unwrap(); - assert_eq!(detail.status, WorkerStatus::Idle); + assert_eq!(detail.status, WorkerStatus::Running); } #[test] @@ -4745,7 +4741,6 @@ mod tests { backend.preserve_commit_ack_submission_id(); backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, "request-test", "forged-submission", protocol::SubmissionDisposition::Started, @@ -4770,7 +4765,6 @@ mod tests { let (runtime, backend) = runtime_and_backend(); backend.set_dispatch_result(WorkerExecutionResult::accepted( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, )); let mut request = task_request("missing initial input commit ack"); request.initial_input = Some(WorkerInput::user("start the ticket")); @@ -4898,7 +4892,7 @@ mod tests { context_window: 128, context_tokens: 64, }, - status: protocol::WorkerStatus::Running, + state: protocol::WorkerStatus::Running.into(), in_flight: protocol::InFlightSnapshot { blocks: Vec::new(), commands: Vec::new(), @@ -4914,13 +4908,13 @@ mod tests { protocol::Event::Snapshot { session, greeting, - status, + state, .. } => { assert_eq!(session.entries.len(), 1); assert_eq!(session.entries[0].entry_id, "restored-log-entry"); assert_eq!(greeting.worker_name, "live-worker"); - assert_eq!(status, protocol::WorkerStatus::Running); + assert_eq!(state.catalog_status(), protocol::WorkerStatus::Running); } other => panic!("expected snapshot, got {other:?}"), } @@ -4936,7 +4930,6 @@ mod tests { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -4951,7 +4944,6 @@ mod tests { ) -> WorkerExecutionResult { WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, "request-test", input .submission_request_id @@ -4993,7 +4985,12 @@ mod tests { .unwrap(); runtime - .send_protocol_method(&detail.worker_ref, Method::Shutdown) + .send_protocol_method( + &detail.worker_ref, + Method::Shutdown { + command: test_command(), + }, + ) .unwrap(); assert_eq!( @@ -5009,7 +5006,12 @@ mod tests { .create_worker(task_request("restore explicitly")) .unwrap(); runtime - .send_protocol_method(&detail.worker_ref, Method::Shutdown) + .send_protocol_method( + &detail.worker_ref, + Method::Shutdown { + command: test_command(), + }, + ) .unwrap(); assert!(matches!( @@ -5027,7 +5029,7 @@ mod tests { assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]); assert_eq!( runtime.worker_detail(&detail.worker_ref).unwrap().status, - WorkerStatus::Idle + WorkerStatus::Running ); } diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 5225dd8b..6c414874 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -10,8 +10,8 @@ use std::collections::HashMap; use std::future::Future; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, mpsc}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock, mpsc}; use std::time::Duration; use crate::auth::{ @@ -25,8 +25,8 @@ use crate::catalog::{ }; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, - WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState, - WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest, + WorkerExecutionSpawnResult, }; use crate::identity::WorkerRef; use crate::interaction::{WorkerInput, WorkerInputKind}; @@ -38,7 +38,26 @@ use crate::working_directory::{ WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer, }; use async_trait::async_trait; -use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus}; +use protocol::{Event, Method, Segment, WorkerCommandEnvelope, WorkerStatus}; + +static NEXT_INTERNAL_COMMAND_ID: AtomicU64 = AtomicU64::new(1); + +fn next_internal_command( + state: &RwLock, +) -> Result { + let snapshot = state + .read() + .map_err(|_| "worker state lock is poisoned".to_string())? + .clone(); + let floor = snapshot.last_command_id.saturating_add(1); + let command_id = NEXT_INTERNAL_COMMAND_ID + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + Some(current.max(floor).saturating_add(1)) + }) + .unwrap_or(floor) + .max(floor); + Ok(WorkerCommandEnvelope::for_snapshot(command_id, &snapshot)) +} use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore}; #[cfg(test)] use session_store::{FsStore, FsWorkerStore}; @@ -172,7 +191,7 @@ impl WorkerObservationProvider for RuntimeGrantedWorkerObservationProvider { }, display_name: grant.worker_id.clone(), relation: "granted_peer".to_string(), - status: format!("{:?}", state.get_status()).to_lowercase(), + status: format!("{:?}", state.catalog_status()).to_lowercase(), }); } subjects.sort_by(|left, right| left.subject.cmp(&right.subject)); @@ -1174,10 +1193,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } } +#[derive(Clone)] struct RuntimeWorkerExecution { handle: WorkerHandle, shutdown: Arc>>, busy: Arc, + worker_state: Arc>, workspace_client: Option>, } @@ -1276,6 +1297,7 @@ where ( WorkerHandle, Arc, + Arc>, Option>, ), WorkerExecutionResult, @@ -1302,6 +1324,7 @@ where ( execution.handle.clone(), execution.busy.clone(), + execution.worker_state.clone(), execution.workspace_client.clone(), ) }) @@ -1318,7 +1341,6 @@ where operation: WorkerExecutionOperation, worker: WorkerHandle, method: Method, - accepted_run_state: WorkerExecutionRunState, ) -> WorkerExecutionResult { self.run_on_adapter_runtime(async move { worker @@ -1326,7 +1348,7 @@ where .await .map_err(|err| format!("failed to send Worker method: {err}")) }) - .map(|_| WorkerExecutionResult::accepted(operation, accepted_run_state)) + .map(|_| WorkerExecutionResult::accepted(operation)) .unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message)) } @@ -1336,7 +1358,6 @@ where worker: WorkerHandle, method: Method, submission_request_id: String, - accepted_run_state: WorkerExecutionRunState, ) -> WorkerExecutionResult { let request_id = submission_request_id.clone(); self.run_on_adapter_runtime(async move { @@ -1395,7 +1416,6 @@ where .map(|(submission_id, disposition)| { WorkerExecutionResult::accepted_submission( operation, - accepted_run_state, submission_request_id, submission_id, disposition, @@ -1415,38 +1435,45 @@ where workspace_client: Option>, ) -> WorkerExecutionSpawnResult { let busy = Arc::new(AtomicBool::new(false)); + let worker_state = Arc::new(RwLock::new(handle.shared_state.snapshot())); #[cfg(feature = "ws-server")] { let streams = subscribe_worker_protocol_session(&handle); let mut events = streams.events; let mut entry_events = streams.log_entries; let bridge_busy = busy.clone(); + let bridge_worker_state = worker_state.clone(); if let Err(message) = self.spawn_on_adapter_runtime(async move { loop { tokio::select! { event = events.recv() => { match event { Ok(event) => { - let next_busy = match &event { - Event::InvokeStart { .. } - | Event::Status { - status: WorkerStatus::Running, - } => Some(true), - Event::RunEnd { .. } - | Event::Error { - code: ErrorCode::NotPaused, - .. + let next_state = match &event { + Event::WorkerState { snapshot } + | Event::Snapshot { state: snapshot, .. } => { + Some(snapshot.clone()) } - | Event::Status { - status: - WorkerStatus::Idle - | WorkerStatus::Paused - | WorkerStatus::Stopped, + Event::CommandAcknowledged { acknowledgement } => { + Some(acknowledgement.state.clone()) } - | Event::Shutdown => Some(false), _ => None, }; + let next_busy = next_state + .as_ref() + .map(worker_state_is_executing) + .or_else(|| matches!(event, Event::Shutdown).then_some(false)); let _ = bridge_context.publish_protocol_event(event); + if let Some(next_state) = next_state { + if let Ok(mut current) = bridge_worker_state.write() { + if next_state.execution_generation > current.execution_generation + || (next_state.execution_generation == current.execution_generation + && next_state.revision >= current.revision) + { + *current = next_state; + } + } + } if let Some(next_busy) = next_busy { bridge_busy.store(next_busy, Ordering::SeqCst); } @@ -1494,13 +1521,13 @@ where handle, shutdown, busy, + worker_state, workspace_client, }, ); WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: working_directory.map(|binding| binding.status()), } } @@ -1516,6 +1543,17 @@ impl Drop for WorkerRuntimeExecutionBackend { } } +fn worker_state_is_executing(snapshot: &protocol::WorkerStateSnapshot) -> bool { + matches!( + snapshot.state, + protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running + | protocol::WorkerRunState::Pausing + | protocol::WorkerRunState::Cancelling + )) | protocol::WorkerState::Busy(protocol::WorkerBusyState::Maintenance(_)) + ) +} + fn method_starts_turn(method: &Method) -> bool { matches!( method, @@ -1523,41 +1561,17 @@ fn method_starts_turn(method: &Method) -> bool { | Method::SubmitTracked { .. } | Method::Notify { auto_run: true, .. } | Method::NotifyTracked { auto_run: true, .. } - | Method::Resume - | Method::Compact + | Method::Resume { .. } ) } fn method_can_start_turn_from_status(method: &Method, status: WorkerStatus) -> bool { match method { - Method::Resume => matches!(status, WorkerStatus::Idle | WorkerStatus::Paused), + Method::Resume { .. } => matches!(status, WorkerStatus::Idle | WorkerStatus::Paused), _ => status == WorkerStatus::Idle, } } -fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExecutionRunState { - match status { - WorkerStatus::Running => WorkerExecutionRunState::Busy, - WorkerStatus::Idle if auto_run => WorkerExecutionRunState::Busy, - WorkerStatus::Idle | WorkerStatus::Paused | WorkerStatus::Stopped => { - WorkerExecutionRunState::Idle - } - } -} - -fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState { - match method { - Method::Submit { .. } - | Method::SubmitTracked { .. } - | Method::Notify { auto_run: true, .. } - | Method::NotifyTracked { auto_run: true, .. } - | Method::Resume - | Method::Compact => WorkerExecutionRunState::Busy, - Method::Shutdown => WorkerExecutionRunState::Stopped, - _ => WorkerExecutionRunState::Idle, - } -} - impl WorkerExecutionBackend for WorkerRuntimeExecutionBackend where F: RuntimeWorkerFactory, @@ -1883,7 +1897,7 @@ where handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - let (worker, busy, _workspace_client) = match self.get_execution(handle) { + let (worker, busy, worker_state, _workspace_client) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::Input; @@ -1892,8 +1906,7 @@ where }; if input.kind == WorkerInputKind::Notify { - let status = worker.shared_state.get_status(); - let accepted_run_state = accepted_notify_run_state(status, true); + let status = worker.shared_state.catalog_status(); let claimed_here = status == WorkerStatus::Idle && busy .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) @@ -1912,7 +1925,6 @@ where operation_id: notification_request_id, }, }, - accepted_run_state, ); if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { @@ -1921,8 +1933,22 @@ where return result; } + if input.kind == WorkerInputKind::Compact { + let command = match next_internal_command(&worker_state) { + Ok(command) => command, + Err(error) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Input, error); + } + }; + return self.send_method( + WorkerExecutionOperation::Input, + worker, + Method::Compact { command }, + ); + } + let is_user_submit = input.kind == WorkerInputKind::User; - let status = worker.shared_state.get_status(); + let status = worker.shared_state.catalog_status(); let claimed_here = status == WorkerStatus::Idle && busy .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) @@ -1962,7 +1988,7 @@ where WorkerInputKind::Notify => { unreachable!("Notify input is dispatched before the turn-start busy guard") } - WorkerInputKind::Compact => (Method::Compact, None), + WorkerInputKind::Compact => unreachable!("compact input is dispatched above"), WorkerInputKind::ListRewindTargets => (Method::ListRewindTargets, None), WorkerInputKind::RegisterPeer => ( Method::RegisterPeer { @@ -1971,15 +1997,6 @@ where None, ), }; - let accepted_run_state = match method { - Method::Submit { .. } - | Method::SubmitTracked { .. } - | Method::Notify { .. } - | Method::NotifyTracked { .. } - | Method::Compact => WorkerExecutionRunState::Busy, - _ => WorkerExecutionRunState::Idle, - }; - let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle; let waits_for_submission_acceptance = submission_request_id.is_some(); let result = if waits_for_submission_acceptance { @@ -1988,20 +2005,11 @@ where worker, method, submission_request_id.expect("Submit must have a submission request id"), - accepted_run_state, ) } else { - self.send_method( - WorkerExecutionOperation::Input, - worker, - method, - accepted_run_state, - ) + self.send_method(WorkerExecutionOperation::Input, worker, method) }; - if accepted_is_idle - || (claimed_here - && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted) - { + if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { busy.store(false, Ordering::SeqCst); } result @@ -2015,7 +2023,7 @@ where content: &[u8], context: Option<&session_store::UploadedFileUploadContext>, ) -> Result { - let (worker, _, _) = self.get_execution(handle).map_err(|mut result| { + let (worker, _, _, _) = self.get_execution(handle).map_err(|mut result| { result.operation = WorkerExecutionOperation::UploadFile; result })?; @@ -2038,7 +2046,7 @@ where handle: &WorkerExecutionHandle, artifact_id: &str, ) -> WorkerExecutionResult { - let (worker, _, _) = match self.get_execution(handle) { + let (worker, _, _, _) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::DeleteUploadedFile; @@ -2046,10 +2054,7 @@ where } }; match worker.delete_uploaded_file(artifact_id) { - Ok(_) => WorkerExecutionResult::accepted( - WorkerExecutionOperation::DeleteUploadedFile, - WorkerExecutionRunState::Idle, - ), + Ok(_) => WorkerExecutionResult::accepted(WorkerExecutionOperation::DeleteUploadedFile), Err(error) => WorkerExecutionResult::rejected( WorkerExecutionOperation::DeleteUploadedFile, format!("uploaded_file_delete_rejected: {error}"), @@ -2062,7 +2067,7 @@ where handle: &WorkerExecutionHandle, method: Method, ) -> WorkerExecutionResult { - let (worker, busy, _workspace_client) = match self.get_execution(handle) { + let (worker, busy, _worker_state, _workspace_client) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::ProtocolMethod; @@ -2076,19 +2081,13 @@ where } _ => None, } { - let status = worker.shared_state.get_status(); - let accepted_run_state = accepted_notify_run_state(status, auto_run); + let status = worker.shared_state.catalog_status(); let claimed_here = status == WorkerStatus::Idle && auto_run && busy .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) .is_ok(); - let result = self.send_method( - WorkerExecutionOperation::ProtocolMethod, - worker, - method, - accepted_run_state, - ); + let result = self.send_method(WorkerExecutionOperation::ProtocolMethod, worker, method); if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { busy.store(false, Ordering::SeqCst); @@ -2098,7 +2097,7 @@ where let starts_turn = method_starts_turn(&method); if starts_turn - && (!method_can_start_turn_from_status(&method, worker.shared_state.get_status()) + && (!method_can_start_turn_from_status(&method, worker.shared_state.catalog_status()) || busy .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) .is_err()) @@ -2109,17 +2108,8 @@ where ); } - let accepted_run_state = accepted_run_state_for_method(&method); - let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle; - let result = self.send_method( - WorkerExecutionOperation::ProtocolMethod, - worker, - method, - accepted_run_state, - ); - if (starts_turn && accepted_is_idle) - || (starts_turn && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted) - { + let result = self.send_method(WorkerExecutionOperation::ProtocolMethod, worker, method); + if starts_turn && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { busy.store(false, Ordering::SeqCst); } result @@ -2137,7 +2127,7 @@ where ); } let execution = match self.workers.lock() { - Ok(mut workers) => workers.remove(handle.worker_ref()), + Ok(workers) => workers.get(handle.worker_ref()).cloned(), Err(_) => { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, @@ -2153,48 +2143,73 @@ where }; let artifact_cleanup = execution.handle.clone(); let shutdown = execution.shutdown.clone(); + let command = match next_internal_command(&execution.worker_state) { + Ok(command) => command, + Err(error) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, error); + } + }; let result = self.send_method( WorkerExecutionOperation::Stop, - execution.handle, - Method::Shutdown, - WorkerExecutionRunState::Stopped, + execution.handle.clone(), + Method::Shutdown { command }, ); if result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { return result; } - match self.run_on_adapter_runtime(async move { - let receiver = shutdown.lock().await.take(); - if let Some(receiver) = receiver { - receiver - .await - .map_err(|_| "Worker shutdown completion channel closed".to_string())?; + let shutdown_wait = self.run_on_adapter_runtime(async move { + let mut guard = shutdown.lock().await; + let Some(mut receiver) = guard.take() else { + return Ok(()); + }; + match tokio::time::timeout(Duration::from_secs(5), &mut receiver).await { + Ok(Ok(())) => Ok(()), + Ok(Err(_)) => Err("Worker shutdown completion channel closed".to_string()), + Err(_) => { + *guard = Some(receiver); + Err("Worker shutdown confirmation timed out; stop remains retryable".into()) + } } - Ok(()) - }) { - Ok(()) => match artifact_cleanup.delete_uncommitted_uploaded_files() { - Ok(_) => result, - Err(error) => WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - format!("uploaded_file_cleanup_failed: {error}"), - ), - }, - Err(message) => WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message), + }); + if let Err(message) = shutdown_wait { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + if let Err(error) = artifact_cleanup.delete_uncommitted_uploaded_files() { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + format!("uploaded_file_cleanup_failed: {error}"), + ); + } + match self.workers.lock() { + Ok(mut workers) => { + workers.remove(handle.worker_ref()); + result + } + Err(_) => WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "worker adapter registry lock is poisoned after shutdown", + ), } } fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - let (worker, _busy, _workspace_client) = match self.get_execution(handle) { + let (worker, _busy, worker_state, _workspace_client) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::Cancel; return result; } }; + let command = match next_internal_command(&worker_state) { + Ok(command) => command, + Err(error) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Cancel, error); + } + }; self.send_method( WorkerExecutionOperation::Cancel, worker, - Method::Cancel, - WorkerExecutionRunState::Idle, + Method::Cancel { command }, ) } @@ -2259,6 +2274,29 @@ mod tests { use manifest::{Scope, WorkerManifest}; use session_store::{LogEntry, WorkerMetadataStore}; + fn test_command() -> WorkerCommandEnvelope { + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 1, + expected_worker_state_revision: 0, + } + } + + fn adapter_command( + backend: &WorkerRuntimeExecutionBackend, + worker_ref: &WorkerRef, + ) -> WorkerCommandEnvelope { + let workers = backend.workers.lock().unwrap(); + let state = workers + .get(worker_ref) + .expect("worker execution") + .worker_state + .read() + .unwrap() + .clone(); + WorkerCommandEnvelope::for_snapshot(state.last_command_id.saturating_add(1), &state) + } + #[test] fn workspace_prompt_projection_notification_advances_shared_cache() { let cache = WorkspacePromptProjectionCache::default(); @@ -2406,41 +2444,39 @@ mod tests { } #[test] - fn notify_run_state_allows_running_worker_inbox_delivery() { - assert_eq!( - accepted_notify_run_state(WorkerStatus::Running, true), - WorkerExecutionRunState::Busy - ); - assert_eq!( - accepted_notify_run_state(WorkerStatus::Idle, true), - WorkerExecutionRunState::Busy - ); - assert_eq!( - accepted_notify_run_state(WorkerStatus::Idle, false), - WorkerExecutionRunState::Idle - ); - assert_eq!( - accepted_notify_run_state(WorkerStatus::Paused, true), - WorkerExecutionRunState::Idle - ); + fn compact_is_maintenance_not_a_turn_start() { + assert!(!method_starts_turn(&Method::Compact { + command: test_command(), + })); + assert!(method_starts_turn(&Method::Resume { + command: test_command(), + })); } #[test] fn resume_turn_claim_accepts_paused_and_idle_but_not_running_status() { assert!(method_can_start_turn_from_status( - &Method::Resume, + &Method::Resume { + command: test_command() + }, WorkerStatus::Paused )); assert!(method_can_start_turn_from_status( - &Method::Resume, + &Method::Resume { + command: test_command() + }, WorkerStatus::Idle )); assert!(!method_can_start_turn_from_status( - &Method::Resume, + &Method::Resume { + command: test_command() + }, WorkerStatus::Running )); assert!(!method_can_start_turn_from_status( - &Method::Compact, + &Method::Compact { + command: test_command() + }, WorkerStatus::Paused )); } @@ -2656,19 +2692,22 @@ mod tests { let observed = { let workers = backend.workers.lock().unwrap(); let execution = workers.get(worker_ref).expect("live Worker execution"); + let projected = execution.worker_state.read().unwrap().catalog_status(); ( - execution.handle.shared_state.get_status(), + execution.handle.shared_state.catalog_status(), + projected, execution.busy.load(Ordering::SeqCst), ) }; - if observed == (expected_status, expected_busy) { + if observed == (expected_status, expected_status, expected_busy) { return; } assert!( std::time::Instant::now() < deadline, - "timed out waiting for adapter state {expected_status:?}, busy={expected_busy}; last observed status={:?}, busy={}", + "timed out waiting for adapter state {expected_status:?}, busy={expected_busy}; last observed controller={:?}, projected={:?}, busy={}", observed.0, observed.1, + observed.2, ); std::thread::sleep(Duration::from_millis(10)); } @@ -3169,13 +3208,19 @@ mod tests { .expect("in-process restore must not bind the overlong Unix socket path"); assert_eq!( - controller.handle.shared_state.get_status(), + controller.handle.shared_state.catalog_status(), WorkerStatus::Idle ); assert!(!socket_path.exists()); assert!(run_dir.join("worker.out.log").is_file()); assert!(run_dir.join("worker.err.log").is_file()); - controller.handle.send(Method::Shutdown).await.unwrap(); + controller + .handle + .send(Method::Shutdown { + command: test_command(), + }) + .await + .unwrap(); if let Some(receiver) = controller.shutdown.lock().await.take() { receiver.await.unwrap(); } @@ -3289,7 +3334,9 @@ mod tests { backend .run_on_adapter_runtime(async move { handle - .send(Method::Shutdown) + .send(Method::Shutdown { + command: test_command(), + }) .await .map_err(|error| error.to_string())?; if let Some(receiver) = shutdown.lock().await.take() { @@ -3614,6 +3661,7 @@ mod tests { #[test] #[cfg(feature = "ws-server")] + #[serial_test::serial(worker_allocation)] fn adapter_resumes_paused_turn_once_and_preserves_idle_not_paused_error() { let hanging_events = || simple_text_events().into_iter().take(2).collect::>(); let client = MockClient::sequential(vec![ @@ -3649,7 +3697,12 @@ mod tests { wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true); let running_resume = runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect_err("Resume while Running must be rejected"); assert!( running_resume @@ -3659,17 +3712,32 @@ mod tests { ); runtime - .send_protocol_method(&detail.worker_ref, Method::Pause) + .send_protocol_method( + &detail.worker_ref, + Method::Pause { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect("pause initial turn"); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false); runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect("resume paused turn"); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true); let duplicate_resume = runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect_err("duplicate Resume must be rejected"); assert!( duplicate_resume @@ -3679,17 +3747,32 @@ mod tests { ); runtime - .send_protocol_method(&detail.worker_ref, Method::Pause) + .send_protocol_method( + &detail.worker_ref, + Method::Pause { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect("pause resumed turn"); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false); runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect("resume paused turn a second time"); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false); assert_eq!(call_count.load(Ordering::SeqCst), 3); runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect("Idle Resume preserves controller NotPaused semantics"); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false); let events = runtime @@ -3698,10 +3781,10 @@ mod tests { assert!(events.iter().any(|event| { matches!( &event.payload, - Event::Error { - code: protocol::ErrorCode::NotPaused, - .. - } + Event::CommandAcknowledged { acknowledgement } + if acknowledgement.command == protocol::WorkerCommandKind::Resume + && acknowledgement.disposition + == protocol::WorkerCommandDisposition::InvalidState ) })); assert_eq!(call_count.load(Ordering::SeqCst), 3); diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 4a493a0b..98bf1be1 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -1,3 +1,4 @@ +use std::collections::VecDeque; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -28,7 +29,9 @@ use protocol::{ AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent, CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus, CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice, - ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef, WorkerStatus, + ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef, + WorkerBusyState, WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope, + WorkerCommandKind, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStatus, }; use workdir::{ CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot, @@ -138,7 +141,7 @@ impl WorkerHandle { let event = Event::Snapshot { session, greeting: self.shared_state.greeting.clone(), - status: self.shared_state.get_status(), + state: self.shared_state.snapshot(), in_flight, internal_workers: self.spawned_registry.internal_worker_snapshots(), }; @@ -178,15 +181,81 @@ impl WorkerHandle { } } +fn validate_command( + envelope: WorkerCommandEnvelope, + shared_state: &WorkerSharedState, +) -> Result<(), WorkerCommandDisposition> { + let snapshot = shared_state.snapshot(); + if envelope.expected_execution_generation != snapshot.execution_generation { + return Err(WorkerCommandDisposition::StaleExecutionGeneration); + } + if envelope.expected_worker_state_revision != snapshot.revision { + return Err(WorkerCommandDisposition::StaleWorkerStateRevision); + } + if !shared_state.accept_command_id(envelope.command_id) { + return Err(WorkerCommandDisposition::StaleCommandId); + } + Ok(()) +} + +fn acknowledge_command( + working_event_tx: &broadcast::Sender, + shared_state: &WorkerSharedState, + command_id: u64, + command: WorkerCommandKind, + disposition: WorkerCommandDisposition, +) { + let _ = working_event_tx.send(Event::CommandAcknowledged { + acknowledgement: WorkerCommandAcknowledgement { + command_id, + command, + disposition, + state: shared_state.snapshot(), + }, + }); +} + +fn reject_invalid_command_state( + working_event_tx: &broadcast::Sender, + shared_state: &WorkerSharedState, + envelope: WorkerCommandEnvelope, + command: WorkerCommandKind, +) { + acknowledge_command( + working_event_tx, + shared_state, + envelope.command_id, + command, + WorkerCommandDisposition::InvalidState, + ); +} + +async fn set_controller_state( + shared_state: &Arc, + runtime_dir: &RuntimeDir, + working_event_tx: &broadcast::Sender, + state: WorkerState, +) -> protocol::WorkerStateSnapshot { + let snapshot = shared_state.transition(state); + let _ = runtime_dir.write_status(shared_state).await; + let _ = working_event_tx.send(Event::WorkerState { + snapshot: snapshot.clone(), + }); + snapshot +} + async fn set_controller_status( shared_state: &Arc, runtime_dir: &RuntimeDir, working_event_tx: &broadcast::Sender, status: WorkerStatus, ) { - shared_state.set_status(status); - let _ = runtime_dir.write_status(shared_state).await; - let _ = working_event_tx.send(Event::Status { status }); + let state = match status { + WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle, + WorkerStatus::Running => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), + WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)), + }; + set_controller_state(shared_state, runtime_dir, working_event_tx, state).await; } async fn finish_controller_run( @@ -659,12 +728,24 @@ impl WorkerController { // === 4. Initial runtime files + WorkerSharedState + WorkerHandle + // SocketServer === let manifest_toml = toml::to_string_pretty(worker.manifest()).unwrap_or_default(); + worker + .recover_unfinished_compaction() + .await + .map_err(|error| std::io::Error::other(error.to_string()))?; let greeting = build_greeting(&worker); - let shared_state = Arc::new(WorkerSharedState::new( + let execution_generation = runtime_dir + .path() + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.parse::().ok()) + .filter(|generation| *generation > 0) + .unwrap_or(1); + let shared_state = Arc::new(WorkerSharedState::new_with_generation( worker.manifest().worker.name.clone(), worker.segment_id(), manifest_toml.clone(), greeting, + execution_generation, )); if let Some(fs_for_view) = fs_for_view { shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view)); @@ -1432,8 +1513,9 @@ async fn controller_loop( } }; - loop { - // Top-of-iteration: if an event handler staged a run, fire it + let mut deferred_methods = VecDeque::new(); + + 'controller: loop { // here so the status flip → drive_turn → finish sequence lives // in one place, regardless of which Method caused it. if let Some(run) = pending.take() { @@ -1584,9 +1666,13 @@ async fn controller_loop( continue; } - let method = match method_rx.recv().await { - Some(m) => m, - None => break, + let method = if let Some(method) = deferred_methods.pop_front() { + method + } else { + match method_rx.recv().await { + Some(method) => method, + None => break, + } }; match method { @@ -1784,7 +1870,7 @@ async fn controller_loop( expected_revision, expected_head_id, } => { - if shared_state.get_status() != WorkerStatus::Idle { + if shared_state.catalog_status() != WorkerStatus::Idle { let _ = working_event_tx.send(Event::Error { code: ErrorCode::InvalidRequest, message: "ContinuePending requires an idle Worker; Resume or Cancel a paused run first".into(), @@ -1811,88 +1897,243 @@ async fn controller_loop( } } } - Method::Resume => { - if shared_state.get_status() != WorkerStatus::Paused { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::NotPaused, - message: "Worker is not paused".into(), - }); + Method::Resume { command } => { + if let Err(disposition) = validate_command(command, &shared_state) { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Resume, + disposition, + ); continue; } + if !matches!( + shared_state.snapshot().state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) + ) { + reject_invalid_command_state( + &working_event_tx, + &shared_state, + command, + WorkerCommandKind::Resume, + ); + continue; + } + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), + ) + .await; + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Resume, + WorkerCommandDisposition::Accepted, + ); pending = Some(PendingRun::Resume); } - Method::Cancel => match shared_state.get_status() { - WorkerStatus::Paused => match worker.cancel_paused_turn() { + Method::Cancel { command } => { + if let Err(disposition) = validate_command(command, &shared_state) { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Cancel, + disposition, + ); + continue; + } + if !matches!( + shared_state.snapshot().state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) + ) { + reject_invalid_command_state( + &working_event_tx, + &shared_state, + command, + WorkerCommandKind::Cancel, + ); + continue; + } + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)), + ) + .await; + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Cancel, + WorkerCommandDisposition::Accepted, + ); + match worker.cancel_paused_turn() { Ok(()) => { worker.clear_in_flight_events(); - set_controller_status( + set_controller_state( &shared_state, &runtime_dir, &working_event_tx, - WorkerStatus::Idle, + WorkerState::Idle, ) .await; } Err(error) => { + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)), + ) + .await; let _ = working_event_tx.send(Event::Error { code: worker_error_code(&error), message: error.to_string(), }); } - }, - WorkerStatus::Idle | WorkerStatus::Stopped => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::NotRunning, - message: "Worker is not running".into(), - }); - } - WorkerStatus::Running => { - // Running turns receive Cancel through drive_turn; this is - // only reachable across a defensive race window. - let _ = cancel_tx.try_send(()); - } - }, - - Method::Pause => { - // Already paused → idempotent no-op. Otherwise the - // Worker is Idle (Running turns go through `drive_turn`, - // not this outer match), so there is nothing to pause. - if shared_state.get_status() != WorkerStatus::Paused { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::NotRunning, - message: "Worker is not running".into(), - }); } } - Method::Compact => match shared_state.get_status() { - WorkerStatus::Idle => { - if let Err(error) = worker.manual_compact().await { - let _ = working_event_tx.send(Event::Error { - code: worker_error_code(&error), - message: error.to_string(), - }); - } + Method::Pause { command } => { + if let Err(disposition) = validate_command(command, &shared_state) { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Pause, + disposition, + ); + } else { + reject_invalid_command_state( + &working_event_tx, + &shared_state, + command, + WorkerCommandKind::Pause, + ); } - WorkerStatus::Paused => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::InvalidRequest, - message: "Cannot compact while the Worker is paused; resume or start a fresh turn first" - .into(), - }); - } - WorkerStatus::Running | WorkerStatus::Stopped => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::AlreadyRunning, - message: - "Worker is already executing a turn; compact can only run while idle" - .into(), - }); - } - }, + } - Method::ListRewindTargets => match shared_state.get_status() { + Method::Compact { command } => { + if let Err(disposition) = validate_command(command, &shared_state) { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Compact, + disposition, + ); + continue; + } + if !matches!(shared_state.snapshot().state, WorkerState::Idle) { + reject_invalid_command_state( + &working_event_tx, + &shared_state, + command, + WorkerCommandKind::Compact, + ); + continue; + } + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Busy(WorkerBusyState::Maintenance( + WorkerMaintenanceState::Compacting, + )), + ) + .await; + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Compact, + WorkerCommandDisposition::Accepted, + ); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + let mut shutdown_after_compaction = false; + let result = { + let mut compact = Box::pin(worker.manual_compact_with_cancel(cancel_rx)); + loop { + tokio::select! { + result = &mut compact => break result, + method = method_rx.recv() => { + match method { + Some(Method::Cancel { command }) => { + if let Err(disposition) = validate_command(command, &shared_state) { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Cancel, + disposition, + ); + continue; + } + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Cancel, + WorkerCommandDisposition::Accepted, + ); + let _ = cancel_tx.send(true); + } + Some(Method::Shutdown { command }) => { + shared_state.accept_command_id(command.command_id); + shutdown_after_compaction = true; + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + WorkerCommandDisposition::Accepted, + ); + let _ = cancel_tx.send(true); + } + Some(method) => deferred_methods.push_back(method), + None => { + shutdown_after_compaction = true; + let _ = cancel_tx.send(true); + } + } + } + } + } + }; + if !matches!( + result, + Err(WorkerError::Store(_)) + | Err(WorkerError::WorkerStore(_)) + | Err(WorkerError::InvalidState(_)) + ) { + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Idle, + ) + .await; + } + if let Err(error) = result { + let _ = working_event_tx.send(Event::Error { + code: worker_error_code(&error), + message: error.to_string(), + }); + } + if shutdown_after_compaction { + let _ = working_event_tx.send(Event::Shutdown); + break 'controller; + } + } + + Method::ListRewindTargets => match shared_state.catalog_status() { WorkerStatus::Idle | WorkerStatus::Paused => { emit_rewind_targets(&worker, &working_event_tx) } @@ -1908,7 +2149,7 @@ async fn controller_loop( Method::RewindTo { target, expected_head_entries, - } => match shared_state.get_status() { + } => match shared_state.catalog_status() { WorkerStatus::Idle => { if apply_rewind( &mut worker, @@ -1919,10 +2160,8 @@ async fn controller_loop( .await { worker.clear_in_flight_events(); - shared_state.set_status(WorkerStatus::Idle); - let _ = working_event_tx.send(Event::Status { - status: WorkerStatus::Idle, - }); + let snapshot = shared_state.transition(WorkerState::Idle); + let _ = working_event_tx.send(Event::WorkerState { snapshot }); } } WorkerStatus::Paused => { @@ -1941,7 +2180,17 @@ async fn controller_loop( } }, - Method::Shutdown => { + Method::Shutdown { command } => { + // Shutdown remains unconditional/retryable even when the caller's + // live-state fence is stale. + shared_state.accept_command_id(command.command_id); + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + WorkerCommandDisposition::Accepted, + ); let _ = working_event_tx.send(Event::Shutdown); break; } @@ -2023,7 +2272,7 @@ async fn controller_loop( // Auto-kick a turn if the Worker is idle so the // notification is not stranded. Matches the // `Method::Notify` idle path. - if shared_state.get_status() == WorkerStatus::Idle { + if shared_state.catalog_status() == WorkerStatus::Idle { pending = Some(PendingRun::RunForNotification { invoke_kind: protocol::InvokeKind::WorkerEvent, notification_request_id: None, @@ -2270,15 +2519,102 @@ where } method = method_rx.recv(), if input_commit.is_none() => { match method { - Some(Method::Cancel) => { + Some(Method::Cancel { command }) => { + if let Err(disposition) = validate_command(command, shared_state) { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Cancel, + disposition, + ); + continue; + } + if !matches!( + shared_state.snapshot().state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)) + ) { + reject_invalid_command_state( + working_event_tx, + shared_state, + command, + WorkerCommandKind::Cancel, + ); + continue; + } + set_controller_state( + shared_state, + runtime_dir, + working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)), + ) + .await; + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Cancel, + WorkerCommandDisposition::Accepted, + ); let _ = cancel_tx.try_send(()); } - Some(Method::Pause) => { + Some(Method::Pause { command }) => { + if let Err(disposition) = validate_command(command, shared_state) { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Pause, + disposition, + ); + continue; + } + if !matches!( + shared_state.snapshot().state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)) + ) { + reject_invalid_command_state( + working_event_tx, + shared_state, + command, + WorkerCommandKind::Pause, + ); + continue; + } pause_requested = true; + set_controller_state( + shared_state, + runtime_dir, + working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Pausing)), + ) + .await; + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Pause, + WorkerCommandDisposition::Accepted, + ); let _ = pause_tx.try_send(()); } - Some(Method::Shutdown) => { + Some(Method::Shutdown { command }) => { + shared_state.accept_command_id(command.command_id); shutdown_requested = true; + set_controller_state( + shared_state, + runtime_dir, + working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)), + ) + .await; + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + WorkerCommandDisposition::Accepted, + ); let _ = cancel_tx.try_send(()); } Some(Method::Submit { @@ -2344,7 +2680,25 @@ where } } } - Some(Method::Resume | Method::ContinuePending { .. }) => { + Some(Method::Resume { command }) => { + if let Err(disposition) = validate_command(command, shared_state) { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Resume, + disposition, + ); + } else { + reject_invalid_command_state( + working_event_tx, + shared_state, + command, + WorkerCommandKind::Resume, + ); + } + } + Some(Method::ContinuePending { .. }) => { let _ = working_event_tx.send(Event::Error { code: ErrorCode::AlreadyRunning, message: "Worker is already executing a turn".into(), @@ -2384,7 +2738,25 @@ where } } } - Some(Method::Compact | Method::ListRewindTargets | Method::RewindTo { .. }) => { + Some(Method::Compact { command }) => { + if let Err(disposition) = validate_command(command, shared_state) { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Compact, + disposition, + ); + } else { + reject_invalid_command_state( + working_event_tx, + shared_state, + command, + WorkerCommandKind::Compact, + ); + } + } + Some(Method::ListRewindTargets | Method::RewindTo { .. }) => { let _ = working_event_tx.send(Event::Error { code: ErrorCode::AlreadyRunning, message: "Worker is already executing a turn; rewind/compact can only run while idle or paused" @@ -2487,7 +2859,7 @@ where } None => { let _ = cancel_tx.try_send(()); - shared_state.set_status(WorkerStatus::Idle); + shared_state.transition(WorkerState::Idle); return (WorkerStatus::Idle, false, false); } } @@ -2863,7 +3235,7 @@ mod tests { context_window: 200_000, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -2919,9 +3291,17 @@ mod tests { async fn pause_waits_for_run_boundary_and_uses_safe_pause_channel() { let mut env = make_env().await; let method_tx = env._method_tx.clone(); + env.shared_state + .transition(WorkerState::Busy(WorkerBusyState::Run( + WorkerRunState::Running, + ))); + let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot()); tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(10)).await; - method_tx.send(Method::Pause).await.expect("send pause"); + method_tx + .send(Method::Pause { command }) + .await + .expect("send pause"); }); let worker_future = async { @@ -3194,8 +3574,13 @@ mod tests { async fn compact_method_is_rejected_while_running() { let mut env = make_env().await; let mut events = env.working_event_tx.subscribe(); + env.shared_state + .transition(WorkerState::Busy(WorkerBusyState::Run( + WorkerRunState::Running, + ))); + let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot()); env._method_tx - .send(Method::Compact) + .send(Method::Compact { command }) .await .expect("send compact"); @@ -3228,11 +3613,93 @@ mod tests { .expect("event timeout") .expect("event"); match event { - Event::Error { code, message } => { - assert_eq!(code, ErrorCode::AlreadyRunning); - assert!(message.contains("compact"), "got message: {message}"); + Event::CommandAcknowledged { acknowledgement } => { + assert_eq!(acknowledgement.command, WorkerCommandKind::Compact); + assert_eq!( + acknowledgement.disposition, + WorkerCommandDisposition::InvalidState + ); + assert!(matches!( + acknowledgement.state.state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)) + )); } - other => panic!("expected compact rejection error, got {other:?}"), + other => panic!("expected compact rejection acknowledgement, got {other:?}"), } } + + #[test] + fn command_admission_rejects_stale_generation_revision_and_order() { + let shared = WorkerSharedState::new_with_generation( + "worker".into(), + session_store::new_segment_id(), + String::new(), + protocol::Greeting { + worker_name: "worker".into(), + cwd: "/tmp".into(), + provider: "test".into(), + model: "test".into(), + scope_summary: String::new(), + tools: Vec::new(), + context_window: 1, + context_tokens: 0, + }, + 9, + ); + assert_eq!( + validate_command( + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 8, + expected_worker_state_revision: 0, + }, + &shared, + ), + Err(WorkerCommandDisposition::StaleExecutionGeneration) + ); + assert_eq!( + validate_command( + WorkerCommandEnvelope { + command_id: 2, + expected_execution_generation: 9, + expected_worker_state_revision: 1, + }, + &shared, + ), + Err(WorkerCommandDisposition::StaleWorkerStateRevision) + ); + assert!( + validate_command( + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 9, + expected_worker_state_revision: 0, + }, + &shared, + ) + .is_ok() + ); + assert_eq!( + validate_command( + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 9, + expected_worker_state_revision: 0, + }, + &shared, + ), + Err(WorkerCommandDisposition::StaleCommandId) + ); + assert!( + validate_command( + WorkerCommandEnvelope { + command_id: 2, + expected_execution_generation: 9, + expected_worker_state_revision: 0, + }, + &shared, + ) + .is_ok() + ); + } } diff --git a/crates/worker/src/discovery.rs b/crates/worker/src/discovery.rs index 755180f4..844d8c52 100644 --- a/crates/worker/src/discovery.rs +++ b/crates/worker/src/discovery.rs @@ -779,10 +779,10 @@ async fn probe_socket(socket_path: &Path) -> LiveInfo { loop { match tokio::time::timeout(PROBE_TIMEOUT, reader.next::()).await { Ok(Ok(Some(Event::Snapshot { - status: snapshot_status, + state: snapshot_state, .. }))) => { - status = Some(snapshot_status); + status = Some(snapshot_state.catalog_status()); break; } Ok(Ok(Some(Event::Alert(_)))) => continue, @@ -1507,7 +1507,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -1543,7 +1543,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -1638,7 +1638,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -1665,7 +1665,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -1773,7 +1773,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Paused, + state: WorkerStatus::Paused.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -1827,7 +1827,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) diff --git a/crates/worker/src/internal_worker.rs b/crates/worker/src/internal_worker.rs index ec7ef586..63c5b85b 100644 --- a/crates/worker/src/internal_worker.rs +++ b/crates/worker/src/internal_worker.rs @@ -295,6 +295,38 @@ impl InternalWorkerSessionStatus { } } +fn send_internal_worker_state( + event_tx: &broadcast::Sender, + state_revision: &std::sync::atomic::AtomicU64, + status: InternalWorkerSessionStatus, +) { + let state = match status { + InternalWorkerSessionStatus::Idle + | InternalWorkerSessionStatus::Stopped + | InternalWorkerSessionStatus::Failed => protocol::WorkerState::Idle, + InternalWorkerSessionStatus::Paused => protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Paused), + ), + InternalWorkerSessionStatus::Running => protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running), + ), + InternalWorkerSessionStatus::Stopping => protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Cancelling), + ), + }; + let revision = state_revision + .fetch_add(1, std::sync::atomic::Ordering::AcqRel) + .saturating_add(1); + let _ = event_tx.send(Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + execution_generation: 1, + revision, + last_command_id: 0, + state, + }, + }); +} + fn classify_internal_turn_result( result: Result, ) -> (InternalWorkerSessionStatus, Option) { @@ -351,6 +383,7 @@ pub(crate) struct InternalWorkerSessionSnapshot { pub(crate) struct InternalWorkerSessionHandle { command_tx: tokio::sync::mpsc::Sender, status: Arc, + state_revision: Arc, store: EphemeralSessionStore, session_id: SessionId, segment_id: SegmentId, @@ -400,6 +433,10 @@ impl InternalWorkerSessionHandle { self.in_flight.text_delta(block_id, text.to_owned()); } + fn emit_worker_state(&self, status: InternalWorkerSessionStatus) { + send_internal_worker_state(&self.event_tx, &self.state_revision, status); + } + pub(crate) fn protocol_snapshot(&self) -> InternalWorkerSessionSnapshot { let (entries, in_flight) = { let guard = self.in_flight.snapshot_guard(); @@ -473,9 +510,7 @@ impl InternalWorkerSessionHandle { }); return Err(InternalWorkerSessionError::Unavailable); } - let _ = self.event_tx.send(Event::Status { - status: WorkerStatus::Running, - }); + self.emit_worker_state(InternalWorkerSessionStatus::Running); Ok(()) } @@ -767,11 +802,13 @@ pub(crate) async fn prepare_internal_worker_session( let status = Arc::new(std::sync::atomic::AtomicU8::new( InternalWorkerSessionStatus::Idle.encode(), )); + let state_revision = Arc::new(std::sync::atomic::AtomicU64::new(0)); let state_changed = Arc::new(tokio::sync::Notify::new()); let last_error = Arc::new(Mutex::new(None)); let handle = InternalWorkerSessionHandle { command_tx, status: status.clone(), + state_revision: state_revision.clone(), store, session_id, segment_id, @@ -807,19 +844,11 @@ pub(crate) async fn prepare_internal_worker_session( message, }); } - let protocol_status = match turn_status { - InternalWorkerSessionStatus::Idle => WorkerStatus::Idle, - InternalWorkerSessionStatus::Paused => WorkerStatus::Paused, - InternalWorkerSessionStatus::Stopped - | InternalWorkerSessionStatus::Failed => WorkerStatus::Stopped, - InternalWorkerSessionStatus::Running - | InternalWorkerSessionStatus::Stopping => { - unreachable!("run completion cannot remain active") - } - }; - let _ = event_tx.send(Event::Status { - status: protocol_status, - }); + send_internal_worker_state( + &event_tx, + &state_revision, + turn_status, + ); if let Some(callback) = &on_turn_end { callback(turn_status); } @@ -861,9 +890,11 @@ pub(crate) async fn prepare_internal_worker_session( InternalWorkerSessionStatus::Stopped.encode(), std::sync::atomic::Ordering::Release, ); - let _ = event_tx.send(Event::Status { - status: WorkerStatus::Stopped, - }); + send_internal_worker_state( + &event_tx, + &state_revision, + InternalWorkerSessionStatus::Stopped, + ); let _ = event_tx.send(Event::Shutdown); state_changed.notify_waiters(); if let Some(done) = stop_done { @@ -1114,6 +1145,7 @@ pub(crate) fn test_internal_worker_session( status: Arc::new(std::sync::atomic::AtomicU8::new( InternalWorkerSessionStatus::Idle.encode(), )), + state_revision: Arc::new(std::sync::atomic::AtomicU64::new(0)), store, session_id, segment_id, diff --git a/crates/worker/src/runtime/dir.rs b/crates/worker/src/runtime/dir.rs index ad462f39..37d8d99e 100644 --- a/crates/worker/src/runtime/dir.rs +++ b/crates/worker/src/runtime/dir.rs @@ -197,7 +197,6 @@ pub fn default_base() -> Result { mod tests { use super::*; use crate::shared_state::WorkerSharedState; - use protocol::WorkerStatus; fn test_state() -> WorkerSharedState { WorkerSharedState::new( @@ -247,7 +246,9 @@ mod tests { let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); let state = test_state(); - state.set_status(WorkerStatus::Running); + state.transition(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + ))); rt.write_status(&state).await.unwrap(); let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap(); diff --git a/crates/worker/src/shared_state.rs b/crates/worker/src/shared_state.rs index 20563691..58c5700b 100644 --- a/crates/worker/src/shared_state.rs +++ b/crates/worker/src/shared_state.rs @@ -1,7 +1,12 @@ -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{OnceLock, RwLock}; +use std::sync::{ + OnceLock, RwLock, + atomic::{AtomicBool, AtomicU64, Ordering}, +}; -use protocol::WorkerStatus; +use protocol::{ + WorkerBusyState, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, + WorkerStatus, +}; use serde_json::json; use session_store::SegmentId; @@ -9,20 +14,16 @@ use crate::fs_view::WorkerFsView; /// Shared state between WorkerController and runtime directory. /// -/// Controller updates this in-memory; RuntimeDir writes the status -/// snapshot to disk. Wrapped in `Arc` for sharing. -/// -/// History and typed user-segment mirrors used to live here so the -/// IPC layer could answer `Method::GetHistory`. Those reads now go -/// directly through the session-log sink (`Event::Snapshot` + -/// live events), so this struct holds only status, identity, -/// greeting, and filesystem completion lookup hubs. +/// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime +/// catalog status remains a separate lifecycle projection because `Stopped` +/// describes the execution handle rather than a live controller state. pub struct WorkerSharedState { pub worker_name: String, pub segment_id: SegmentId, pub manifest_toml: String, pub greeting: protocol::Greeting, - pub status: RwLock, + state: RwLock, + last_command_id: AtomicU64, /// Worker-from-the-inside view of the filesystem. Set once in /// `WorkerController::start` after the local WorkdirSession provider is /// materialised, and read from the IPC server layer to answer @@ -38,13 +39,24 @@ impl WorkerSharedState { segment_id: SegmentId, manifest_toml: String, greeting: protocol::Greeting, + ) -> Self { + Self::new_with_generation(worker_name, segment_id, manifest_toml, greeting, 1) + } + + pub fn new_with_generation( + worker_name: String, + segment_id: SegmentId, + manifest_toml: String, + greeting: protocol::Greeting, + execution_generation: u64, ) -> Self { Self { worker_name, segment_id, manifest_toml, greeting, - status: RwLock::new(WorkerStatus::Idle), + state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)), + last_command_id: AtomicU64::new(0), fs_view: OnceLock::new(), flow_transition_enabled: AtomicBool::new(false), } @@ -70,21 +82,57 @@ impl WorkerSharedState { self.flow_transition_enabled.load(Ordering::Acquire) } - pub fn set_status(&self, status: WorkerStatus) { - if let Ok(mut s) = self.status.write() { - *s = status; + pub fn transition(&self, state: WorkerState) -> WorkerStateSnapshot { + let mut snapshot = self + .state + .write() + .expect("worker state lock poisoned; refusing an inferred fallback state"); + if snapshot.state != state { + snapshot.revision = snapshot.revision.saturating_add(1); + snapshot.state = state; + } + snapshot.last_command_id = self.last_command_id.load(Ordering::Acquire); + snapshot.clone() + } + + pub fn accept_command_id(&self, command_id: u64) -> bool { + self.last_command_id + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (command_id > current).then_some(command_id) + }) + .is_ok() + } + + pub fn snapshot(&self) -> WorkerStateSnapshot { + let mut snapshot = self + .state + .read() + .expect("worker state lock poisoned; refusing an inferred fallback state") + .clone(); + snapshot.last_command_id = self.last_command_id.load(Ordering::Acquire); + snapshot + } + + /// Runtime catalog projection. This must not be used as live command + /// admission authority. + pub fn catalog_status(&self) -> WorkerStatus { + match self.snapshot().state { + WorkerState::Idle => WorkerStatus::Idle, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused, + WorkerState::Busy(WorkerBusyState::Run(_)) + | WorkerState::Busy(WorkerBusyState::Maintenance(WorkerMaintenanceState::Compacting)) => { + WorkerStatus::Running + } } } - pub fn get_status(&self) -> WorkerStatus { - self.status.read().map(|s| *s).unwrap_or(WorkerStatus::Idle) - } - - /// Serialize status as JSON. + /// Serialize the runtime-directory lifecycle projection as JSON while + /// retaining the full state snapshot for diagnostics and reconnects. pub fn status_json(&self) -> String { - let status = self.get_status(); + let snapshot = self.snapshot(); json!({ - "state": status, + "state": self.catalog_status(), + "worker_state": snapshot, "segment_id": self.segment_id.to_string(), "worker_name": self.worker_name, }) @@ -97,11 +145,12 @@ mod tests { use super::*; fn test_state() -> WorkerSharedState { - WorkerSharedState::new( + WorkerSharedState::new_with_generation( "test-worker".into(), session_store::new_segment_id(), "[engine]\nname = \"test-worker\"".into(), test_greeting(), + 7, ) } @@ -119,36 +168,40 @@ mod tests { } #[test] - fn initial_status_is_idle() { + fn initial_snapshot_is_idle() { let state = test_state(); - assert_eq!(state.get_status(), WorkerStatus::Idle); + assert_eq!(state.snapshot(), WorkerStateSnapshot::initial(7)); + assert_eq!(state.catalog_status(), WorkerStatus::Idle); } #[test] - fn set_and_get_status() { + fn transitions_increment_revision_only_when_state_changes() { let state = test_state(); - state.set_status(WorkerStatus::Running); - assert_eq!(state.get_status(), WorkerStatus::Running); - state.set_status(WorkerStatus::Paused); - assert_eq!(state.get_status(), WorkerStatus::Paused); + let running = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)); + let snapshot = state.transition(running.clone()); + assert_eq!(snapshot.revision, 1); + assert_eq!(snapshot.state, running); + assert_eq!(state.transition(running).revision, 1); + + let paused = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)); + let snapshot = state.transition(paused.clone()); + assert_eq!(snapshot.revision, 2); + assert_eq!(snapshot.state, paused); + assert_eq!(state.catalog_status(), WorkerStatus::Paused); } #[test] - fn status_json_contains_fields() { + fn status_json_contains_full_snapshot_and_catalog_projection() { let state = test_state(); - let json = state.status_json(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["state"], "idle"); + state.transition(WorkerState::Busy(WorkerBusyState::Maintenance( + WorkerMaintenanceState::Compacting, + ))); + let parsed: serde_json::Value = serde_json::from_str(&state.status_json()).unwrap(); + assert_eq!(parsed["state"], "running"); + assert_eq!(parsed["worker_state"]["execution_generation"], 7); + assert_eq!(parsed["worker_state"]["revision"], 1); + assert_eq!(parsed["worker_state"]["state"]["kind"], "busy"); assert_eq!(parsed["worker_name"], "test-worker"); assert!(parsed["segment_id"].is_string()); } - - #[test] - fn status_json_reflects_changes() { - let state = test_state(); - state.set_status(WorkerStatus::Running); - let json = state.status_json(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["state"], "running"); - } } diff --git a/crates/worker/src/spawn/comm_tools.rs b/crates/worker/src/spawn/comm_tools.rs index 0389f288..28efb9a6 100644 --- a/crates/worker/src/spawn/comm_tools.rs +++ b/crates/worker/src/spawn/comm_tools.rs @@ -97,7 +97,7 @@ mod tests { context_window: 200_000, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), } @@ -137,10 +137,16 @@ mod tests { ], ); - connect_and_send(&socket, &Method::Shutdown).await.unwrap(); + let method = Method::Shutdown { + command: protocol::WorkerCommandEnvelope::for_snapshot( + 1, + &protocol::WorkerStateSnapshot::initial(1), + ), + }; + connect_and_send(&socket, &method).await.unwrap(); let method = received.await.unwrap().expect("expected method"); - assert!(matches!(method, Method::Shutdown)); + assert!(matches!(method, Method::Shutdown { .. })); } #[tokio::test] diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 7b051970..93ec40ac 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -4571,15 +4571,6 @@ impl Worker { Ok(()) } - fn persist_and_send_compact_done( - &mut self, - lifecycle: CompactionLifecycle, - ) -> Result<(), WorkerError> { - self.persist_compaction_lifecycle(&lifecycle)?; - self.send_event(Event::CompactDone { lifecycle }); - Ok(()) - } - fn persist_and_send_compact_failed( &mut self, lifecycle: CompactionLifecycle, @@ -4724,7 +4715,97 @@ impl Worker { Ok(rewrite_guard) } + /// Terminalize and clean up any compaction that was left active by the + /// previous controller generation. This runs before the restored + /// controller publishes its first Idle state. + pub async fn recover_unfinished_compaction(&mut self) -> Result<(), WorkerError> { + let (entries, _) = self.sink.subscribe_with_snapshot(); + let latest_payload = entries.iter().rev().find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == COMPACTION_EXTENSION_DOMAIN => Some(payload.clone()), + _ => None, + }); + let Some(payload) = latest_payload else { + return Ok(()); + }; + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct CompactionLifecycleWire { + schema_version: u32, + compaction_id: String, + revision: u64, + #[serde(default)] + internal_worker: Option, + state: CompactionLifecycleState, + started_at_ms: u64, + #[serde(default)] + ended_at_ms: Option, + #[serde(default)] + summary: Option, + #[serde(default)] + error: Option, + #[serde(default)] + new_segment_id: Option, + } + let wire: CompactionLifecycleWire = serde_json::from_value(payload).map_err(|error| { + WorkerError::InvalidState(format!("decode compaction lifecycle: {error}")) + })?; + if !matches!(wire.schema_version, 2 | 3) { + return Err(WorkerError::InvalidState(format!( + "unsupported compaction lifecycle schema version {}", + wire.schema_version + ))); + } + let mut lifecycle = CompactionLifecycle { + schema_version: wire.schema_version, + compaction_id: wire.compaction_id, + revision: wire.revision, + internal_worker: wire.internal_worker, + state: wire.state, + started_at_ms: wire.started_at_ms, + ended_at_ms: wire.ended_at_ms, + summary: wire.summary, + error: wire.error, + new_segment_id: wire.new_segment_id, + }; + match lifecycle.state { + CompactionLifecycleState::Running => { + lifecycle.schema_version = 3; + lifecycle.revision = lifecycle.revision.saturating_add(1); + lifecycle.state = CompactionLifecycleState::Interrupted; + lifecycle.ended_at_ms = Some(segment_log::now_millis()); + lifecycle.error = + Some("worker execution restarted before compaction completed".into()); + self.persist_compaction_lifecycle(&lifecycle)?; + self.send_event(Event::CompactFailed { + lifecycle: lifecycle.clone(), + }); + self.release_compaction_service(&lifecycle).await; + } + CompactionLifecycleState::Interrupted => { + self.release_compaction_service(&lifecycle).await; + } + CompactionLifecycleState::Done | CompactionLifecycleState::Failed => {} + } + Ok(()) + } + pub async fn manual_compact(&mut self) -> Result { + self.manual_compact_inner(None).await + } + + pub async fn manual_compact_with_cancel( + &mut self, + cancel: tokio::sync::watch::Receiver, + ) -> Result { + self.manual_compact_inner(Some(cancel)).await + } + + async fn manual_compact_inner( + &mut self, + mut cancel: Option>, + ) -> Result { if self.manifest.compaction.is_none() { let message = "manual compact is unavailable because [compaction] is not configured".to_string(); @@ -4764,7 +4845,7 @@ impl Worker { return Ok(ManualCompactResult::Skipped { message }); } - match self.compact(retained).await { + match self.compact_with_cancel(retained, cancel.take()).await { Ok(new_segment_id) => { info!(new_segment_id = %new_segment_id, "Manual compaction succeeded"); if let Some(ref state) = state { @@ -4937,11 +5018,19 @@ impl Worker { /// Runs one parent-owned observable compaction service and returns the new /// Segment ID. Lifecycle revisions are committed before they are broadcast. pub async fn compact(&mut self, retained_tokens: u64) -> Result { + self.compact_with_cancel(retained_tokens, None).await + } + + async fn compact_with_cancel( + &mut self, + retained_tokens: u64, + mut cancel: Option>, + ) -> Result { let _rewrite_guard = self .prepare_session_rewrite(SessionRewriteKind::Compact) .await?; let mut lifecycle = CompactionLifecycle { - schema_version: 2, + schema_version: 3, compaction_id: uuid::Uuid::now_v7().to_string(), revision: 1, internal_worker: None, @@ -4953,16 +5042,25 @@ impl Worker { new_segment_id: None, }; self.persist_and_send_compact_start(lifecycle.clone())?; - match self.compact_impl(retained_tokens, &mut lifecycle).await { - Ok((new_segment_id, summary)) => { - lifecycle.revision = lifecycle.revision.saturating_add(1); - lifecycle.state = CompactionLifecycleState::Done; - lifecycle.ended_at_ms = Some(segment_log::now_millis()); - lifecycle.summary = Some(summary); - lifecycle.new_segment_id = Some(new_segment_id.to_string()); - let terminal = self.persist_and_send_compact_done(lifecycle.clone()); + let outcome = if let Some(cancel) = cancel.as_mut() { + tokio::select! { + biased; + changed = cancel.changed() => { + let _ = changed; + Err(WorkerError::CompactCancelled) + } + result = self.compact_impl(retained_tokens, &mut lifecycle) => result, + } + } else { + self.compact_impl(retained_tokens, &mut lifecycle).await + }; + match outcome { + Ok((new_segment_id, _summary)) => { + debug_assert_eq!(lifecycle.state, CompactionLifecycleState::Done); + self.send_event(Event::CompactDone { + lifecycle: lifecycle.clone(), + }); self.release_compaction_service(&lifecycle).await; - terminal?; Ok(new_segment_id) } Err(error) => { @@ -5543,6 +5641,24 @@ impl Worker { })?, }); } + // Commit the terminal lifecycle in the same atomic replacement-segment + // creation as the rewritten history. Restore can therefore never see a + // replacement segment without the Done fact for the compaction that + // created it. + lifecycle.revision = lifecycle.revision.saturating_add(1); + lifecycle.state = CompactionLifecycleState::Done; + lifecycle.ended_at_ms = Some(segment_log::now_millis()); + lifecycle.summary = Some(summary_text.clone()); + lifecycle.new_segment_id = Some(new_segment_id.to_string()); + initial_entries.push(LogEntry::Extension { + ts: segment_log::now_millis(), + domain: COMPACTION_EXTENSION_DOMAIN.to_string(), + payload: serde_json::to_value(&*lifecycle).map_err(|error| { + WorkerError::InvalidState(format!( + "serialize terminal compaction lifecycle: {error}" + )) + })?, + }); self.store .create_segment(old_loc.session_id, new_segment_id, &initial_entries)?; self.segment_state.set_location(SegmentLocation { @@ -10165,6 +10281,56 @@ mod build_summary_prompt_tests { assert_eq!(state.notification_receipts.len(), 1); } + #[tokio::test] + async fn restore_terminalizes_running_compaction_before_idle_publication() { + let (_dir, mut worker) = rewind_test_worker().await; + let lifecycle = CompactionLifecycle { + schema_version: 3, + compaction_id: "compact-before-restart".into(), + revision: 1, + internal_worker: None, + state: CompactionLifecycleState::Running, + started_at_ms: segment_log::now_millis(), + ended_at_ms: None, + summary: None, + error: None, + new_segment_id: None, + }; + worker.persist_compaction_lifecycle(&lifecycle).unwrap(); + + worker.recover_unfinished_compaction().await.unwrap(); + + let (entries, _) = worker.sink.subscribe_with_snapshot(); + let restored = entries.iter().rev().find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == COMPACTION_EXTENSION_DOMAIN => { + serde_json::from_value::(payload.clone()).ok() + } + _ => None, + }); + let restored = restored.expect("terminal compaction lifecycle"); + assert_eq!(restored.state, CompactionLifecycleState::Interrupted); + assert_eq!(restored.revision, 2); + assert!( + restored + .error + .as_deref() + .is_some_and(|error| error.contains("restarted")) + ); + + let mut future = lifecycle; + future.schema_version = 4; + future.compaction_id = "future-compaction".into(); + worker.persist_compaction_lifecycle(&future).unwrap(); + let error = worker.recover_unfinished_compaction().await.unwrap_err(); + assert!( + error + .to_string() + .contains("unsupported compaction lifecycle schema version 4") + ); + } + fn minimal_manifest() -> WorkerManifest { let toml_str = r#" [worker] diff --git a/crates/worker/tests/compact_events_test.rs b/crates/worker/tests/compact_events_test.rs index 8265f9e2..1521b909 100644 --- a/crates/worker/tests/compact_events_test.rs +++ b/crates/worker/tests/compact_events_test.rs @@ -72,6 +72,41 @@ impl LlmClient for MockClient { } } +#[derive(Clone)] +struct BlockingCompactClient { + calls: Arc, +} + +impl BlockingCompactClient { + fn new() -> Self { + Self { + calls: Arc::new(AtomicUsize::new(0)), + } + } +} + +#[async_trait] +impl LlmClient for BlockingCompactClient { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + + async fn stream( + &self, + _request: Request, + ) -> Result> + Send>>, ClientError> + { + let call = self.calls.fetch_add(1, Ordering::SeqCst); + if call == 0 { + Ok(Box::pin(futures::stream::iter( + single_text_events("seed").into_iter().map(Ok), + ))) + } else { + Ok(Box::pin(futures::stream::pending())) + } + } +} + fn single_text_events(text: &str) -> Vec { vec![ LlmEvent::text_block_start(0), @@ -156,10 +191,10 @@ target = "./" permission = "write" "#; -async fn make_worker_with_manifest( - manifest_toml: &str, - client: MockClient, -) -> Worker { +async fn make_worker_with_manifest(manifest_toml: &str, client: C) -> Worker +where + C: LlmClient + Clone + Send + Sync + 'static, +{ let manifest = worker::WorkerManifest::from_toml(manifest_toml).unwrap(); let store_tmp = tempfile::tempdir().unwrap(); @@ -614,12 +649,144 @@ async fn pre_run_compact_failure_broadcasts_start_and_failed() { ); } +#[tokio::test] +async fn manual_compact_cancel_terminalizes_before_returning_idle() { + let worker = + make_worker_with_manifest(POST_RUN_MANIFEST_TOML, BlockingCompactClient::new()).await; + let runtime_tmp = tempfile::tempdir().unwrap(); + let bash_output_dir = runtime_tmp.path().join("bash-output"); + let (handle, shutdown_receiver) = + WorkerController::spawn(worker, runtime_tmp.path(), &bash_output_dir) + .await + .unwrap(); + let mut rx = handle.subscribe(); + + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "seed history", + )) + .await + .expect("send seed run"); + loop { + if matches!( + tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for seed run") + .expect("event"), + Event::RunEnd { + result: RunResult::Finished + } + ) { + break; + } + } + + let compact = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot()); + handle + .send(Method::Compact { command: compact }) + .await + .expect("send compact"); + loop { + if matches!( + tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for compact start") + .expect("event"), + Event::CompactStart { .. } + ) { + break; + } + } + + let cancel = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot()); + handle + .send(Method::Cancel { command: cancel }) + .await + .expect("send compact cancel"); + let mut saw_interrupted = false; + let mut saw_idle = false; + while !(saw_interrupted && saw_idle) { + match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for compact cancellation") + .expect("event") + { + Event::CompactFailed { lifecycle } + if lifecycle.state == protocol::CompactionLifecycleState::Interrupted => + { + saw_interrupted = true; + } + Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Idle => + { + assert!( + saw_interrupted, + "Idle must follow durable Interrupted evidence" + ); + saw_idle = true; + } + _ => {} + } + } + + let compact = protocol::WorkerCommandEnvelope::for_snapshot(3, &handle.shared_state.snapshot()); + handle + .send(Method::Compact { command: compact }) + .await + .expect("send second compact"); + loop { + if matches!( + tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for second compact start") + .expect("event"), + Event::CompactStart { .. } + ) { + break; + } + } + let shutdown = + protocol::WorkerCommandEnvelope::for_snapshot(4, &handle.shared_state.snapshot()); + handle + .send(Method::Shutdown { command: shutdown }) + .await + .expect("send shutdown during compact"); + let mut interrupted_before_shutdown = false; + loop { + match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for shutdown") + .expect("event") + { + Event::CompactFailed { lifecycle } + if lifecycle.state == protocol::CompactionLifecycleState::Interrupted => + { + interrupted_before_shutdown = true; + } + Event::Shutdown => { + assert!( + interrupted_before_shutdown, + "shutdown must await terminal compaction evidence" + ); + break; + } + _ => {} + } + } + tokio::time::timeout(std::time::Duration::from_secs(2), shutdown_receiver) + .await + .expect("controller shutdown timeout") + .expect("shutdown confirmation"); +} + #[tokio::test] async fn controller_compact_method_emits_start_and_done() { let client = MockClient::new(vec![ text_events_with_usage("hi", 1000), write_summary_tool_use_events("manual-summary", "manual compact summary"), single_text_events("done"), + single_text_events("follow-up"), ]); let worker = make_worker_with_manifest(POST_RUN_MANIFEST_TOML, client).await; let runtime_tmp = tempfile::tempdir().unwrap(); @@ -649,7 +816,11 @@ async fn controller_compact_method_emits_start_and_done() { } } - handle.send(Method::Compact).await.expect("send compact"); + let command = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot()); + handle + .send(Method::Compact { command }) + .await + .expect("send compact"); let mut saw_start = false; loop { match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) @@ -670,5 +841,30 @@ async fn controller_compact_method_emits_start_and_done() { } assert!(saw_start, "manual compact should emit CompactStart"); - let _ = handle.send(Method::Shutdown).await; + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "run after compact", + )) + .await + .expect("send follow-up run"); + loop { + match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for follow-up run") + .expect("event") + { + Event::RunEnd { + result: RunResult::Finished, + } => break, + _ => {} + } + } + assert_eq!( + handle.shared_state.catalog_status(), + protocol::WorkerStatus::Idle, + "successful manual compaction must release the execution fence" + ); + let command = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot()); + let _ = handle.send(Method::Shutdown { command }).await; } diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 319f773d..35bf5193 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -1,5 +1,5 @@ use std::pin::Pin; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use agen::Engine; @@ -25,6 +25,15 @@ use worker::{ type TestStore = CombinedStore; +static NEXT_COMMAND_ID: AtomicU64 = AtomicU64::new(1); + +fn worker_command(handle: &WorkerHandle) -> protocol::WorkerCommandEnvelope { + protocol::WorkerCommandEnvelope::for_snapshot( + NEXT_COMMAND_ID.fetch_add(1, Ordering::Relaxed), + &handle.shared_state.snapshot(), + ) +} + /// Reconstruct a worker-history-like `Vec` from the live session /// log mirror held by the Worker's broadcast sink. Replaces the previous /// `WorkerSharedState.history()` test helper now that the mirror lives in @@ -313,7 +322,12 @@ async fn controller_grants_read_scope_for_exact_bash_output_directory() { })); assert!(!handle.runtime_dir.path().join("bash-output").exists()); - handle.send(Method::Shutdown).await.unwrap(); + handle + .send(Method::Shutdown { + command: worker_command(&handle), + }) + .await + .unwrap(); shutdown_rx.await.unwrap(); } @@ -345,7 +359,12 @@ async fn shutdown_closes_bound_workdir_session() { WorkerController::spawn(worker, runtime_base.path(), &bash_output_dir) .await .unwrap(); - handle.send(Method::Shutdown).await.unwrap(); + handle + .send(Method::Shutdown { + command: worker_command(&handle), + }) + .await + .unwrap(); tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx) .await .expect("controller should shut down") @@ -459,7 +478,12 @@ async fn controller_projects_workdir_command_events_and_snapshot_state() { !durable_history.contains("ready") && !durable_history.contains("done"), "operational command chunks must not be appended to Worker history: {durable_history}" ); - handle.send(Method::Shutdown).await.unwrap(); + handle + .send(Method::Shutdown { + command: worker_command(&handle), + }) + .await + .unwrap(); } #[tokio::test] @@ -530,7 +554,12 @@ async fn controller_refreshes_command_snapshot_after_high_output_provider_lag() .await .unwrap(); assert_eq!(output.status, workdir::CommandStatus::Cancelled); - handle.send(Method::Shutdown).await.unwrap(); + handle + .send(Method::Shutdown { + command: worker_command(&handle), + }) + .await + .unwrap(); } #[tokio::test] @@ -571,13 +600,13 @@ async fn controller_startup_failure_closes_bound_workdir_session() { async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) { let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); loop { - if handle.shared_state.get_status() == status { + if handle.shared_state.catalog_status() == status { return; } assert!( tokio::time::Instant::now() < deadline, "timed out waiting for status {status:?}; current={:?}", - handle.shared_state.get_status() + handle.shared_state.catalog_status() ); tokio::time::sleep(std::time::Duration::from_millis(10)).await; } @@ -1029,7 +1058,8 @@ async fn run_end_returns_to_idle_without_busy_status() { Ok(Event::RunEnd { result: protocol::RunResult::Finished }) => { saw_run_end = true; } - Ok(Event::Status { status: WorkerStatus::Idle }) if saw_run_end => { + Ok(Event::WorkerState { snapshot }) + if saw_run_end && snapshot.catalog_status() == WorkerStatus::Idle => { saw_idle_status = true; break; } @@ -1046,7 +1076,7 @@ async fn run_end_returns_to_idle_without_busy_status() { saw_idle_status, "expected idle status immediately after RunEnd" ); - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); } #[tokio::test] @@ -1124,9 +1154,7 @@ async fn snapshot_includes_user_input_for_in_flight_turn() { loop { if matches!( events.recv().await, - Ok(Event::Status { - status: WorkerStatus::Running, - }) + Ok(Event::WorkerState { snapshot }) if snapshot.catalog_status() == WorkerStatus::Running ) { break; } @@ -1201,8 +1229,8 @@ async fn attach_snapshot_includes_current_status() { loop { let event = reader.next::().await.unwrap().unwrap(); match event { - Event::Snapshot { status, .. } => { - assert_eq!(status, WorkerStatus::Running); + Event::Snapshot { state, .. } => { + assert_eq!(state.catalog_status(), WorkerStatus::Running); return; } Event::Alert(_) => continue, @@ -1217,7 +1245,7 @@ async fn shared_state_starts_idle() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); } #[tokio::test] @@ -1237,7 +1265,7 @@ async fn run_updates_shared_state_to_idle_after_completion() { // Wait for the run to complete tokio::time::sleep(std::time::Duration::from_millis(100)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); } #[tokio::test] @@ -1360,7 +1388,12 @@ async fn submit_while_running_is_durably_queued() { assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued)); let pending_snapshot = pending_snapshot.expect("pending snapshot"); assert_eq!(pending_snapshot.submissions.len(), 1); - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Paused).await; handle .send(Method::ContinuePending { @@ -1382,17 +1415,22 @@ async fn submit_while_running_is_durably_queued() { .await .expect("paused ContinuePending rejection"); assert!(rejection.contains("Resume or Cancel")); - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused); } #[tokio::test] -async fn resume_without_pause_returns_error() { +async fn resume_without_pause_returns_invalid_state_acknowledgement() { let client = MockClient::new(simple_text_events()); let worker = make_worker(client).await; let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::Resume).await.unwrap(); + handle + .send(Method::Resume { + command: worker_command(&handle), + }) + .await + .unwrap(); let mut saw_not_paused = false; let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); @@ -1400,7 +1438,10 @@ async fn resume_without_pause_returns_error() { tokio::select! { event = rx.recv() => { match event { - Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotPaused => { + Ok(Event::CommandAcknowledged { acknowledgement }) + if acknowledgement.command == protocol::WorkerCommandKind::Resume + && acknowledgement.disposition + == protocol::WorkerCommandDisposition::InvalidState => { saw_not_paused = true; break; } @@ -1412,17 +1453,22 @@ async fn resume_without_pause_returns_error() { } } - assert!(saw_not_paused, "should see not_paused error"); + assert!(saw_not_paused, "should see invalid-state acknowledgement"); } #[tokio::test] -async fn cancel_without_run_returns_error() { +async fn cancel_without_run_returns_invalid_state_acknowledgement() { let client = MockClient::new(simple_text_events()); let worker = make_worker(client).await; let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::Cancel).await.unwrap(); + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); let mut saw_not_running = false; let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); @@ -1430,7 +1476,10 @@ async fn cancel_without_run_returns_error() { tokio::select! { event = rx.recv() => { match event { - Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotRunning => { + Ok(Event::CommandAcknowledged { acknowledgement }) + if acknowledgement.command == protocol::WorkerCommandKind::Cancel + && acknowledgement.disposition + == protocol::WorkerCommandDisposition::InvalidState => { saw_not_running = true; break; } @@ -1442,7 +1491,7 @@ async fn cancel_without_run_returns_error() { } } - assert!(saw_not_running, "should see not_running error"); + assert!(saw_not_running, "should see invalid-state acknowledgement"); } #[tokio::test] @@ -1818,7 +1867,7 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() { } tokio::time::sleep(std::time::Duration::from_millis(100)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); assert!( client_for_assert.captured_requests().is_empty(), "weak Notify must not stage RunForNotification while idle" @@ -1915,7 +1964,7 @@ async fn worker_event_turn_ended_while_idle_auto_starts_turn_and_injects_system_ saw_worker_event_in_mirror, "Method::WorkerEvent should commit a SystemItem::WorkerEvent entry" ); - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); let requests = client_for_assert.captured_requests(); assert_eq!( @@ -1978,7 +2027,7 @@ async fn worker_event_scope_sub_delegated_while_idle_stays_control_plane_only() tokio::time::sleep(std::time::Duration::from_millis(100)).await; assert_eq!( - handle.shared_state.get_status(), + handle.shared_state.catalog_status(), WorkerStatus::Idle, "control-plane ScopeSubDelegated must not auto-start the parent LLM" ); @@ -2081,7 +2130,12 @@ async fn weak_notify_while_running_is_deduped_and_survives_until_next_submit() { .await .unwrap(); } - handle.send(Method::Cancel).await.unwrap(); + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let mut rx = handle.subscribe(); @@ -2478,7 +2532,12 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { "text_delta should arrive before pause" ); - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); // The controller emits RunEnd { Paused } when the // EngineError::Cancelled is translated under pause_requested. @@ -2494,9 +2553,14 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { ); tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused); - handle.send(Method::Resume).await.unwrap(); + handle + .send(Method::Resume { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( @@ -2510,7 +2574,7 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { ); tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); // History consistency: exactly [user "hello", assistant // "resumed output"]. No artifacts from the aborted stream @@ -2610,7 +2674,12 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() { "tool_call_done should arrive before pause" ); - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2622,7 +2691,7 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() { "expected RunEnd::Paused" ); tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused); // New user input while Paused → `Worker::run` observes // `last_run_interrupted` and runs its interrupt-prep step, which @@ -2781,7 +2850,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { "tool_call_done should arrive before pause" ); - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2794,7 +2868,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { ); wait_for_status(&handle, WorkerStatus::Paused).await; - handle.send(Method::Cancel).await.unwrap(); + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let (entries_after_cancel, _rx_after_cancel) = handle.sink.subscribe_with_snapshot(); assert!( @@ -2820,17 +2899,22 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { "paused cancel must not resume or start another LLM request" ); - handle.send(Method::Resume).await.unwrap(); + handle + .send(Method::Resume { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, - Event::Error { - code: worker::ErrorCode::NotPaused, - .. - } + Event::CommandAcknowledged { acknowledgement } + if acknowledgement.command == protocol::WorkerCommandKind::Resume + && acknowledgement.disposition + == protocol::WorkerCommandDisposition::InvalidState )) .await, - "resume after paused cancel should be rejected as not paused" + "resume after paused cancel should receive invalid-state acknowledgement" ); assert_eq!( client_for_assert.captured_requests().len(), @@ -2939,7 +3023,12 @@ async fn empty_turn_cancel_rolls_back_submit_entries_and_emits_signal() { .await .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; - handle.send(Method::Cancel).await.unwrap(); + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( @@ -2977,7 +3066,12 @@ async fn empty_turn_pause_rolls_back_and_snapshot_does_not_restore_input() { .await .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( @@ -3034,7 +3128,12 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() { .await .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; - handle.send(Method::Cancel).await.unwrap(); + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -3091,7 +3190,12 @@ async fn pause_after_assistant_token_does_not_rollback() { .await, "assistant token should be visible before pause" ); - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 4a90e192..f615685d 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -36,8 +36,6 @@ use worker_runtime::config_bundle::{ ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor, }; use worker_runtime::error::RuntimeError as EmbeddedRuntimeError; -#[cfg(test)] -use worker_runtime::execution::WorkerExecutionRunState; use worker_runtime::fs_store::FsRuntimeStoreOptions; use worker_runtime::http_server::{ RUNTIME_PING_PERMISSION, RUNTIME_WORKSPACE_SCOPE_HEADER, @@ -5170,7 +5168,6 @@ mod tests { request.worker_ref, self.backend_id(), ), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -5199,8 +5196,8 @@ mod tests { let content = input.content; std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(10)); - let _ = context.publish_protocol_event(protocol::Event::Status { - status: protocol::WorkerStatus::Running, + let _ = context.publish_protocol_event(protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }); let _ = context.publish_protocol_event(protocol::Event::TextDone { text: format!("echo: {content}"), @@ -5208,14 +5205,13 @@ mod tests { let _ = context.publish_protocol_event(protocol::Event::RunEnd { result: protocol::RunResult::Finished, }); - let _ = context.publish_protocol_event(protocol::Event::Status { - status: protocol::WorkerStatus::Idle, + let _ = context.publish_protocol_event(protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Idle.into(), }); }); if let Some(submission_request_id) = submission_request_id { worker_runtime::execution::WorkerExecutionResult::accepted_submission( worker_runtime::execution::WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, submission_request_id, uuid::Uuid::now_v7().to_string(), protocol::SubmissionDisposition::Started, @@ -5223,7 +5219,6 @@ mod tests { } else { worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, ) } } diff --git a/crates/workspace-server/src/runtime_subscription_tests.rs b/crates/workspace-server/src/runtime_subscription_tests.rs index 6c14c56a..5c67f983 100644 --- a/crates/workspace-server/src/runtime_subscription_tests.rs +++ b/crates/workspace-server/src/runtime_subscription_tests.rs @@ -6,7 +6,7 @@ use worker_runtime::catalog::{ }; use worker_runtime::execution::{ WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionResult, - WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, }; use worker_runtime::identity::WorkerId; use worker_runtime::profile_archive::{ProfileSourceArchiveRef, ProfileSourceGraphSummary}; @@ -22,7 +22,6 @@ impl WorkerExecutionBackend for TestExecutionBackend { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::connected( WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - WorkerExecutionRunState::Idle, None, ) } @@ -35,24 +34,17 @@ impl WorkerExecutionBackend for TestExecutionBackend { if let Some(submission_request_id) = input.submission_request_id { WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, submission_request_id, uuid::Uuid::now_v7().to_string(), protocol::SubmissionDisposition::Started, ) } else { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Input) } } fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Stop, - WorkerExecutionRunState::Stopped, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) } } @@ -199,8 +191,8 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() { runtime .observe_worker_event( &worker.worker_ref, - protocol::Event::Status { - status: protocol::WorkerStatus::Running, + protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }, ) .unwrap(); @@ -339,8 +331,8 @@ async fn embedded_runtime_uses_in_process_subscription_source() { runtime .observe_worker_event( &worker.worker_ref, - protocol::Event::Status { - status: protocol::WorkerStatus::Running, + protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }, ) .unwrap(); diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 3fab8457..b55f3d2a 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -18559,7 +18559,6 @@ mod tests { request.worker_ref, self.backend_id(), ), - run_state: worker_runtime::execution::WorkerExecutionRunState::Idle, working_directory, } } @@ -18575,7 +18574,6 @@ mod tests { .push((handle.worker_ref().clone(), method)); worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod, - worker_runtime::execution::WorkerExecutionRunState::Idle, ) } @@ -18585,7 +18583,6 @@ mod tests { ) -> worker_runtime::execution::WorkerExecutionResult { worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Stop, - worker_runtime::execution::WorkerExecutionRunState::Stopped, ) } @@ -18595,7 +18592,6 @@ mod tests { ) -> worker_runtime::execution::WorkerExecutionResult { worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Cancel, - worker_runtime::execution::WorkerExecutionRunState::Stopped, ) } @@ -18632,16 +18628,16 @@ mod tests { if let Some(submission_request_id) = submission_request_id { worker_runtime::execution::WorkerExecutionResult::accepted_submission( worker_runtime::execution::WorkerExecutionOperation::Input, - worker_runtime::execution::WorkerExecutionRunState::Idle, submission_request_id, uuid::Uuid::now_v7().to_string(), protocol::SubmissionDisposition::Started, ) + .with_worker_state(protocol::WorkerStateSnapshot::initial(1)) } else { worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Input, - worker_runtime::execution::WorkerExecutionRunState::Idle, ) + .with_worker_state(protocol::WorkerStateSnapshot::initial(1)) } } } @@ -27497,7 +27493,13 @@ mod tests { protocol::subscription::SubscriptionFramePayload::WorkerProtocol( protocol::subscription::SubscriptionWorkerProtocolMethod { subscription_id: second_protocol_subscription_id, - method: protocol::Method::Resume, + method: protocol::Method::Resume { + command: protocol::WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 1, + expected_worker_state_revision: 0, + }, + }, }, ), ); @@ -27514,7 +27516,7 @@ mod tests { .iter() .any(|(worker_ref, method)| { worker_ref.worker_id.to_string() == worker_id - && matches!(method, protocol::Method::Resume) + && matches!(method, protocol::Method::Resume { .. }) }) { break; @@ -27527,7 +27529,7 @@ mod tests { let protocol_methods = execution_backend.protocol_methods(); assert!(protocol_methods.iter().any(|(worker_ref, method)| { worker_ref.worker_id.to_string() == worker_id - && matches!(method, protocol::Method::Resume) + && matches!(method, protocol::Method::Resume { .. }) })); server.abort(); } diff --git a/web/workspace/src/lib/generated/protocol.ts b/web/workspace/src/lib/generated/protocol.ts index 6055d3e7..c15fbdca 100644 --- a/web/workspace/src/lib/generated/protocol.ts +++ b/web/workspace/src/lib/generated/protocol.ts @@ -10,6 +10,37 @@ export type CompletionKind = "file"; export type WorkerStatus = "idle" | "running" | "paused" | "stopped"; +export type WorkerCommandEnvelope = { +/** + * Caller-owned sequence. A controller accepts command ids in strictly + * increasing order for one execution generation. + */ +command_id: number, expected_execution_generation: number, expected_worker_state_revision: number, }; + +export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown"; + +export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "invalid_state"; + +export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition, +/** + * The complete authoritative state observed after command admission. + */ +state: WorkerStateSnapshot, }; + +export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling"; + +export type WorkerMaintenanceState = "compacting"; + +export type WorkerBusyState = { "kind": "run", "state": WorkerRunState } | { "kind": "maintenance", "state": WorkerMaintenanceState }; + +export type WorkerState = { "kind": "idle" } | { "kind": "busy", "state": WorkerBusyState }; + +export type WorkerStateSnapshot = { execution_generation: number, revision: number, +/** + * Highest lifecycle command id observed by this controller generation. + */ +last_command_id: number, state: WorkerState, }; + export type TurnResult = "finished" | "paused"; export type InvokeKind = "user_send" | "notify" | "worker_event" | "system_reminder" | "wakeup"; @@ -231,7 +262,7 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod }); -export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; +export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume", "params": { command: WorkerCommandEnvelope, } } | { "method": "cancel", "params": { command: WorkerCommandEnvelope, } } | { "method": "pause", "params": { command: WorkerCommandEnvelope, } } | { "method": "compact", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number, /** @@ -247,7 +278,12 @@ summary: string, * Full tool output. Absent when the tool chose to return * summary-only, or when the result was pruned. */ -output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting, status: WorkerStatus, +output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting, +/** + * Full revisioned live execution state. `Stopped` remains Runtime + * catalog authority and is deliberately not represented here. + */ +state: WorkerStateSnapshot, /** * Unfinished model output that has already streamed in the current * run but is not yet represented by committed snapshot entries. @@ -257,4 +293,4 @@ in_flight?: InFlightSnapshot, * Parent-owned Internal Worker sessions visible to this client. * Service-private Internal Workers are deliberately excluded. */ -internal_workers?: Array, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "status", "data": { status: WorkerStatus, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" }; +internal_workers?: Array, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "worker_state", "data": { snapshot: WorkerStateSnapshot, } } | { "event": "command_acknowledged", "data": { acknowledgement: WorkerCommandAcknowledgement, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" }; diff --git a/web/workspace/src/lib/workspace/console/model.test.ts b/web/workspace/src/lib/workspace/console/model.test.ts index 60b98633..c1f1b72a 100644 --- a/web/workspace/src/lib/workspace/console/model.test.ts +++ b/web/workspace/src/lib/workspace/console/model.test.ts @@ -1,4 +1,4 @@ -import type { Event } from "$lib/generated/protocol"; +import type { Event, WorkerStateSnapshot, WorkerStatus } from "$lib/generated/protocol"; import { type ConsoleEventInput, type ConsoleLine, @@ -19,6 +19,23 @@ declare const Deno: { test(name: string, fn: () => void): void; }; +function workerState(status: WorkerStatus): WorkerStateSnapshot { + return { + execution_generation: 1, + revision: status === "idle" ? 0 : 1, + last_command_id: 0, + state: status === "idle" + ? { kind: "idle" } + : { + kind: "busy", + state: { + kind: "run", + state: status === "paused" ? "paused" : "running", + }, + }, + }; +} + function assert(condition: unknown, message: string): asserts condition { if (!condition) { throw new Error(message); @@ -131,7 +148,7 @@ function snapshotEvent(cwd: string, entries: unknown[] = []): Event { context_window: 100, context_tokens: 20, }, - status: "idle", + state: workerState("idle"), in_flight: { blocks: [] }, }, }; @@ -213,7 +230,7 @@ Deno.test("snapshot replaces a live error with one durable run_errored row", () }, { eventId: "idle-after-error", - event: { event: "status", data: { status: "idle" } } satisfies Event, + event: { event: "worker_state", data: { snapshot: workerState("idle") } } satisfies Event, }, ]); @@ -653,7 +670,7 @@ Deno.test("projectConsole streams distinct Bash stdout and stderr through termin Deno.test("snapshot restores bounded in-flight Bash command output", () => { const snapshot = snapshotEvent("/repo"); if (snapshot.event !== "snapshot") throw new Error("snapshot fixture expected"); - snapshot.data.status = "running"; + snapshot.data.state = workerState("running"); snapshot.data.in_flight = { blocks: [{ kind: "tool_call", @@ -1403,7 +1420,7 @@ Deno.test("projectConsole hides lifecycle events and renders system items", () = const projection = projectConsole([ { eventId: "30", - event: { event: "status", data: { status: "running" } } satisfies Event, + event: { event: "worker_state", data: { snapshot: workerState("running") } } satisfies Event, }, { eventId: "31", @@ -1527,7 +1544,7 @@ Deno.test("projectConsole renders snapshot entries and in-flight output", () => context_window: 100, context_tokens: 20, }, - status: "running", + state: workerState("running"), in_flight: { blocks: [ { kind: "text", text: "partial" }, @@ -1578,7 +1595,7 @@ Deno.test("projectConsole restores system items from snapshot entries", () => { context_window: 100, context_tokens: 20, }, - status: "idle", + state: workerState("idle"), }, } satisfies Event, }]); @@ -1922,7 +1939,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => { kind: "sub_worker", }, revision: 1, - event: { event: "status", data: { status: "running" } }, + event: { event: "worker_state", data: { snapshot: workerState("running") } }, }, }, }, @@ -1941,7 +1958,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => { kind: "sub_worker", }, revision: 1, - event: { event: "status", data: { status: "idle" } }, + event: { event: "worker_state", data: { snapshot: workerState("idle") } }, }, }, }]); @@ -2033,7 +2050,7 @@ Deno.test("parent snapshot authoritatively replaces Internal Worker projections" kind: "sub_worker", }, revision: 1, - event: { event: "status", data: { status: "running" } }, + event: { event: "worker_state", data: { snapshot: workerState("running") } }, }, }, }]); diff --git a/web/workspace/src/lib/workspace/console/model.ts b/web/workspace/src/lib/workspace/console/model.ts index 4464c89a..857e1b30 100644 --- a/web/workspace/src/lib/workspace/console/model.ts +++ b/web/workspace/src/lib/workspace/console/model.ts @@ -10,6 +10,8 @@ import type { InternalWorkerRef, InternalWorkerSnapshot, Segment, + WorkerStateSnapshot, + WorkerStatus, } from "$lib/generated/protocol"; import { stringify as stringifyYaml } from "yaml"; import { workspaceRoute } from "$lib/workspace/api/http"; @@ -169,6 +171,7 @@ export type ConsoleProjection = { tasks: ConsoleTask[]; taskNextId: number; status: string | null; + workerState: WorkerStateSnapshot | null; usage: string | null; runActivity: RunActivityStats; cwd: string | null; @@ -251,12 +254,22 @@ export function isConsoleProjectionEvent(event: ProtocolEvent): boolean { return event.event !== "completions"; } +function workerStatusFromState(snapshot: WorkerStateSnapshot): WorkerStatus { + if (snapshot.state.kind === "idle") return "idle"; + if ( + snapshot.state.state.kind === "run" && + snapshot.state.state.state === "paused" + ) return "paused"; + return "running"; +} + export function emptyConsoleProjection(): ConsoleProjection { return { lines: [], tasks: [], taskNextId: 1, status: null, + workerState: null, usage: null, runActivity: emptyRunActivityStats(), cwd: null, @@ -793,6 +806,7 @@ export function applyProtocolEvent( tasks: [...projection.tasks], taskNextId: projection.taskNextId, status: projection.status, + workerState: projection.workerState, usage: projection.usage, runActivity: applyRunActivityEvent( projection.runActivity, @@ -903,7 +917,8 @@ export function applyProtocolEvent( ); break; case "snapshot": { - next.status = event.data.status; + next.workerState = event.data.state; + next.status = workerStatusFromState(event.data.state); next.cwd = event.data.greeting.cwd; const snapshot = snapshotProjectionFromSession( envelope.eventId, @@ -1000,8 +1015,13 @@ export function applyProtocolEvent( if (existingIndex >= 0) next.internalWorkers.splice(existingIndex, 1); break; } - case "status": - next.status = event.data.status; + case "worker_state": + next.workerState = event.data.snapshot; + next.status = workerStatusFromState(event.data.snapshot); + break; + case "command_acknowledged": + next.workerState = event.data.acknowledgement.state; + next.status = workerStatusFromState(event.data.acknowledgement.state); break; case "command": applyCommandEvent(next, envelope.eventId, event.data.event); @@ -1939,6 +1959,7 @@ function snapshotProjectionFromSession( tasks: [], taskNextId: 1, status: null, + workerState: null, usage: null, runActivity: emptyRunActivityStats(), cwd, diff --git a/web/workspace/src/lib/workspace/console/run-status.test.ts b/web/workspace/src/lib/workspace/console/run-status.test.ts index 575ce733..0063a242 100644 --- a/web/workspace/src/lib/workspace/console/run-status.test.ts +++ b/web/workspace/src/lib/workspace/console/run-status.test.ts @@ -75,7 +75,12 @@ Deno.test("new invoke and running snapshot reset run activity", () => { data: { entries: [], greeting: { text: "", profile: "" }, - status: "idle", + state: { + execution_generation: 1, + revision: 0, + last_command_id: 0, + state: { kind: "idle" }, + }, in_flight: {}, internal_workers: [], }, diff --git a/web/workspace/src/lib/workspace/console/run-status.ts b/web/workspace/src/lib/workspace/console/run-status.ts index aabb45c2..37495dc1 100644 --- a/web/workspace/src/lib/workspace/console/run-status.ts +++ b/web/workspace/src/lib/workspace/console/run-status.ts @@ -25,7 +25,9 @@ export function applyRunActivityEvent( case "invoke_start": return { ...emptyRunActivityStats(), startedAtMs: observedAtMs }; case "snapshot": - return event.data.status === "running" + return event.data.state.state.kind === "busy" && + !(event.data.state.state.state.kind === "run" && + event.data.state.state.state.state === "paused") ? { ...emptyRunActivityStats(), startedAtMs: observedAtMs } : emptyRunActivityStats(); case "turn_start": diff --git a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte index c7852975..cf28f294 100644 --- a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte @@ -541,9 +541,42 @@ } } + let nextWorkerCommandId = 1; + + function lifecycleMethod( + command: "pause" | "cancel" | "resume" | "compact", + ): ProtocolMethod | null { + const state = consoleProjection.workerState; + if (!state) { + sendError = "Worker state snapshot is not available; reconnect before sending control."; + return null; + } + const commandId = Math.max( + nextWorkerCommandId, + state.last_command_id + 1, + ); + nextWorkerCommandId = commandId + 1; + const envelope = { + command_id: commandId, + expected_execution_generation: state.execution_generation, + expected_worker_state_revision: state.revision, + }; + switch (command) { + case "pause": + return { method: "pause", params: { command: envelope } }; + case "cancel": + return { method: "cancel", params: { command: envelope } }; + case "resume": + return { method: "resume", params: { command: envelope } }; + case "compact": + return { method: "compact", params: { command: envelope } }; + } + } + function sendWorkerControl(command: "pause" | "cancel" | "resume") { const label = command[0].toUpperCase() + command.slice(1); - sendControl({ method: command }, label); + const method = lifecycleMethod(command); + if (method) sendControl(method, label); } function isEditableTarget(target: EventTarget | null): boolean { @@ -627,8 +660,11 @@ auto_run: true, }, }; - case "compact": - return { method: "compact" }; + case "compact": { + const method = lifecycleMethod("compact"); + if (!method) throw new Error("Worker state snapshot is not available"); + return method; + } case "list_rewind_targets": return { method: "list_rewind_targets" }; case "register_peer": @@ -691,7 +727,7 @@ function handleComposerSubmit() { if (workerRunning) { - sendControl({ method: "cancel" }, "Stop"); + sendWorkerControl("cancel"); return; } void submitDraft(composerInputElement?.snapshot() ?? draft); @@ -894,8 +930,26 @@ ): string | null { switch (event.event) { case "snapshot": - case "status": - return event.data.status; + return event.data.state.state.kind === "idle" + ? "idle" + : event.data.state.state.state.kind === "run" && + event.data.state.state.state.state === "paused" + ? "paused" + : "running"; + case "worker_state": + return event.data.snapshot.state.kind === "idle" + ? "idle" + : event.data.snapshot.state.state.kind === "run" && + event.data.snapshot.state.state.state === "paused" + ? "paused" + : "running"; + case "command_acknowledged": + return event.data.acknowledgement.state.state.kind === "idle" + ? "idle" + : event.data.acknowledgement.state.state.state.kind === "run" && + event.data.acknowledgement.state.state.state.state === "paused" + ? "paused" + : "running"; case "shutdown": return "shutdown"; default: @@ -1620,7 +1674,10 @@ type="button" class="secondary-button" disabled={protocolState !== "open"} - onclick={() => sendControl({ method: "compact" }, "Compact")} + onclick={() => { + const method = lifecycleMethod("compact"); + if (method) sendControl(method, "Compact"); + }} > Compact From e3e9e83bc19dbb11115d8d23b03f90d4e749e9ed Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 07:22:33 +0900 Subject: [PATCH 29/39] feat: add guarded workspace deletion workflow --- crates/workspace-api/src/lib.rs | 154 +++- crates/workspace-server/src/latest_schema.sql | 23 + crates/workspace-server/src/lib.rs | 1 + crates/workspace-server/src/server.rs | 411 ++++++++++- crates/workspace-server/src/store.rs | 171 +++-- .../src/workspace_deletion.rs | 696 ++++++++++++++++++ .../src/lib/generated/workspace-api.ts | 68 ++ .../src/lib/workspace/api/workspace-model.ts | 215 +++++- .../settings/workspace-deletion-api.ts | 63 ++ .../settings/workspace/+page.svelte | 134 ++++ web/workspace/tests/workspace-model.test.ts | 87 +++ 11 files changed, 1940 insertions(+), 83 deletions(-) create mode 100644 crates/workspace-server/src/workspace_deletion.rs create mode 100644 web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts diff --git a/crates/workspace-api/src/lib.rs b/crates/workspace-api/src/lib.rs index a2cf8f87..1f043422 100644 --- a/crates/workspace-api/src/lib.rs +++ b/crates/workspace-api/src/lib.rs @@ -540,6 +540,7 @@ pub struct WorkspacePermissionSummary { pub manage_repositories: bool, pub manage_secrets: bool, pub manage_runtimes: bool, + pub delete_workspace: bool, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -606,6 +607,108 @@ pub struct WorkspaceMetadataMutationResponse { pub diagnostics: Vec, } +/// Lifecycle state for one durable Workspace deletion operation. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkspaceDeletionState { + Queued, + Running, + Blocked, + Failed, + Succeeded, +} + +/// Stable category explaining why Workspace deletion cannot currently advance. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkspaceDeletionBlockerKind { + LastAccessibleWorkspace, + RevisionConflict, + DirtyWorkdir, + WorkerRemovalBlocked, + WorkdirRemovalBlocked, + RetentionHold, + CleanupUnavailable, +} + +/// One bounded, user-actionable blocker returned by preflight or execution. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceDeletionBlocker { + pub kind: WorkspaceDeletionBlockerKind, + pub resource_kind: Option, + pub resource_key: Option, + pub message: String, +} + +/// Workspace-owned resources summarized before destructive confirmation. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceDeletionResourceCounts { + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub workers: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub workdirs: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub repositories: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub runtime_bindings: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub secrets: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub artifacts: u64, +} + +/// Owner-only impact preview for deleting one Workspace. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceDeletionPreflightResponse { + pub workspace_id: String, + pub display_name: String, + /// Opaque persisted Workspace metadata revision used as a CAS fence. + pub expected_revision: String, + pub can_delete: bool, + pub force_delete_dirty_workdirs_available: bool, + pub resources: WorkspaceDeletionResourceCounts, + pub blockers: Vec, +} + +/// Idempotent request to start or resume Workspace deletion. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceDeletionRequest { + pub operation_id: String, + pub expected_revision: String, + pub confirmation: String, + #[serde(default)] + pub force_delete_dirty_workdirs: bool, +} + +/// Durable deletion operation projection used by request responses and polling. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceDeletionOperationResponse { + pub operation_id: String, + pub workspace_id: String, + pub display_name: String, + pub state: WorkspaceDeletionState, + pub force_delete_dirty_workdirs: bool, + pub resources: WorkspaceDeletionResourceCounts, + pub child_operation_ids: Vec, + pub blockers: Vec, + pub failure_category: Option, + pub created_at: String, + pub updated_at: String, + pub completed_at: Option, +} + /// Read-only Profile catalog projected from one active Workspace config revision. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] @@ -2492,6 +2595,13 @@ pub fn catalog_typescript() -> String { WorkspaceCreateResponse::decl(&config), WorkspaceAuthConfig::decl(&config), WorkspacePermissionSummary::decl(&config), + WorkspaceDeletionState::decl(&config), + WorkspaceDeletionBlockerKind::decl(&config), + WorkspaceDeletionBlocker::decl(&config), + WorkspaceDeletionResourceCounts::decl(&config), + WorkspaceDeletionPreflightResponse::decl(&config), + WorkspaceDeletionRequest::decl(&config), + WorkspaceDeletionOperationResponse::decl(&config), DiagnosticSeverity::decl(&config), Diagnostic::decl(&config), WorkspaceExtensionPointState::decl(&config), @@ -3145,6 +3255,47 @@ mod tests { } } + #[test] + fn workspace_deletion_wire_contract_is_closed_and_typed() { + let preflight = WorkspaceDeletionPreflightResponse { + workspace_id: "workspace-test".to_string(), + display_name: "Test".to_string(), + expected_revision: "revision-7".to_string(), + can_delete: true, + force_delete_dirty_workdirs_available: true, + resources: WorkspaceDeletionResourceCounts { + workers: 2, + workdirs: 1, + repositories: 1, + runtime_bindings: 1, + secrets: 0, + artifacts: 3, + }, + blockers: Vec::new(), + }; + let value = serde_json::to_value(&preflight).unwrap(); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + preflight + ); + let mut stale = value.as_object().unwrap().clone(); + stale.insert("revision".to_string(), serde_json::json!(7)); + assert!( + serde_json::from_value::(stale.into()).is_err() + ); + + assert!( + serde_json::from_value::(serde_json::json!({ + "operation_id": "delete-test", + "expected_revision": "revision-7", + "confirmation": "delete Test", + "force_delete_dirty_workdirs": false, + "workspace_id": "caller-controlled" + })) + .is_err() + ); + } + #[test] fn workspace_and_repository_response_shapes_round_trip() { let workspace = serde_json::json!({ @@ -3161,7 +3312,8 @@ mod tests { "permissions": { "manage_repositories": true, "manage_secrets": true, - "manage_runtimes": true + "manage_runtimes": true, + "delete_workspace": true }, "extension_points": { "store": "sqlite", diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 720ff8ea..a93fe544 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -940,6 +940,29 @@ CREATE UNIQUE INDEX worker_workdir_links_active_worker_unique WHERE unlinked_at IS NULL; CREATE INDEX worker_workdir_links_workdir ON worker_workdir_links(workspace_id, workdir_id); +CREATE TABLE workspace_deletion_operations ( + operation_id TEXT PRIMARY KEY, + request_fingerprint TEXT NOT NULL, + workspace_id TEXT NOT NULL, + workspace_display_name TEXT NOT NULL, + workspace_revision TEXT NOT NULL, + owner_account_id TEXT NOT NULL, + actor_account_id TEXT NOT NULL, + force_delete_dirty_workdirs INTEGER NOT NULL CHECK(force_delete_dirty_workdirs IN (0, 1)), + state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')), + resource_counts_json TEXT NOT NULL, + child_operation_ids_json TEXT NOT NULL, + blockers_json TEXT NOT NULL, + failure_category TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + completed_at TEXT, + FOREIGN KEY(owner_account_id) REFERENCES accounts(account_id) ON DELETE RESTRICT, + FOREIGN KEY(actor_account_id) REFERENCES accounts(account_id) ON DELETE RESTRICT +); +CREATE INDEX workspace_deletion_operations_workspace_recent + ON workspace_deletion_operations(workspace_id, created_at DESC); + CREATE TRIGGER seed_worker_retention_policy_after_workspace_insert AFTER INSERT ON workspaces BEGIN INSERT INTO workspace_worker_retention_policy_revisions (workspace_id,policy_id,revision,session_disposition,metadata_disposition,archive_retention_kind,archive_retention_seconds,diagnostics_disposition,diagnostics_retention_seconds,created_at) diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index 496750a1..f95318f2 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -33,6 +33,7 @@ pub mod workdir_create_operations; mod workdir_removal; pub mod worker_source; pub mod workspace_catalog; +mod workspace_deletion; mod workspace_subscription; pub use authority::{ diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 41db9ae9..6cc419e8 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -89,8 +89,10 @@ use workspace_api::{ WorkingDirectoryListResponse as BrowserWorkingDirectoryListResponse, WorkingDirectoryRemovalDisposition, WorkingDirectoryRemovalRequest, WorkingDirectoryRemovalResponse, WorkingDirectoryRepositoryOption, - WorkspaceCatalogListResponse, WorkspaceCreateResponse, WorkspaceExtensionPointState, - WorkspaceExtensionPoints, WorkspaceMetadataMutationResponse, WorkspaceMetadataSettingsResponse, + WorkspaceCatalogListResponse, WorkspaceCreateResponse, WorkspaceDeletionBlocker, + WorkspaceDeletionBlockerKind, WorkspaceDeletionOperationResponse, WorkspaceDeletionRequest, + WorkspaceDeletionState, WorkspaceExtensionPointState, WorkspaceExtensionPoints, + WorkspaceMetadataMutationResponse, WorkspaceMetadataSettingsResponse, WorkspacePermissionSummary, WorkspaceRepositoryRecord, WorkspaceResponse, WorkspaceRuntimeDetail, WorkspaceRuntimeResource, WorkspaceSummary, WorkspaceWorkerDiscoveryItem, WorkspaceWorkerDiscoveryPage, WorkspaceWorkerSubject, @@ -744,6 +746,17 @@ impl WorkspaceWorkerRemoveExecutor { )); } + self.execute_target_removal(&runtime, &target, reason, false) + .await + } + + async fn execute_target_removal( + &self, + runtime: &RuntimeRegistry, + target: &RuntimeWorkerRef, + reason: &str, + allow_internal: bool, + ) -> std::result::Result { let remove_lock = { let mut locks = self .worker_remove_locks @@ -830,7 +843,7 @@ impl WorkspaceWorkerRemoveExecutor { )); } return self - .resume_worker_retention(&runtime, &target, prepared) + .resume_worker_retention(runtime, &target, prepared) .await; } @@ -844,7 +857,7 @@ impl WorkspaceWorkerRemoveExecutor { )); } }; - if worker.singleton_key.is_some() { + if worker.singleton_key.is_some() && !allow_internal { return Ok(worker_remove_error_response( StatusCode::CONFLICT, "internal_worker_forbidden", @@ -1008,6 +1021,133 @@ impl WorkspaceServerApi { } } + async fn execute_workspace_deletion( + &self, + operation_id: &str, + ) -> Result { + let operation = self.store.update_workspace_deletion_operation( + operation_id, + WorkspaceDeletionState::Running, + &[], + &[], + None, + )?; + let workspace = self + .store + .get_workspace(&operation.workspace_id) + .await? + .ok_or_else(|| Error::InvalidInput("Workspace no longer exists".to_string()))?; + let repositories = self.store.list_repositories(&operation.workspace_id)?; + let config = self + .template + .for_catalog_workspace(&workspace, repositories)?; + let api = WorkspaceApi::new(config, self.store.clone()).await?; + self.store + .release_workspace_assignments_for_deletion(&operation.workspace_id)?; + + let mut child_operation_ids = Vec::new(); + let mut blockers = Vec::new(); + for worker in self + .store + .list_worker_registry(&operation.workspace_id, 10_000)? + { + let worker_key = worker.display_name.clone(); + let target = worker.worker; + let lifecycle = WorkerLifecycleRequest { + reason: Some("Workspace deletion".to_string()), + ticket_assignment: None, + }; + let _ = api.runtime.cancel_worker(&target, lifecycle.clone()); + if api.runtime.stop_worker(&target, lifecycle).is_err() { + blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, + resource_kind: Some("worker".to_string()), + resource_key: Some(worker_key.clone()), + message: "Worker stop did not reach a retryable terminal state.".to_string(), + }); + continue; + } + let response = WorkspaceWorkerRemoveExecutor::new(&api) + .execute_target_removal(api.runtime.as_ref(), &target, "Workspace deletion", true) + .await + .map_err(Error::Store)?; + if let Some(child_operation_id) = self.store.latest_worker_removal_operation_id( + &operation.workspace_id, + &target.runtime_id, + &target.worker_id, + )? { + child_operation_ids.push(child_operation_id); + } + if response.status != 200 { + blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, + resource_kind: Some("worker".to_string()), + resource_key: Some(worker_key.clone()), + message: "Worker retention or removal policy blocked deletion.".to_string(), + }); + } + } + + if blockers.is_empty() { + for workdir in self + .store + .list_workdir_registry(&operation.workspace_id, 10_000)? + { + match execute_workdir_removal_for_workspace_deletion( + &api, + &workdir.workdir_id, + operation_id, + operation.force_delete_dirty_workdirs, + ) { + Ok(child) => { + child_operation_ids.push(child.operation_id.clone()); + if child.state != WorkdirRemovalOperationState::Completed + || child.disposition != Some(WorkdirRemovalDisposition::Removed) + { + let dirty = + child.failure_category.as_deref() == Some("dirty_or_unknown"); + blockers.push(WorkspaceDeletionBlocker { + kind: if dirty { + WorkspaceDeletionBlockerKind::DirtyWorkdir + } else { + WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked + }, + resource_kind: Some("workdir".to_string()), + resource_key: Some(workdir.workdir_id), + message: if dirty { + "Workdir is dirty or its cleanliness is unknown. Enable force deletion only after reviewing the impact." + .to_string() + } else { + "Workdir removal did not complete; retry the Workspace deletion operation." + .to_string() + }, + }); + } + } + Err(error) => blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked, + resource_kind: Some("workdir".to_string()), + resource_key: Some(workdir.workdir_id), + message: format!("Workdir removal failed: {error}"), + }), + } + } + } + + if !blockers.is_empty() { + return self.store.update_workspace_deletion_operation( + operation_id, + WorkspaceDeletionState::Blocked, + &child_operation_ids, + &blockers, + None, + ); + } + let completed = self.store.finalize_workspace_deletion(operation_id)?; + self.routers.lock().await.remove(&completed.workspace_id); + Ok(completed) + } + async fn router_for_workspace(&self, workspace_id: &str) -> Result> { let mut routers = self.routers.lock().await; if let Some(router) = routers.get(workspace_id) { @@ -1118,6 +1258,107 @@ async fn create_server_workspace( (status, Json(workspace_create_response(created))).into_response() } +async fn preflight_server_workspace_deletion( + State(api): State, + AxumPath(workspace_id): AxumPath, + headers: HeaderMap, +) -> Response { + let actor_account_id = match resolve_server_actor(&api, &headers).await { + Ok(Some(actor)) => actor.account_id, + Ok(None) => return forbidden_server_response("Workspace deletion requires its owner"), + Err(error) => return server_error_response(error), + }; + match api + .store + .workspace_deletion_preflight(&actor_account_id, &workspace_id) + { + Ok(preflight) => Json(preflight).into_response(), + Err(error) => server_error_response(error), + } +} + +async fn start_server_workspace_deletion( + State(api): State, + AxumPath(workspace_id): AxumPath, + headers: HeaderMap, + Json(request): Json, +) -> Response { + let actor_account_id = match resolve_server_actor(&api, &headers).await { + Ok(Some(actor)) => actor.account_id, + Ok(None) => return forbidden_server_response("Workspace deletion requires its owner"), + Err(error) => return server_error_response(error), + }; + let reservation = + match api + .store + .reserve_workspace_deletion(&actor_account_id, &workspace_id, &request) + { + Ok(reservation) => reservation, + Err(error) => return server_error_response(error), + }; + let operation = + if reservation.replay && reservation.operation.state == WorkspaceDeletionState::Succeeded { + reservation.operation + } else { + match api.execute_workspace_deletion(&request.operation_id).await { + Ok(operation) => operation, + Err(error) => { + let _ = api.store.update_workspace_deletion_operation( + &request.operation_id, + WorkspaceDeletionState::Failed, + &reservation.operation.child_operation_ids, + &[], + Some("workspace_deletion_execution_failed"), + ); + return server_error_response(error); + } + } + }; + let status = if operation.state == WorkspaceDeletionState::Succeeded { + StatusCode::OK + } else { + StatusCode::ACCEPTED + }; + (status, Json(operation)).into_response() +} + +async fn get_server_workspace_deletion( + State(api): State, + AxumPath(operation_id): AxumPath, + headers: HeaderMap, +) -> Response { + let actor_account_id = match resolve_server_actor(&api, &headers).await { + Ok(Some(actor)) => actor.account_id, + Ok(None) => { + return forbidden_server_response("Workspace deletion status requires its owner"); + } + Err(error) => return server_error_response(error), + }; + match api + .store + .workspace_deletion_operation(&actor_account_id, &operation_id) + { + Ok(Some(operation)) + if matches!( + operation.state, + WorkspaceDeletionState::Queued | WorkspaceDeletionState::Running + ) => + { + match api.execute_workspace_deletion(&operation_id).await { + Ok(operation) => Json(operation).into_response(), + Err(error) => server_error_response(error), + } + } + Ok(Some(operation)) => Json(operation).into_response(), + Ok(None) => ( + StatusCode::NOT_FOUND, + "Workspace deletion operation not found", + ) + .into_response(), + Err(error) => server_error_response(error), + } +} + fn workspace_summary(record: WorkspaceRecord) -> WorkspaceSummary { WorkspaceSummary { workspace_id: record.workspace_id, @@ -1217,6 +1458,22 @@ async fn authorize_scoped_workspace_request( .await .map_err(|_| StatusCode::UNAUTHORIZED.into_response())?; request.extensions_mut().insert(source); + if !matches!( + *request.method(), + Method::GET | Method::HEAD | Method::OPTIONS + ) && !api + .store + .get_workspace(workspace_id) + .await + .map_err(server_error_response)? + .is_some_and(|workspace| workspace.state == "active") + { + return Err(( + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ) + .into_response()); + } return Ok(()); } @@ -1243,6 +1500,20 @@ async fn authorize_scoped_workspace_request( } } request.extensions_mut().insert(actor); + if mutating + && !api + .store + .get_workspace(workspace_id) + .await + .map_err(server_error_response)? + .is_some_and(|workspace| workspace.state == "active") + { + return Err(( + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ) + .into_response()); + } Ok(()) } @@ -1297,6 +1568,23 @@ async fn authorize_workspace_api_request( return StatusCode::UNAUTHORIZED.into_response(); }; request.extensions_mut().insert(source); + if !matches!( + *request.method(), + Method::GET | Method::HEAD | Method::OPTIONS + ) && !api + .store + .get_workspace(&workspace_id) + .await + .ok() + .flatten() + .is_some_and(|workspace| workspace.state == "active") + { + return ( + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ) + .into_response(); + } return next.run(request).await; } @@ -1330,6 +1618,21 @@ async fn authorize_workspace_api_request( } } request.extensions_mut().insert(actor); + if mutating + && !api + .store + .get_workspace(&workspace_id) + .await + .ok() + .flatten() + .is_some_and(|workspace| workspace.state == "active") + { + return ( + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ) + .into_response(); + } next.run(request).await } @@ -1482,6 +1785,14 @@ pub async fn build_workspace_server_router( "/api/workspaces", get(list_server_workspaces).post(create_server_workspace), ) + .route( + "/api/workspaces/{workspace_id}/deletion", + get(preflight_server_workspace_deletion).post(start_server_workspace_deletion), + ) + .route( + "/api/workspace-deletions/{operation_id}", + get(get_server_workspace_deletion), + ) .fallback(dispatch_workspace_request) .with_state(api.clone()); Ok(auth @@ -10209,7 +10520,27 @@ fn execute_reserved_workdir_removal( operation: WorkdirRemovalOperation, recovery: bool, ) -> Result { - execute_reserved_workdir_removal_with_provider(api, operation, recovery, api.runtime.as_ref()) + execute_reserved_workdir_removal_with_provider( + api, + operation, + recovery, + api.runtime.as_ref(), + false, + ) +} + +fn execute_reserved_workdir_removal_for_workspace_deletion( + api: &WorkspaceApi, + operation: WorkdirRemovalOperation, + force_dirty: bool, +) -> Result { + execute_reserved_workdir_removal_with_provider( + api, + operation, + false, + api.runtime.as_ref(), + force_dirty, + ) } fn execute_reserved_workdir_removal_with_provider( @@ -10217,6 +10548,7 @@ fn execute_reserved_workdir_removal_with_provider( operation: WorkdirRemovalOperation, recovery: bool, provider: &dyn WorkdirRemovalRuntimeProvider, + force_dirty: bool, ) -> Result { if operation.state == WorkdirRemovalOperationState::Completed { return Ok(operation); @@ -10291,8 +10623,9 @@ fn execute_reserved_workdir_removal_with_provider( true, ); }; - if status.summary.cleanliness.as_deref() != Some("clean") - || status.summary.status != WorkingDirectoryStatusKind::Active + if !force_dirty + && (status.summary.cleanliness.as_deref() != Some("clean") + || status.summary.status != WorkingDirectoryStatusKind::Active) { return api.config_store.complete_workdir_removal_retained( &operation, @@ -10386,6 +10719,40 @@ fn execute_workdir_removal( .map(|operation| workdir_removal_response(&operation)) } +fn execute_workdir_removal_for_workspace_deletion( + api: &WorkspaceApi, + working_directory_id: &str, + parent_operation_id: &str, + force_dirty: bool, +) -> Result { + let source_actor = format!("workspace-deletion:{parent_operation_id}"); + let reason = "Workspace deletion"; + let lock = workdir_removal_execution_lock(api, working_directory_id)?; + let _guard = lock + .lock() + .map_err(|_| Error::Store("Workdir removal lock was poisoned".to_string()))?; + let operation = if let Some(existing) = + api.config_store.find_workdir_removal_operation_by_intent( + api.workspace_id(), + working_directory_id, + &source_actor, + reason, + )? { + existing + } else { + let workdir = api + .config_store + .get_workdir_registry(api.workspace_id(), working_directory_id)? + .ok_or_else(|| { + Error::InvalidInput(format!("Unknown Workdir `{working_directory_id}`")) + })?; + let intent = workdir_removal_intent(&workdir, &source_actor, reason)?; + api.config_store + .reserve_workdir_removal_operation(&intent)? + }; + execute_reserved_workdir_removal_for_workspace_deletion(api, operation, force_dirty) +} + fn recover_workdir_removals(api: &WorkspaceApi) -> Result<()> { for operation in api .config_store @@ -12380,6 +12747,7 @@ async fn get_workspace( manage_repositories: is_owner, manage_secrets: is_owner, manage_runtimes: is_owner, + delete_workspace: is_owner, }, extension_points: WorkspaceExtensionPoints { store: "sqlite".to_string(), @@ -24003,6 +24371,7 @@ mod tests { clean_operation.clone(), false, &clean_provider, + false, ) .unwrap(); assert_eq!( @@ -24015,6 +24384,7 @@ mod tests { removed.clone(), false, &clean_provider, + false, ) .unwrap(); assert_eq!(replay, removed); @@ -24039,6 +24409,7 @@ mod tests { missing_operation, false, &missing_provider, + false, ) .unwrap(); assert_eq!( @@ -24065,6 +24436,7 @@ mod tests { unknown_operation, false, &unknown_provider, + false, ) .unwrap(); assert_eq!(unknown.state, WorkdirRemovalOperationState::Failed); @@ -24086,11 +24458,34 @@ mod tests { dirty_operation, false, &dirty_provider, + false, ) .unwrap(); assert_eq!(dirty.disposition, Some(WorkdirRemovalDisposition::Retained)); assert_eq!(dirty_provider.cleanup_calls(), 0); + let (forced_operation, mut forced_summary) = + reserve_removal_fixture(&api, "provider-dirty-forced"); + forced_summary.cleanliness = Some("dirty".to_string()); + let forced_provider = FakeWorkdirRemovalProvider::new( + workdir_removal_result( + WorkerOperationState::Accepted, + Some(forced_summary), + Vec::new(), + ), + workdir_removal_result(WorkerOperationState::Accepted, None, Vec::new()), + ); + let forced = execute_reserved_workdir_removal_with_provider( + &api, + forced_operation, + false, + &forced_provider, + true, + ) + .unwrap(); + assert_eq!(forced.disposition, Some(WorkdirRemovalDisposition::Removed)); + assert_eq!(forced_provider.cleanup_calls(), 1); + let (unsupported_operation, unsupported_summary) = reserve_removal_fixture(&api, "provider-unsupported"); let unsupported_provider = FakeWorkdirRemovalProvider::new( @@ -24114,6 +24509,7 @@ mod tests { unsupported_operation, false, &unsupported_provider, + false, ) .unwrap(); assert_eq!(unsupported.state, WorkdirRemovalOperationState::Failed); @@ -24168,6 +24564,7 @@ mod tests { operation, true, provider.as_ref(), + false, ) })); } diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 0c7039b4..d7a7263f 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -13,11 +13,12 @@ use uuid::Uuid; use worker_runtime::identity::{RuntimeWorkerRef, WorkerId}; use workspace_api::{RepositoryObservedStatus, RepositorySource}; +use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; -const PREVIOUS_SCHEMA_VERSION: i64 = 51; -const LATEST_SCHEMA_VERSION: i64 = 52; -const RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace Runtime binding revision and audit"; +const PREVIOUS_SCHEMA_VERSION: i64 = 52; +const LATEST_SCHEMA_VERSION: i64 = 53; +const WORKSPACE_DELETION_MIGRATION_NAME: &str = "durable Workspace deletion operations"; const MIGRATIONS: &[Migration] = &[Migration { version: LATEST_SCHEMA_VERSION, @@ -546,7 +547,7 @@ impl WorkspaceResourceKind { } #[async_trait] -pub trait ControlPlaneStore: Send + Sync { +pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { async fn schema_version(&self) -> Result; fn resource_key( &self, @@ -1133,6 +1134,18 @@ impl SqliteWorkspaceStore { f(&mut conn) } + pub(crate) fn with_transaction( + &self, + f: impl FnOnce(&rusqlite::Transaction<'_>) -> Result, + ) -> Result { + self.with_conn_mut(|conn| { + let tx = conn.transaction()?; + let value = f(&tx)?; + tx.commit()?; + Ok(value) + }) + } + pub(crate) fn get_workspace_memory_settings( &self, workspace_id: &str, @@ -6406,46 +6419,83 @@ CREATE TABLE IF NOT EXISTS __yoi_schema_migrations ( Ok(()) } -fn migrate_workspace_runtime_bindings_v51_to_v52(conn: &Connection) -> Result<()> { +fn migrate_workspace_deletion_v52_to_v53(conn: &Connection) -> Result<()> { let current = current_schema_version(conn)?; if current != PREVIOUS_SCHEMA_VERSION { return Err(Error::Store(format!( - "expected schema version {PREVIOUS_SCHEMA_VERSION} before {RUNTIME_BINDINGS_MIGRATION_NAME} migration, found {current}" + "expected schema version {PREVIOUS_SCHEMA_VERSION} before {WORKSPACE_DELETION_MIGRATION_NAME} migration, found {current}" ))); } let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; tx.execute_batch( r#" - ALTER TABLE workspace_runtime_bindings - ADD COLUMN binding_revision INTEGER NOT NULL DEFAULT 1 CHECK (binding_revision > 0); - CREATE TABLE workspace_runtime_binding_audit ( + CREATE TABLE workspace_deletion_operations ( + operation_id TEXT PRIMARY KEY, + request_fingerprint TEXT NOT NULL, workspace_id TEXT NOT NULL, - runtime_id TEXT NOT NULL, + workspace_display_name TEXT NOT NULL, + workspace_revision TEXT NOT NULL, + owner_account_id TEXT NOT NULL, actor_account_id TEXT NOT NULL, - action TEXT NOT NULL CHECK (action IN ('created', 'replaced', 'reactivated', 'revoked')), - old_fingerprint TEXT, - new_fingerprint TEXT, - binding_revision INTEGER NOT NULL CHECK (binding_revision > 0), - at TEXT NOT NULL, - PRIMARY KEY (workspace_id, runtime_id, binding_revision), - FOREIGN KEY(workspace_id, runtime_id) - REFERENCES workspace_runtime_bindings(workspace_id, runtime_id) ON DELETE RESTRICT, + force_delete_dirty_workdirs INTEGER NOT NULL CHECK(force_delete_dirty_workdirs IN (0, 1)), + state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')), + resource_counts_json TEXT NOT NULL, + child_operation_ids_json TEXT NOT NULL, + blockers_json TEXT NOT NULL, + failure_category TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + completed_at TEXT, + FOREIGN KEY(owner_account_id) REFERENCES accounts(account_id) ON DELETE RESTRICT, FOREIGN KEY(actor_account_id) REFERENCES accounts(account_id) ON DELETE RESTRICT ); - CREATE INDEX idx_workspace_runtime_binding_audit_recent - ON workspace_runtime_binding_audit(workspace_id, runtime_id, binding_revision DESC); + CREATE INDEX workspace_deletion_operations_workspace_recent + ON workspace_deletion_operations(workspace_id, created_at DESC); "#, )?; - verify_workspace_runtime_binding_schema(&tx)?; + verify_workspace_deletion_schema(&tx)?; tx.execute( "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)", - params![LATEST_SCHEMA_VERSION, RUNTIME_BINDINGS_MIGRATION_NAME], + params![LATEST_SCHEMA_VERSION, WORKSPACE_DELETION_MIGRATION_NAME], )?; tx.commit()?; Ok(()) } +fn verify_workspace_deletion_schema(conn: &Connection) -> Result<()> { + let columns = table_columns(conn, "workspace_deletion_operations")? + .into_iter() + .collect::>(); + let expected = [ + "operation_id", + "request_fingerprint", + "workspace_id", + "workspace_display_name", + "workspace_revision", + "owner_account_id", + "actor_account_id", + "force_delete_dirty_workdirs", + "state", + "resource_counts_json", + "child_operation_ids_json", + "blockers_json", + "failure_category", + "created_at", + "updated_at", + "completed_at", + ] + .into_iter() + .map(str::to_string) + .collect::>(); + if columns != expected { + return Err(Error::Store( + "workspace_deletion_operations schema does not match schema-53".to_string(), + )); + } + Ok(()) +} + fn verify_workspace_runtime_binding_schema(conn: &Connection) -> Result<()> { let columns = table_columns(conn, "workspace_runtime_bindings")? .into_iter() @@ -7034,7 +7084,7 @@ fn verify_current_schema_history(conn: &Connection) -> Result<()> { ), ( LATEST_SCHEMA_VERSION, - RUNTIME_BINDINGS_MIGRATION_NAME.to_string(), + WORKSPACE_DELETION_MIGRATION_NAME.to_string(), ), ]; if rows != fresh && rows != upgraded { @@ -7077,17 +7127,20 @@ fn apply_migrations(conn: &Connection) -> Result<()> { params![baseline.version, baseline.name], )?; tx.commit()?; - verify_workspace_runtime_binding_schema(conn) + verify_workspace_runtime_binding_schema(conn)?; + verify_workspace_deletion_schema(conn) } PREVIOUS_SCHEMA_VERSION => { verify_previous_schema_history(conn)?; - migrate_workspace_runtime_bindings_v51_to_v52(conn)?; + migrate_workspace_deletion_v52_to_v53(conn)?; verify_current_schema_history(conn)?; - verify_workspace_runtime_binding_schema(conn) + verify_workspace_runtime_binding_schema(conn)?; + verify_workspace_deletion_schema(conn) } LATEST_SCHEMA_VERSION => { verify_current_schema_history(conn)?; - verify_workspace_runtime_binding_schema(conn) + verify_workspace_runtime_binding_schema(conn)?; + verify_workspace_deletion_schema(conn) } version if version > LATEST_SCHEMA_VERSION => Err(Error::Store(format!( "database schema version {version} is newer than this server supports ({LATEST_SCHEMA_VERSION}); refusing to serve with an older binary" @@ -7156,7 +7209,7 @@ mod tests { .unwrap(); } - fn prepare_schema_v51(path: &Path) { + fn prepare_schema_v52(path: &Path) { let conn = Connection::open(path).unwrap(); configure_sqlite(&conn).unwrap(); ticket::migrate_sqlite_ticket_schema(&conn).unwrap(); @@ -7164,56 +7217,27 @@ mod tests { create_latest_workspace_schema(&conn).unwrap(); conn.execute_batch( r#" - DROP INDEX idx_workspace_runtime_binding_audit_recent; - DROP TABLE workspace_runtime_binding_audit; - ALTER TABLE workspace_runtime_bindings DROP COLUMN binding_revision; + DROP INDEX workspace_deletion_operations_workspace_recent; + DROP TABLE workspace_deletion_operations; DELETE FROM __yoi_schema_migrations; INSERT INTO __yoi_schema_migrations(version, name) - VALUES (51, 'workspace schema baseline'); - INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at) - VALUES ('owner', 'user', 'owner', 'Owner', '1', '1'); - INSERT INTO workspaces( - workspace_id, owner_account_id, display_name, state, created_at, updated_at - ) VALUES ('workspace-a', 'owner', 'Workspace A', 'active', '1', '1'); + VALUES (52, 'workspace schema baseline'); "#, ) .unwrap(); - let identity = - worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-a").unwrap(); - let (_, fingerprint) = normalize_runtime_public_key(&identity.public_key).unwrap(); - conn.execute( - r#"INSERT INTO workspace_runtime_bindings( - workspace_id, runtime_id, display_name, base_url, public_key, - public_key_fingerprint, created_at, updated_at, revoked_at - ) VALUES ('workspace-a', 'runtime-a', 'Runtime A', 'https://runtime.test', - ?1, ?2, '1', '1', NULL)"#, - params![identity.public_key, fingerprint], - ) - .unwrap(); } #[test] - fn schema_v51_runtime_binding_migrates_with_revision_and_empty_audit() { + fn schema_v52_migrates_workspace_deletion_operations() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - prepare_schema_v51(&path); + prepare_schema_v52(&path); let store = SqliteWorkspaceStore::open(&path).unwrap(); - let binding = store - .get_workspace_runtime_binding("workspace-a", "runtime-a") - .unwrap() - .unwrap(); - assert_eq!(binding.binding_revision, 1); - assert!( - store - .list_workspace_runtime_binding_audit("workspace-a", "runtime-a", 50) - .unwrap() - .is_empty() - ); store .with_conn(|conn| { - let version = current_schema_version(conn)?; - assert_eq!(version, LATEST_SCHEMA_VERSION); + assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); + verify_workspace_deletion_schema(conn)?; let violations: i64 = conn.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |row| { row.get(0) @@ -7225,13 +7249,13 @@ mod tests { } #[test] - fn schema_v51_runtime_binding_migration_rolls_back_all_changes_on_failure() { + fn schema_v52_workspace_deletion_migration_rolls_back_on_failure() { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); - prepare_schema_v51(&path); + prepare_schema_v52(&path); let conn = Connection::open(&path).unwrap(); conn.execute_batch( - "CREATE TABLE workspace_runtime_binding_audit (unexpected TEXT NOT NULL);", + "CREATE TABLE workspace_deletion_operations (unexpected TEXT NOT NULL);", ) .unwrap(); drop(conn); @@ -7242,12 +7266,11 @@ mod tests { .to_string(); assert!(error.contains("already exists"), "{error}"); let conn = Connection::open(&path).unwrap(); - assert!( - !table_columns(&conn, "workspace_runtime_bindings") - .unwrap() - .contains(&"binding_revision".to_string()) + assert_eq!( + table_columns(&conn, "workspace_deletion_operations").unwrap(), + vec!["unexpected".to_string()] ); - assert_eq!(current_schema_version(&conn).unwrap(), 51); + assert_eq!(current_schema_version(&conn).unwrap(), 52); } #[test] @@ -8536,13 +8559,13 @@ INSERT INTO worker_registry ( let conn = Connection::open_in_memory().unwrap(); configure_sqlite(&conn).unwrap(); conn.execute( - "INSERT INTO __yoi_schema_migrations (version, name) VALUES (53, 'future')", + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (54, 'future')", [], ) .unwrap(); let error = apply_migrations(&conn).unwrap_err().to_string(); - assert!(error.contains("schema version 53 is newer"), "{error}"); + assert!(error.contains("schema version 54 is newer"), "{error}"); assert!(error.contains("refusing to serve"), "{error}"); } diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs new file mode 100644 index 00000000..da90cb31 --- /dev/null +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -0,0 +1,696 @@ +use chrono::Utc; +use rusqlite::{OptionalExtension, params}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use workspace_api::{ + WorkspaceDeletionBlocker, WorkspaceDeletionBlockerKind, WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, WorkspaceDeletionRequest, WorkspaceDeletionResourceCounts, + WorkspaceDeletionState, +}; + +use crate::store::{SqliteWorkspaceStore, WorkspaceRecord}; +use crate::{Error, Result}; + +const MAX_OPERATION_ID_BYTES: usize = 128; +const CONFIRMATION_PREFIX: &str = "delete "; + +#[derive(Debug, Clone)] +pub struct WorkspaceDeletionReservation { + pub operation: WorkspaceDeletionOperationResponse, + pub replay: bool, +} + +pub trait WorkspaceDeletionStore: Send + Sync { + fn workspace_deletion_preflight( + &self, + actor_account_id: &str, + workspace_id: &str, + ) -> Result; + + fn reserve_workspace_deletion( + &self, + actor_account_id: &str, + workspace_id: &str, + request: &WorkspaceDeletionRequest, + ) -> Result; + + fn workspace_deletion_operation( + &self, + actor_account_id: &str, + operation_id: &str, + ) -> Result>; + + fn release_workspace_assignments_for_deletion(&self, workspace_id: &str) -> Result; + + fn latest_worker_removal_operation_id( + &self, + workspace_id: &str, + runtime_id: &str, + worker_id: &str, + ) -> Result>; + + fn update_workspace_deletion_operation( + &self, + operation_id: &str, + state: WorkspaceDeletionState, + child_operation_ids: &[String], + blockers: &[WorkspaceDeletionBlocker], + failure_category: Option<&str>, + ) -> Result; + + fn finalize_workspace_deletion( + &self, + operation_id: &str, + ) -> Result; +} + +impl WorkspaceDeletionStore for SqliteWorkspaceStore { + fn workspace_deletion_preflight( + &self, + actor_account_id: &str, + workspace_id: &str, + ) -> Result { + self.with_conn(|conn| { + let workspace = owner_workspace(conn, actor_account_id, workspace_id)?; + let resources = resource_counts(conn, workspace_id)?; + let accessible: u64 = conn.query_row( + "SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1 AND state = 'active'", + params![actor_account_id], + |row| row.get(0), + )?; + let mut blockers = Vec::new(); + if accessible <= 1 { + blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::LastAccessibleWorkspace, + resource_kind: None, + resource_key: None, + message: "You cannot delete your last accessible Workspace.".to_string(), + }); + } + Ok(WorkspaceDeletionPreflightResponse { + workspace_id: workspace.workspace_id, + display_name: workspace.display_name, + expected_revision: workspace.updated_at, + can_delete: blockers.is_empty(), + force_delete_dirty_workdirs_available: true, + resources, + blockers, + }) + }) + } + + fn reserve_workspace_deletion( + &self, + actor_account_id: &str, + workspace_id: &str, + request: &WorkspaceDeletionRequest, + ) -> Result { + validate_operation_id(&request.operation_id)?; + self.with_transaction(|tx| { + if let Some(existing) = read_operation(tx, &request.operation_id)? { + if existing.actor_account_id != actor_account_id { + return Err(Error::WorkspacePermissionDenied( + "Workspace deletion operation is not owned by the current account".to_string(), + )); + } + let expected_fingerprint = request_fingerprint(actor_account_id, workspace_id, request); + if existing.request_fingerprint != expected_fingerprint { + return Err(Error::WorkspaceConfigConflict( + "Workspace deletion operation_id was reused with different intent".to_string(), + )); + } + return Ok(WorkspaceDeletionReservation { + operation: existing.response, + replay: true, + }); + } + + let workspace = owner_workspace(tx, actor_account_id, workspace_id)?; + let expected_confirmation = format!("{CONFIRMATION_PREFIX}{}", workspace.display_name); + if request.confirmation != expected_confirmation { + return Err(Error::InvalidInput( + "confirmation must exactly match the displayed Workspace name".to_string(), + )); + } + if workspace.updated_at != request.expected_revision { + return Err(Error::WorkspaceConfigConflict( + "Workspace metadata changed; reload deletion impact before confirming".to_string(), + )); + } + let accessible: u64 = tx.query_row( + "SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1 AND state = 'active'", + params![actor_account_id], + |row| row.get(0), + )?; + if accessible <= 1 { + return Err(Error::WorkspaceConfigConflict( + "last_accessible_workspace: create or retain another accessible Workspace first" + .to_string(), + )); + } + + let resources = resource_counts(tx, workspace_id)?; + let now = Utc::now().to_rfc3339(); + let fingerprint = request_fingerprint(actor_account_id, workspace_id, request); + tx.execute( + "INSERT INTO workspace_deletion_operations ( + operation_id, request_fingerprint, workspace_id, workspace_display_name, + workspace_revision, owner_account_id, actor_account_id, + force_delete_dirty_workdirs, state, resource_counts_json, + child_operation_ids_json, blockers_json, failure_category, + created_at, updated_at, completed_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'queued', ?9, '[]', '[]', NULL, ?10, ?10, NULL)", + params![ + request.operation_id, + fingerprint, + workspace_id, + workspace.display_name, + request.expected_revision, + workspace.owner_account_id, + actor_account_id, + request.force_delete_dirty_workdirs as i64, + serde_json::to_string(&resources).map_err(|error| Error::Store(error.to_string()))?, + now, + ], + )?; + let changed = tx.execute( + "UPDATE workspaces SET state = 'deleting', updated_at = ?2 + WHERE workspace_id = ?1 AND updated_at = ?3 AND state = 'active'", + params![workspace_id, now, request.expected_revision], + )?; + if changed != 1 { + return Err(Error::WorkspaceConfigConflict( + "Workspace lifecycle changed before deletion could be reserved".to_string(), + )); + } + let operation = read_operation(tx, &request.operation_id)? + .ok_or_else(|| Error::Store("reserved Workspace deletion operation disappeared".to_string()))?; + Ok(WorkspaceDeletionReservation { + operation: operation.response, + replay: false, + }) + }) + } + + fn workspace_deletion_operation( + &self, + actor_account_id: &str, + operation_id: &str, + ) -> Result> { + self.with_conn(|conn| { + let Some(operation) = read_operation(conn, operation_id)? else { + return Ok(None); + }; + if operation.actor_account_id != actor_account_id { + return Err(Error::WorkspacePermissionDenied( + "Workspace deletion operation is not owned by the current account".to_string(), + )); + } + Ok(Some(operation.response)) + }) + } + + fn release_workspace_assignments_for_deletion(&self, workspace_id: &str) -> Result { + self.with_conn(|conn| { + let changed = conn.execute( + "DELETE FROM ticket_current_worker_assignments WHERE workspace_id = ?1", + params![workspace_id], + )?; + u64::try_from(changed) + .map_err(|_| Error::Store("assignment deletion count overflow".to_string())) + }) + } + + fn latest_worker_removal_operation_id( + &self, + workspace_id: &str, + runtime_id: &str, + worker_id: &str, + ) -> Result> { + self.with_conn(|conn| { + conn.query_row( + "SELECT operation_id FROM worker_removal_operations + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + ORDER BY created_at DESC LIMIT 1", + params![workspace_id, runtime_id, worker_id], + |row| row.get(0), + ) + .optional() + .map_err(Into::into) + }) + } + + fn update_workspace_deletion_operation( + &self, + operation_id: &str, + state: WorkspaceDeletionState, + child_operation_ids: &[String], + blockers: &[WorkspaceDeletionBlocker], + failure_category: Option<&str>, + ) -> Result { + self.with_transaction(|tx| { + let now = Utc::now().to_rfc3339(); + let completed_at = + matches!(state, WorkspaceDeletionState::Succeeded).then_some(now.as_str()); + let changed = tx.execute( + "UPDATE workspace_deletion_operations + SET state = ?2, child_operation_ids_json = ?3, blockers_json = ?4, + failure_category = ?5, updated_at = ?6, completed_at = ?7 + WHERE operation_id = ?1", + params![ + operation_id, + deletion_state_label(state), + serde_json::to_string(child_operation_ids) + .map_err(|error| Error::Store(error.to_string()))?, + serde_json::to_string(blockers) + .map_err(|error| Error::Store(error.to_string()))?, + failure_category, + now, + completed_at, + ], + )?; + if changed != 1 { + return Err(Error::InvalidInput( + "Workspace deletion operation".to_string(), + )); + } + Ok(read_operation(tx, operation_id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + })? + .response) + }) + } + + fn finalize_workspace_deletion( + &self, + operation_id: &str, + ) -> Result { + self.with_transaction(|tx| { + tx.execute_batch("PRAGMA defer_foreign_keys = ON;")?; + let operation = read_operation(tx, operation_id)? + .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; + if operation.response.state == WorkspaceDeletionState::Succeeded { + return Ok(operation.response); + } + let workspace_id = operation.response.workspace_id.clone(); + + let mut scoped_tables = Vec::new(); + let mut statement = tx.prepare( + "SELECT m.name + FROM sqlite_master m + WHERE m.type = 'table' AND m.name NOT LIKE 'sqlite_%' + ORDER BY m.name", + )?; + let names = statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + drop(statement); + for table in names { + if table == "workspaces" || table == "workspace_deletion_operations" { + continue; + } + let escaped = table.replace('"', "\"\""); + let mut info = tx.prepare(&format!("PRAGMA table_info(\"{escaped}\")"))?; + let columns = info + .query_map([], |row| row.get::<_, String>(1))? + .collect::, _>>()?; + if columns.contains("workspace_id") { + scoped_tables.push(escaped); + } + } + for table in scoped_tables { + tx.execute( + &format!("DELETE FROM \"{table}\" WHERE workspace_id = ?1"), + params![workspace_id], + )?; + } + let deleted = tx.execute( + "DELETE FROM workspaces WHERE workspace_id = ?1", + params![workspace_id], + )?; + if deleted != 1 { + return Err(Error::WorkspaceConfigConflict( + "Workspace disappeared before deletion finalized".to_string(), + )); + } + let now = Utc::now().to_rfc3339(); + tx.execute( + "UPDATE workspace_deletion_operations + SET state = 'succeeded', blockers_json = '[]', failure_category = NULL, + updated_at = ?2, completed_at = ?2 + WHERE operation_id = ?1", + params![operation_id, now], + )?; + let fk_failures: u64 = + tx.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |row| { + row.get(0) + })?; + if fk_failures != 0 { + return Err(Error::Store( + "foreign key check failed while finalizing Workspace deletion".to_string(), + )); + } + Ok(read_operation(tx, operation_id)? + .ok_or_else(|| { + Error::Store("completed Workspace deletion operation disappeared".to_string()) + })? + .response) + }) + } +} + +#[derive(Debug)] +struct StoredOperation { + request_fingerprint: String, + actor_account_id: String, + response: WorkspaceDeletionOperationResponse, +} + +fn read_operation( + conn: &rusqlite::Connection, + operation_id: &str, +) -> Result> { + conn.query_row( + "SELECT request_fingerprint, actor_account_id, workspace_id, workspace_display_name, + state, force_delete_dirty_workdirs, resource_counts_json, + child_operation_ids_json, blockers_json, failure_category, + created_at, updated_at, completed_at + FROM workspace_deletion_operations WHERE operation_id = ?1", + params![operation_id], + |row| { + let state: String = row.get(4)?; + let resource_counts_json: String = row.get(6)?; + let child_operation_ids_json: String = row.get(7)?; + let blockers_json: String = row.get(8)?; + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + state, + row.get::<_, bool>(5)?, + resource_counts_json, + child_operation_ids_json, + blockers_json, + row.get::<_, Option>(9)?, + row.get::<_, String>(10)?, + row.get::<_, String>(11)?, + row.get::<_, Option>(12)?, + )) + }, + ) + .optional()? + .map( + |( + fingerprint, + actor, + workspace_id, + display_name, + state, + force, + resources, + children, + blockers, + failure, + created_at, + updated_at, + completed_at, + )| { + Ok(StoredOperation { + request_fingerprint: fingerprint, + actor_account_id: actor, + response: WorkspaceDeletionOperationResponse { + operation_id: operation_id.to_string(), + workspace_id, + display_name, + state: parse_deletion_state(&state)?, + force_delete_dirty_workdirs: force, + resources: serde_json::from_str(&resources) + .map_err(|error| Error::Store(error.to_string()))?, + child_operation_ids: serde_json::from_str(&children) + .map_err(|error| Error::Store(error.to_string()))?, + blockers: serde_json::from_str(&blockers) + .map_err(|error| Error::Store(error.to_string()))?, + failure_category: failure, + created_at, + updated_at, + completed_at, + }, + }) + }, + ) + .transpose() +} + +fn owner_workspace( + conn: &rusqlite::Connection, + actor_account_id: &str, + workspace_id: &str, +) -> Result { + let workspace = conn + .query_row( + "SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1", + params![workspace_id], + |row| { + Ok(WorkspaceRecord { + workspace_id: row.get(0)?, + owner_account_id: row.get(1)?, + display_name: row.get(2)?, + state: row.get(3)?, + created_at: row.get(4)?, + updated_at: row.get(5)?, + }) + }, + ) + .optional()? + .ok_or_else(|| Error::InvalidInput("Workspace".to_string()))?; + if workspace.owner_account_id != actor_account_id { + return Err(Error::WorkspacePermissionDenied( + "Workspace owner permission is required".to_string(), + )); + } + Ok(workspace) +} + +fn resource_counts( + conn: &rusqlite::Connection, + workspace_id: &str, +) -> Result { + Ok(WorkspaceDeletionResourceCounts { + workers: table_count(conn, "worker_registry", workspace_id)?, + workdirs: table_count(conn, "workdir_registry", workspace_id)?, + repositories: table_count(conn, "repositories", workspace_id)?, + runtime_bindings: table_count(conn, "workspace_runtime_bindings", workspace_id)?, + secrets: table_count(conn, "server_secret_versions", workspace_id)?, + artifacts: table_count(conn, "artifacts", workspace_id)?, + }) +} + +fn table_count(conn: &rusqlite::Connection, table: &str, workspace_id: &str) -> Result { + conn.query_row( + &format!("SELECT COUNT(*) FROM \"{table}\" WHERE workspace_id = ?1"), + params![workspace_id], + |row| row.get(0), + ) + .map_err(Into::into) +} + +fn validate_operation_id(operation_id: &str) -> Result<()> { + if operation_id.is_empty() + || operation_id.len() > MAX_OPERATION_ID_BYTES + || !operation_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + return Err(Error::InvalidInput("operation_id is invalid".to_string())); + } + Ok(()) +} + +fn request_fingerprint( + actor_account_id: &str, + workspace_id: &str, + request: &WorkspaceDeletionRequest, +) -> String { + let canonical = format!( + "workspace-delete-v1\0{actor_account_id}\0{workspace_id}\0{}\0{}\0{}", + request.expected_revision, request.confirmation, request.force_delete_dirty_workdirs + ); + encode_hex(&Sha256::digest(canonical.as_bytes())) +} + +fn encode_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + output.push(HEX[(byte >> 4) as usize] as char); + output.push(HEX[(byte & 0x0f) as usize] as char); + } + output +} + +fn deletion_state_label(state: WorkspaceDeletionState) -> &'static str { + match state { + WorkspaceDeletionState::Queued => "queued", + WorkspaceDeletionState::Running => "running", + WorkspaceDeletionState::Blocked => "blocked", + WorkspaceDeletionState::Failed => "failed", + WorkspaceDeletionState::Succeeded => "succeeded", + } +} + +fn parse_deletion_state(value: &str) -> Result { + match value { + "queued" => Ok(WorkspaceDeletionState::Queued), + "running" => Ok(WorkspaceDeletionState::Running), + "blocked" => Ok(WorkspaceDeletionState::Blocked), + "failed" => Ok(WorkspaceDeletionState::Failed), + "succeeded" => Ok(WorkspaceDeletionState::Succeeded), + other => Err(Error::Store(format!( + "invalid Workspace deletion state `{other}`" + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + fn setup() -> (SqliteWorkspaceStore, String, String) { + let dir = tempdir().expect("tempdir"); + let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).expect("store"); + store.with_conn(|conn| { + let now = Utc::now().to_rfc3339(); + conn.execute( + "INSERT INTO accounts (account_id, kind, handle, display_name, created_at, updated_at) + VALUES ('owner', 'user', 'owner', 'Owner', ?1, ?1)", + params![now], + )?; + for (id, name) in [("workspace-a", "Alpha"), ("workspace-b", "Beta")] { + conn.execute( + "INSERT INTO workspaces (workspace_id, owner_account_id, display_name, state, created_at, updated_at) + VALUES (?1, 'owner', ?2, 'active', ?3, ?3)", + params![id, name, now], + )?; + } + Ok(()) + }).expect("fixtures"); + (store, "owner".to_string(), "workspace-a".to_string()) + } + + #[test] + fn deletion_is_idempotent_and_removes_workspace_scoped_rows() { + let (store, owner, workspace_id) = setup(); + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + let request = WorkspaceDeletionRequest { + operation_id: "delete-workspace-a".to_string(), + expected_revision: preflight.expected_revision, + confirmation: "delete Alpha".to_string(), + force_delete_dirty_workdirs: false, + }; + let first = store + .reserve_workspace_deletion(&owner, &workspace_id, &request) + .expect("reserve"); + assert!(!first.replay); + let replay = store + .reserve_workspace_deletion(&owner, &workspace_id, &request) + .expect("replay"); + assert!(replay.replay); + let completed = store + .finalize_workspace_deletion(&request.operation_id) + .expect("finalize"); + assert_eq!(completed.state, WorkspaceDeletionState::Succeeded); + let replayed = store + .finalize_workspace_deletion(&request.operation_id) + .expect("finalize replay"); + assert_eq!(completed, replayed); + let workspace_count: u64 = store + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM workspaces WHERE workspace_id = ?1", + params![workspace_id], + |row| row.get(0), + ) + .map_err(Into::into) + }) + .expect("read"); + assert_eq!(workspace_count, 0); + } + + #[test] + fn owner_confirmation_and_revision_are_required_before_reservation() { + let (store, owner, workspace_id) = setup(); + store.with_conn(|conn| { + conn.execute( + "INSERT INTO accounts (account_id, kind, handle, display_name, created_at, updated_at) + VALUES ('other', 'user', 'other', 'Other', '1', '1')", + [], + )?; + Ok(()) + }).expect("other account"); + assert!(matches!( + store.workspace_deletion_preflight("other", &workspace_id), + Err(Error::WorkspacePermissionDenied(_)) + )); + + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + let mut request = WorkspaceDeletionRequest { + operation_id: "delete-alpha-guarded".to_string(), + expected_revision: "stale".to_string(), + confirmation: "delete Alpha".to_string(), + force_delete_dirty_workdirs: false, + }; + assert!(matches!( + store.reserve_workspace_deletion(&owner, &workspace_id, &request), + Err(Error::WorkspaceConfigConflict(_)) + )); + request.expected_revision = preflight.expected_revision; + request.confirmation = "Alpha".to_string(); + assert!(matches!( + store.reserve_workspace_deletion(&owner, &workspace_id, &request), + Err(Error::InvalidInput(_)) + )); + } + + #[test] + fn last_accessible_workspace_and_revision_conflicts_fail_closed() { + let (store, owner, workspace_id) = setup(); + let other = "workspace-b"; + let preflight = store + .workspace_deletion_preflight(&owner, other) + .expect("preflight"); + store + .finalize_workspace_deletion( + &store + .reserve_workspace_deletion( + &owner, + other, + &WorkspaceDeletionRequest { + operation_id: "delete-beta".to_string(), + expected_revision: preflight.expected_revision, + confirmation: "delete Beta".to_string(), + force_delete_dirty_workdirs: false, + }, + ) + .expect("reserve") + .operation + .operation_id, + ) + .expect("delete beta"); + let blocked = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("blocked"); + assert!(!blocked.can_delete); + assert_eq!( + blocked.blockers[0].kind, + WorkspaceDeletionBlockerKind::LastAccessibleWorkspace + ); + } +} diff --git a/web/workspace/src/lib/generated/workspace-api.ts b/web/workspace/src/lib/generated/workspace-api.ts index 09be8bf9..a48b3bb1 100644 --- a/web/workspace/src/lib/generated/workspace-api.ts +++ b/web/workspace/src/lib/generated/workspace-api.ts @@ -48,6 +48,74 @@ export type WorkspacePermissionSummary = { manage_repositories: boolean; manage_secrets: boolean; manage_runtimes: boolean; + delete_workspace: boolean; +}; + +export type WorkspaceDeletionState = + | "queued" + | "running" + | "blocked" + | "failed" + | "succeeded"; + +export type WorkspaceDeletionBlockerKind = + | "last_accessible_workspace" + | "revision_conflict" + | "dirty_workdir" + | "worker_removal_blocked" + | "workdir_removal_blocked" + | "retention_hold" + | "cleanup_unavailable"; + +export type WorkspaceDeletionBlocker = { + kind: WorkspaceDeletionBlockerKind; + resource_kind: string | null; + resource_key: string | null; + message: string; +}; + +export type WorkspaceDeletionResourceCounts = { + workers: number; + workdirs: number; + repositories: number; + runtime_bindings: number; + secrets: number; + artifacts: number; +}; + +export type WorkspaceDeletionPreflightResponse = { + workspace_id: string; + display_name: string; + /** + * Opaque persisted Workspace metadata revision used as a CAS fence. + */ + expected_revision: string; + can_delete: boolean; + force_delete_dirty_workdirs_available: boolean; + resources: WorkspaceDeletionResourceCounts; + blockers: Array; +}; + +export type WorkspaceDeletionRequest = { + operation_id: string; + expected_revision: string; + confirmation: string; + force_delete_dirty_workdirs: boolean; +}; + +export type WorkspaceDeletionOperationResponse = { + operation_id: string; + workspace_id: string; + display_name: string; + state: WorkspaceDeletionState; + force_delete_dirty_workdirs: boolean; + resources: WorkspaceDeletionResourceCounts; + child_operation_ids: Array; + blockers: Array; + failure_category: string | null; + created_at: string; + updated_at: string; + completed_at: string | null; }; export type DiagnosticSeverity = "info" | "warning" | "error"; diff --git a/web/workspace/src/lib/workspace/api/workspace-model.ts b/web/workspace/src/lib/workspace/api/workspace-model.ts index 2088a18f..5178fc73 100644 --- a/web/workspace/src/lib/workspace/api/workspace-model.ts +++ b/web/workspace/src/lib/workspace/api/workspace-model.ts @@ -14,6 +14,12 @@ import type { WorkspaceAuthConfig, WorkspaceCatalogListResponse, WorkspaceCreateResponse, + WorkspaceDeletionBlocker, + WorkspaceDeletionBlockerKind, + WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, + WorkspaceDeletionResourceCounts, + WorkspaceDeletionState, WorkspaceExtensionPoints, WorkspaceExtensionPointState, WorkspacePermissionSummary, @@ -32,6 +38,8 @@ export type { RepositorySummary, WorkspaceCatalogListResponse, WorkspaceCreateResponse, + WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, WorkspacePermissionSummary, WorkspaceResponse, WorkspaceSummary, @@ -369,7 +377,12 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary { const item = object(value, path); exactKeys( item, - ["manage_repositories", "manage_secrets", "manage_runtimes"], + [ + "manage_repositories", + "manage_secrets", + "manage_runtimes", + "delete_workspace", + ], path, ); return { @@ -379,6 +392,10 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary { ), manage_secrets: boolean(item.manage_secrets, `${path}.manage_secrets`), manage_runtimes: boolean(item.manage_runtimes, `${path}.manage_runtimes`), + delete_workspace: boolean( + item.delete_workspace, + `${path}.delete_workspace`, + ), }; } @@ -566,6 +583,202 @@ export function parseRepositoryDetailResponse( }; } +const deletionStates = new Set([ + "queued", + "running", + "blocked", + "failed", + "succeeded", +]); +const deletionBlockerKinds = new Set([ + "last_accessible_workspace", + "revision_conflict", + "dirty_workdir", + "worker_removal_blocked", + "workdir_removal_blocked", + "retention_hold", + "cleanup_unavailable", +]); + +function deletionState(value: unknown, path: string): WorkspaceDeletionState { + const candidate = string(value, path) as WorkspaceDeletionState; + if (!deletionStates.has(candidate)) throw new Error(`${path} is invalid`); + return candidate; +} + +function deletionBlocker( + value: unknown, + path: string, +): WorkspaceDeletionBlocker { + const item = object(value, path); + exactKeys(item, ["kind", "resource_kind", "resource_key", "message"], path); + const kind = string( + item.kind, + `${path}.kind`, + ) as WorkspaceDeletionBlockerKind; + if (!deletionBlockerKinds.has(kind)) { + throw new Error(`${path}.kind is invalid`); + } + return { + kind, + resource_kind: + optionalNullableString(item.resource_kind, `${path}.resource_kind`) ?? + null, + resource_key: + optionalNullableString(item.resource_key, `${path}.resource_key`) ?? null, + message: string(item.message, `${path}.message`), + }; +} + +function deletionResourceCounts( + value: unknown, + path: string, +): WorkspaceDeletionResourceCounts { + const item = object(value, path); + exactKeys(item, [ + "workers", + "workdirs", + "repositories", + "runtime_bindings", + "secrets", + "artifacts", + ], path); + return { + workers: integer(item.workers, `${path}.workers`), + workdirs: integer(item.workdirs, `${path}.workdirs`), + repositories: integer(item.repositories, `${path}.repositories`), + runtime_bindings: integer( + item.runtime_bindings, + `${path}.runtime_bindings`, + ), + secrets: integer(item.secrets, `${path}.secrets`), + artifacts: integer(item.artifacts, `${path}.artifacts`), + }; +} + +export function parseWorkspaceDeletionPreflightResponse( + value: unknown, +): WorkspaceDeletionPreflightResponse { + const item = object(value, "Workspace deletion preflight"); + exactKeys(item, [ + "workspace_id", + "display_name", + "expected_revision", + "can_delete", + "force_delete_dirty_workdirs_available", + "resources", + "blockers", + ], "Workspace deletion preflight"); + return { + workspace_id: string( + item.workspace_id, + "Workspace deletion preflight.workspace_id", + ), + display_name: string( + item.display_name, + "Workspace deletion preflight.display_name", + ), + expected_revision: string( + item.expected_revision, + "Workspace deletion preflight.expected_revision", + ), + can_delete: boolean( + item.can_delete, + "Workspace deletion preflight.can_delete", + ), + force_delete_dirty_workdirs_available: boolean( + item.force_delete_dirty_workdirs_available, + "Workspace deletion preflight.force_delete_dirty_workdirs_available", + ), + resources: deletionResourceCounts( + item.resources, + "Workspace deletion preflight.resources", + ), + blockers: array(item.blockers, "Workspace deletion preflight.blockers").map( + (entry, index) => + deletionBlocker( + entry, + `Workspace deletion preflight.blockers[${index}]`, + ), + ), + }; +} + +export function parseWorkspaceDeletionOperationResponse( + value: unknown, +): WorkspaceDeletionOperationResponse { + const item = object(value, "Workspace deletion operation"); + exactKeys(item, [ + "operation_id", + "workspace_id", + "display_name", + "state", + "force_delete_dirty_workdirs", + "resources", + "child_operation_ids", + "blockers", + "failure_category", + "created_at", + "updated_at", + "completed_at", + ], "Workspace deletion operation"); + return { + operation_id: string( + item.operation_id, + "Workspace deletion operation.operation_id", + ), + workspace_id: string( + item.workspace_id, + "Workspace deletion operation.workspace_id", + ), + display_name: string( + item.display_name, + "Workspace deletion operation.display_name", + ), + state: deletionState(item.state, "Workspace deletion operation.state"), + force_delete_dirty_workdirs: boolean( + item.force_delete_dirty_workdirs, + "Workspace deletion operation.force_delete_dirty_workdirs", + ), + resources: deletionResourceCounts( + item.resources, + "Workspace deletion operation.resources", + ), + child_operation_ids: array( + item.child_operation_ids, + "Workspace deletion operation.child_operation_ids", + ).map((entry, index) => + string( + entry, + `Workspace deletion operation.child_operation_ids[${index}]`, + ) + ), + blockers: array(item.blockers, "Workspace deletion operation.blockers").map( + (entry, index) => + deletionBlocker( + entry, + `Workspace deletion operation.blockers[${index}]`, + ), + ), + failure_category: optionalNullableString( + item.failure_category, + "Workspace deletion operation.failure_category", + ) ?? null, + created_at: string( + item.created_at, + "Workspace deletion operation.created_at", + ), + updated_at: string( + item.updated_at, + "Workspace deletion operation.updated_at", + ), + completed_at: optionalNullableString( + item.completed_at, + "Workspace deletion operation.completed_at", + ) ?? null, + }; +} + export function parseRepositoryLogResponse( value: unknown, ): RepositoryLogResponse { diff --git a/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts b/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts new file mode 100644 index 00000000..8106f1f7 --- /dev/null +++ b/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts @@ -0,0 +1,63 @@ +import type { + WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, + WorkspaceDeletionRequest, +} from "$lib/generated/workspace-api"; +import { + parseWorkspaceDeletionOperationResponse, + parseWorkspaceDeletionPreflightResponse, +} from "$lib/workspace/api/workspace-model"; + +async function responseJson( + response: Response, + context: string, +): Promise { + const value: unknown = await response.json().catch(() => null); + if (!response.ok) { + const message = typeof value === "object" && value !== null && + "error" in value && typeof value.error === "string" + ? value.error + : `${context} failed (${response.status})`; + throw new Error(message); + } + return value; +} + +export async function preflightWorkspaceDeletion( + workspaceId: string, +): Promise { + const response = await fetch( + `/api/workspaces/${encodeURIComponent(workspaceId)}/deletion`, + ); + return parseWorkspaceDeletionPreflightResponse( + await responseJson(response, "Workspace deletion preflight"), + ); +} + +export async function startWorkspaceDeletion( + workspaceId: string, + request: WorkspaceDeletionRequest, +): Promise { + const response = await fetch( + `/api/workspaces/${encodeURIComponent(workspaceId)}/deletion`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(request), + }, + ); + return parseWorkspaceDeletionOperationResponse( + await responseJson(response, "Workspace deletion"), + ); +} + +export async function getWorkspaceDeletion( + operationId: string, +): Promise { + const response = await fetch( + `/api/workspace-deletions/${encodeURIComponent(operationId)}`, + ); + return parseWorkspaceDeletionOperationResponse( + await responseJson(response, "Workspace deletion status"), + ); +} diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte index 5c1fdf7d..b5293f31 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte @@ -1,8 +1,17 @@ @@ -113,3 +176,74 @@ {/if} + +{#if data.workspace?.permissions.delete_workspace} +
+
+

Danger zone

+

Deleting this Workspace permanently removes its Workers, Workdirs, repositories, configuration, Memory, Tickets, and audit data.

+
+ +
+{/if} + +{#if deletionOpen} + +{/if} + + diff --git a/web/workspace/tests/workspace-model.test.ts b/web/workspace/tests/workspace-model.test.ts index 66efbae9..5374dfc4 100644 --- a/web/workspace/tests/workspace-model.test.ts +++ b/web/workspace/tests/workspace-model.test.ts @@ -6,6 +6,8 @@ declare const Deno: { import { parseRepositoryListApiResult, parseRepositoryListResponse, + parseWorkspaceDeletionOperationResponse, + parseWorkspaceDeletionPreflightResponse, parseWorkspaceResponse, } from "../src/lib/workspace/api/workspace-model.ts"; @@ -109,6 +111,91 @@ Deno.test("workspace response requires the permission projection", () => { ); }); +Deno.test("Workspace deletion DTOs fail closed and preserve durable operation state", () => { + const preflight = parseWorkspaceDeletionPreflightResponse({ + workspace_id: "workspace-a", + display_name: "Alpha", + expected_revision: "2026-01-01T00:00:00Z", + can_delete: true, + force_delete_dirty_workdirs_available: true, + resources: { + workers: 2, + workdirs: 1, + repositories: 1, + runtime_bindings: 1, + secrets: 0, + artifacts: 3, + }, + blockers: [], + }); + if (preflight.resources.workers !== 2) { + throw new Error("worker count was not preserved"); + } + + const operation = parseWorkspaceDeletionOperationResponse({ + operation_id: "delete-alpha", + workspace_id: "workspace-a", + display_name: "Alpha", + state: "blocked", + force_delete_dirty_workdirs: false, + resources: preflight.resources, + child_operation_ids: ["worker-remove:arcadia/7"], + blockers: [{ + kind: "dirty_workdir", + resource_kind: "workdir", + resource_key: "WD-1", + message: "Workdir is dirty", + }], + failure_category: null, + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:01:00Z", + completed_at: null, + }); + if (operation.state !== "blocked") { + throw new Error("operation state was not preserved"); + } + + assertThrows( + () => + parseWorkspaceDeletionPreflightResponse({ + ...preflight, + unexpected: true, + }), + "unexpected is not part", + ); + assertThrows( + () => + parseWorkspaceDeletionOperationResponse({ + ...operation, + state: "unknown", + }), + ".state is invalid", + ); +}); + +Deno.test("Workspace settings exposes owner-gated typed destructive confirmation", async () => { + const source = await Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/workspace/+page.svelte", + import.meta.url, + ), + ); + for ( + const token of [ + "permissions.delete_workspace", + "preflightWorkspaceDeletion", + "startWorkspaceDeletion", + "delete ${", + "force_delete_dirty_workdirs", + "disposeWorkspaceMultiplexer(workspaceId)", + ] + ) { + if (!source.includes(token)) { + throw new Error(`Workspace deletion UI should include ${token}`); + } + } +}); + Deno.test("Repository settings consume the validated shared wire shape", async () => { const [loadSource, pageSource] = await Promise.all([ Deno.readTextFile( From ab4fb4c1eeaad818a2c1f40e5f7835bfefbeb14a Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 07:54:52 +0900 Subject: [PATCH 30/39] fix: enforce workspace deletion blockers --- crates/workspace-api/src/lib.rs | 8 +- crates/workspace-server/src/latest_schema.sql | 1 - crates/workspace-server/src/server.rs | 209 +++++++++--------- crates/workspace-server/src/store.rs | 2 - .../src/workspace_deletion.rs | 195 ++++++++++++---- .../src/lib/generated/workspace-api.ts | 3 - .../src/lib/workspace/api/workspace-model.ts | 10 - .../settings/workspace/+page.svelte | 14 +- web/workspace/tests/workspace-model.test.ts | 5 +- 9 files changed, 261 insertions(+), 186 deletions(-) diff --git a/crates/workspace-api/src/lib.rs b/crates/workspace-api/src/lib.rs index 1f043422..14a25295 100644 --- a/crates/workspace-api/src/lib.rs +++ b/crates/workspace-api/src/lib.rs @@ -673,7 +673,6 @@ pub struct WorkspaceDeletionPreflightResponse { /// Opaque persisted Workspace metadata revision used as a CAS fence. pub expected_revision: String, pub can_delete: bool, - pub force_delete_dirty_workdirs_available: bool, pub resources: WorkspaceDeletionResourceCounts, pub blockers: Vec, } @@ -686,8 +685,6 @@ pub struct WorkspaceDeletionRequest { pub operation_id: String, pub expected_revision: String, pub confirmation: String, - #[serde(default)] - pub force_delete_dirty_workdirs: bool, } /// Durable deletion operation projection used by request responses and polling. @@ -699,7 +696,6 @@ pub struct WorkspaceDeletionOperationResponse { pub workspace_id: String, pub display_name: String, pub state: WorkspaceDeletionState, - pub force_delete_dirty_workdirs: bool, pub resources: WorkspaceDeletionResourceCounts, pub child_operation_ids: Vec, pub blockers: Vec, @@ -3262,7 +3258,6 @@ mod tests { display_name: "Test".to_string(), expected_revision: "revision-7".to_string(), can_delete: true, - force_delete_dirty_workdirs_available: true, resources: WorkspaceDeletionResourceCounts { workers: 2, workdirs: 1, @@ -3288,8 +3283,7 @@ mod tests { serde_json::from_value::(serde_json::json!({ "operation_id": "delete-test", "expected_revision": "revision-7", - "confirmation": "delete Test", - "force_delete_dirty_workdirs": false, + "confirmation": "Test", "workspace_id": "caller-controlled" })) .is_err() diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index a93fe544..919e0957 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -948,7 +948,6 @@ CREATE TABLE workspace_deletion_operations ( workspace_revision TEXT NOT NULL, owner_account_id TEXT NOT NULL, actor_account_id TEXT NOT NULL, - force_delete_dirty_workdirs INTEGER NOT NULL CHECK(force_delete_dirty_workdirs IN (0, 1)), state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')), resource_counts_json TEXT NOT NULL, child_operation_ids_json TEXT NOT NULL, diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 6cc419e8..01be9ad8 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -90,11 +90,11 @@ use workspace_api::{ WorkingDirectoryRemovalDisposition, WorkingDirectoryRemovalRequest, WorkingDirectoryRemovalResponse, WorkingDirectoryRepositoryOption, WorkspaceCatalogListResponse, WorkspaceCreateResponse, WorkspaceDeletionBlocker, - WorkspaceDeletionBlockerKind, WorkspaceDeletionOperationResponse, WorkspaceDeletionRequest, - WorkspaceDeletionState, WorkspaceExtensionPointState, WorkspaceExtensionPoints, - WorkspaceMetadataMutationResponse, WorkspaceMetadataSettingsResponse, - WorkspacePermissionSummary, WorkspaceRepositoryRecord, WorkspaceResponse, - WorkspaceRuntimeDetail, WorkspaceRuntimeResource, WorkspaceSummary, + WorkspaceDeletionBlockerKind, WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, WorkspaceDeletionRequest, WorkspaceDeletionState, + WorkspaceExtensionPointState, WorkspaceExtensionPoints, WorkspaceMetadataMutationResponse, + WorkspaceMetadataSettingsResponse, WorkspacePermissionSummary, WorkspaceRepositoryRecord, + WorkspaceResponse, WorkspaceRuntimeDetail, WorkspaceRuntimeResource, WorkspaceSummary, WorkspaceWorkerDiscoveryItem, WorkspaceWorkerDiscoveryPage, WorkspaceWorkerSubject, }; @@ -746,8 +746,7 @@ impl WorkspaceWorkerRemoveExecutor { )); } - self.execute_target_removal(&runtime, &target, reason, false) - .await + self.execute_target_removal(&runtime, &target, reason).await } async fn execute_target_removal( @@ -755,7 +754,6 @@ impl WorkspaceWorkerRemoveExecutor { runtime: &RuntimeRegistry, target: &RuntimeWorkerRef, reason: &str, - allow_internal: bool, ) -> std::result::Result { let remove_lock = { let mut locks = self @@ -857,7 +855,7 @@ impl WorkspaceWorkerRemoveExecutor { )); } }; - if worker.singleton_key.is_some() && !allow_internal { + if worker.singleton_key.is_some() { return Ok(worker_remove_error_response( StatusCode::CONFLICT, "internal_worker_forbidden", @@ -1009,6 +1007,7 @@ pub struct WorkspaceServerApi { store: Arc, catalog: WorkspaceCatalogService, routers: Arc>>, + apis: Arc>>, } impl WorkspaceServerApi { @@ -1018,9 +1017,71 @@ impl WorkspaceServerApi { catalog: WorkspaceCatalogService::new(store.clone()), store, routers: Arc::new(AsyncMutex::new(HashMap::new())), + apis: Arc::new(AsyncMutex::new(HashMap::new())), } } + async fn api_for_workspace(&self, workspace_id: &str) -> Result> { + let mut apis = self.apis.lock().await; + if let Some(api) = apis.get(workspace_id).cloned() { + return Ok(Some(api)); + } + let Some(workspace) = self.store.get_workspace(workspace_id).await? else { + return Ok(None); + }; + let repositories = self.store.list_repositories(workspace_id)?; + let config = self + .template + .for_catalog_workspace(&workspace, repositories)?; + let api = WorkspaceApi::new(config, self.store.clone()).await?; + apis.insert(workspace_id.to_string(), api.clone()); + Ok(Some(api)) + } + + async fn workspace_deletion_preflight( + &self, + actor_account_id: &str, + workspace_id: &str, + ) -> Result { + let mut preflight = self + .store + .workspace_deletion_preflight(actor_account_id, workspace_id)?; + let Some(api) = self.api_for_workspace(workspace_id).await? else { + return Err(Error::InvalidInput("Workspace does not exist".to_string())); + }; + for registry_worker in self.store.list_worker_registry(workspace_id, 10_000)? { + let worker_key = registry_worker.display_name; + match api.runtime.worker(®istry_worker.worker) { + Ok(worker) if worker.state == "stopped" && worker.singleton_key.is_none() => {} + Ok(worker) => preflight.blockers.push(WorkspaceDeletionBlocker { + kind: if worker.singleton_key.is_some() { + WorkspaceDeletionBlockerKind::RetentionHold + } else { + WorkspaceDeletionBlockerKind::WorkerRemovalBlocked + }, + resource_kind: Some("worker".to_string()), + resource_key: Some(worker_key), + message: if worker.singleton_key.is_some() { + "Internal or singleton Workers must be released by their owning service first." + .to_string() + } else { + "Stop running, restoring, or otherwise active Workers before deleting the Workspace." + .to_string() + }, + }), + Err(_) => preflight.blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::CleanupUnavailable, + resource_kind: Some("worker".to_string()), + resource_key: Some(worker_key), + message: "Worker state is unavailable; retry after Runtime state is healthy." + .to_string(), + }), + } + } + preflight.can_delete = preflight.blockers.is_empty(); + Ok(preflight) + } + async fn execute_workspace_deletion( &self, operation_id: &str, @@ -1032,18 +1093,10 @@ impl WorkspaceServerApi { &[], None, )?; - let workspace = self - .store - .get_workspace(&operation.workspace_id) + let api = self + .api_for_workspace(&operation.workspace_id) .await? .ok_or_else(|| Error::InvalidInput("Workspace no longer exists".to_string()))?; - let repositories = self.store.list_repositories(&operation.workspace_id)?; - let config = self - .template - .for_catalog_workspace(&workspace, repositories)?; - let api = WorkspaceApi::new(config, self.store.clone()).await?; - self.store - .release_workspace_assignments_for_deletion(&operation.workspace_id)?; let mut child_operation_ids = Vec::new(); let mut blockers = Vec::new(); @@ -1053,22 +1106,8 @@ impl WorkspaceServerApi { { let worker_key = worker.display_name.clone(); let target = worker.worker; - let lifecycle = WorkerLifecycleRequest { - reason: Some("Workspace deletion".to_string()), - ticket_assignment: None, - }; - let _ = api.runtime.cancel_worker(&target, lifecycle.clone()); - if api.runtime.stop_worker(&target, lifecycle).is_err() { - blockers.push(WorkspaceDeletionBlocker { - kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, - resource_kind: Some("worker".to_string()), - resource_key: Some(worker_key.clone()), - message: "Worker stop did not reach a retryable terminal state.".to_string(), - }); - continue; - } let response = WorkspaceWorkerRemoveExecutor::new(&api) - .execute_target_removal(api.runtime.as_ref(), &target, "Workspace deletion", true) + .execute_target_removal(api.runtime.as_ref(), &target, "Workspace deletion") .await .map_err(Error::Store)?; if let Some(child_operation_id) = self.store.latest_worker_removal_operation_id( @@ -1097,7 +1136,6 @@ impl WorkspaceServerApi { &api, &workdir.workdir_id, operation_id, - operation.force_delete_dirty_workdirs, ) { Ok(child) => { child_operation_ids.push(child.operation_id.clone()); @@ -1115,7 +1153,7 @@ impl WorkspaceServerApi { resource_kind: Some("workdir".to_string()), resource_key: Some(workdir.workdir_id), message: if dirty { - "Workdir is dirty or its cleanliness is unknown. Enable force deletion only after reviewing the impact." + "Workdir is dirty or its cleanliness is unknown. Clean it and refresh status before retrying deletion." .to_string() } else { "Workdir removal did not complete; retry the Workspace deletion operation." @@ -1145,25 +1183,23 @@ impl WorkspaceServerApi { } let completed = self.store.finalize_workspace_deletion(operation_id)?; self.routers.lock().await.remove(&completed.workspace_id); + self.apis.lock().await.remove(&completed.workspace_id); Ok(completed) } async fn router_for_workspace(&self, workspace_id: &str) -> Result> { - let mut routers = self.routers.lock().await; - if let Some(router) = routers.get(workspace_id) { - return Ok(Some(router.clone())); + if let Some(router) = self.routers.lock().await.get(workspace_id).cloned() { + return Ok(Some(router)); } - let Some(workspace) = self.store.get_workspace(workspace_id).await? else { + let Some(api) = self.api_for_workspace(workspace_id).await? else { return Ok(None); }; - let repositories = self.store.list_repositories(workspace_id)?; - let config = self - .template - .for_catalog_workspace(&workspace, repositories)?; - let api = WorkspaceApi::new(config, self.store.clone()).await?; tokio::spawn(run_orchestrator_turn_end_hook(api.clone())); let router = build_inner_router(api); - routers.insert(workspace_id.to_string(), router.clone()); + self.routers + .lock() + .await + .insert(workspace_id.to_string(), router.clone()); Ok(Some(router)) } @@ -1269,8 +1305,8 @@ async fn preflight_server_workspace_deletion( Err(error) => return server_error_response(error), }; match api - .store .workspace_deletion_preflight(&actor_account_id, &workspace_id) + .await { Ok(preflight) => Json(preflight).into_response(), Err(error) => server_error_response(error), @@ -1288,6 +1324,25 @@ async fn start_server_workspace_deletion( Ok(None) => return forbidden_server_response("Workspace deletion requires its owner"), Err(error) => return server_error_response(error), }; + let existing = match api + .store + .workspace_deletion_operation(&actor_account_id, &request.operation_id) + { + Ok(existing) => existing, + Err(error) => return server_error_response(error), + }; + if existing.is_none() { + let preflight = match api + .workspace_deletion_preflight(&actor_account_id, &workspace_id) + .await + { + Ok(preflight) => preflight, + Err(error) => return server_error_response(error), + }; + if !preflight.can_delete { + return (StatusCode::CONFLICT, Json(preflight)).into_response(); + } + } let reservation = match api .store @@ -10520,27 +10575,7 @@ fn execute_reserved_workdir_removal( operation: WorkdirRemovalOperation, recovery: bool, ) -> Result { - execute_reserved_workdir_removal_with_provider( - api, - operation, - recovery, - api.runtime.as_ref(), - false, - ) -} - -fn execute_reserved_workdir_removal_for_workspace_deletion( - api: &WorkspaceApi, - operation: WorkdirRemovalOperation, - force_dirty: bool, -) -> Result { - execute_reserved_workdir_removal_with_provider( - api, - operation, - false, - api.runtime.as_ref(), - force_dirty, - ) + execute_reserved_workdir_removal_with_provider(api, operation, recovery, api.runtime.as_ref()) } fn execute_reserved_workdir_removal_with_provider( @@ -10548,7 +10583,6 @@ fn execute_reserved_workdir_removal_with_provider( operation: WorkdirRemovalOperation, recovery: bool, provider: &dyn WorkdirRemovalRuntimeProvider, - force_dirty: bool, ) -> Result { if operation.state == WorkdirRemovalOperationState::Completed { return Ok(operation); @@ -10623,9 +10657,8 @@ fn execute_reserved_workdir_removal_with_provider( true, ); }; - if !force_dirty - && (status.summary.cleanliness.as_deref() != Some("clean") - || status.summary.status != WorkingDirectoryStatusKind::Active) + if status.summary.cleanliness.as_deref() != Some("clean") + || status.summary.status != WorkingDirectoryStatusKind::Active { return api.config_store.complete_workdir_removal_retained( &operation, @@ -10723,7 +10756,6 @@ fn execute_workdir_removal_for_workspace_deletion( api: &WorkspaceApi, working_directory_id: &str, parent_operation_id: &str, - force_dirty: bool, ) -> Result { let source_actor = format!("workspace-deletion:{parent_operation_id}"); let reason = "Workspace deletion"; @@ -10750,7 +10782,7 @@ fn execute_workdir_removal_for_workspace_deletion( api.config_store .reserve_workdir_removal_operation(&intent)? }; - execute_reserved_workdir_removal_for_workspace_deletion(api, operation, force_dirty) + execute_reserved_workdir_removal(api, operation, false) } fn recover_workdir_removals(api: &WorkspaceApi) -> Result<()> { @@ -24371,7 +24403,6 @@ mod tests { clean_operation.clone(), false, &clean_provider, - false, ) .unwrap(); assert_eq!( @@ -24384,7 +24415,6 @@ mod tests { removed.clone(), false, &clean_provider, - false, ) .unwrap(); assert_eq!(replay, removed); @@ -24409,7 +24439,6 @@ mod tests { missing_operation, false, &missing_provider, - false, ) .unwrap(); assert_eq!( @@ -24436,7 +24465,6 @@ mod tests { unknown_operation, false, &unknown_provider, - false, ) .unwrap(); assert_eq!(unknown.state, WorkdirRemovalOperationState::Failed); @@ -24458,34 +24486,11 @@ mod tests { dirty_operation, false, &dirty_provider, - false, ) .unwrap(); assert_eq!(dirty.disposition, Some(WorkdirRemovalDisposition::Retained)); assert_eq!(dirty_provider.cleanup_calls(), 0); - let (forced_operation, mut forced_summary) = - reserve_removal_fixture(&api, "provider-dirty-forced"); - forced_summary.cleanliness = Some("dirty".to_string()); - let forced_provider = FakeWorkdirRemovalProvider::new( - workdir_removal_result( - WorkerOperationState::Accepted, - Some(forced_summary), - Vec::new(), - ), - workdir_removal_result(WorkerOperationState::Accepted, None, Vec::new()), - ); - let forced = execute_reserved_workdir_removal_with_provider( - &api, - forced_operation, - false, - &forced_provider, - true, - ) - .unwrap(); - assert_eq!(forced.disposition, Some(WorkdirRemovalDisposition::Removed)); - assert_eq!(forced_provider.cleanup_calls(), 1); - let (unsupported_operation, unsupported_summary) = reserve_removal_fixture(&api, "provider-unsupported"); let unsupported_provider = FakeWorkdirRemovalProvider::new( @@ -24509,7 +24514,6 @@ mod tests { unsupported_operation, false, &unsupported_provider, - false, ) .unwrap(); assert_eq!(unsupported.state, WorkdirRemovalOperationState::Failed); @@ -24564,7 +24568,6 @@ mod tests { operation, true, provider.as_ref(), - false, ) })); } diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index d7a7263f..aa0d7b85 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -6438,7 +6438,6 @@ fn migrate_workspace_deletion_v52_to_v53(conn: &Connection) -> Result<()> { workspace_revision TEXT NOT NULL, owner_account_id TEXT NOT NULL, actor_account_id TEXT NOT NULL, - force_delete_dirty_workdirs INTEGER NOT NULL CHECK(force_delete_dirty_workdirs IN (0, 1)), state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')), resource_counts_json TEXT NOT NULL, child_operation_ids_json TEXT NOT NULL, @@ -6475,7 +6474,6 @@ fn verify_workspace_deletion_schema(conn: &Connection) -> Result<()> { "workspace_revision", "owner_account_id", "actor_account_id", - "force_delete_dirty_workdirs", "state", "resource_counts_json", "child_operation_ids_json", diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index da90cb31..f5777cba 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -12,7 +12,6 @@ use crate::store::{SqliteWorkspaceStore, WorkspaceRecord}; use crate::{Error, Result}; const MAX_OPERATION_ID_BYTES: usize = 128; -const CONFIRMATION_PREFIX: &str = "delete "; #[derive(Debug, Clone)] pub struct WorkspaceDeletionReservation { @@ -40,8 +39,6 @@ pub trait WorkspaceDeletionStore: Send + Sync { operation_id: &str, ) -> Result>; - fn release_workspace_assignments_for_deletion(&self, workspace_id: &str) -> Result; - fn latest_worker_removal_operation_id( &self, workspace_id: &str, @@ -74,11 +71,11 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { let workspace = owner_workspace(conn, actor_account_id, workspace_id)?; let resources = resource_counts(conn, workspace_id)?; let accessible: u64 = conn.query_row( - "SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1 AND state = 'active'", + "SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1", params![actor_account_id], |row| row.get(0), )?; - let mut blockers = Vec::new(); + let mut blockers = workspace_database_blockers(conn, workspace_id)?; if accessible <= 1 { blockers.push(WorkspaceDeletionBlocker { kind: WorkspaceDeletionBlockerKind::LastAccessibleWorkspace, @@ -92,7 +89,6 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { display_name: workspace.display_name, expected_revision: workspace.updated_at, can_delete: blockers.is_empty(), - force_delete_dirty_workdirs_available: true, resources, blockers, }) @@ -126,8 +122,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { } let workspace = owner_workspace(tx, actor_account_id, workspace_id)?; - let expected_confirmation = format!("{CONFIRMATION_PREFIX}{}", workspace.display_name); - if request.confirmation != expected_confirmation { + if request.confirmation != workspace.display_name { return Err(Error::InvalidInput( "confirmation must exactly match the displayed Workspace name".to_string(), )); @@ -138,7 +133,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { )); } let accessible: u64 = tx.query_row( - "SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1 AND state = 'active'", + "SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1", params![actor_account_id], |row| row.get(0), )?; @@ -148,6 +143,11 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { .to_string(), )); } + if !workspace_database_blockers(tx, workspace_id)?.is_empty() { + return Err(Error::WorkspaceConfigConflict( + "Workspace deletion preflight changed; reload current blockers".to_string(), + )); + } let resources = resource_counts(tx, workspace_id)?; let now = Utc::now().to_rfc3339(); @@ -156,10 +156,10 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { "INSERT INTO workspace_deletion_operations ( operation_id, request_fingerprint, workspace_id, workspace_display_name, workspace_revision, owner_account_id, actor_account_id, - force_delete_dirty_workdirs, state, resource_counts_json, + state, resource_counts_json, child_operation_ids_json, blockers_json, failure_category, created_at, updated_at, completed_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'queued', ?9, '[]', '[]', NULL, ?10, ?10, NULL)", + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'queued', ?8, '[]', '[]', NULL, ?9, ?9, NULL)", params![ request.operation_id, fingerprint, @@ -168,7 +168,6 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { request.expected_revision, workspace.owner_account_id, actor_account_id, - request.force_delete_dirty_workdirs as i64, serde_json::to_string(&resources).map_err(|error| Error::Store(error.to_string()))?, now, ], @@ -210,17 +209,6 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { }) } - fn release_workspace_assignments_for_deletion(&self, workspace_id: &str) -> Result { - self.with_conn(|conn| { - let changed = conn.execute( - "DELETE FROM ticket_current_worker_assignments WHERE workspace_id = ?1", - params![workspace_id], - )?; - u64::try_from(changed) - .map_err(|_| Error::Store("assignment deletion count overflow".to_string())) - }) - } - fn latest_worker_removal_operation_id( &self, workspace_id: &str, @@ -373,30 +361,28 @@ fn read_operation( ) -> Result> { conn.query_row( "SELECT request_fingerprint, actor_account_id, workspace_id, workspace_display_name, - state, force_delete_dirty_workdirs, resource_counts_json, - child_operation_ids_json, blockers_json, failure_category, - created_at, updated_at, completed_at + state, resource_counts_json, child_operation_ids_json, + blockers_json, failure_category, created_at, updated_at, completed_at FROM workspace_deletion_operations WHERE operation_id = ?1", params![operation_id], |row| { let state: String = row.get(4)?; - let resource_counts_json: String = row.get(6)?; - let child_operation_ids_json: String = row.get(7)?; - let blockers_json: String = row.get(8)?; + let resource_counts_json: String = row.get(5)?; + let child_operation_ids_json: String = row.get(6)?; + let blockers_json: String = row.get(7)?; Ok(( row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?, row.get::<_, String>(3)?, state, - row.get::<_, bool>(5)?, resource_counts_json, child_operation_ids_json, blockers_json, - row.get::<_, Option>(9)?, + row.get::<_, Option>(8)?, + row.get::<_, String>(9)?, row.get::<_, String>(10)?, - row.get::<_, String>(11)?, - row.get::<_, Option>(12)?, + row.get::<_, Option>(11)?, )) }, ) @@ -408,7 +394,6 @@ fn read_operation( workspace_id, display_name, state, - force, resources, children, blockers, @@ -425,7 +410,6 @@ fn read_operation( workspace_id, display_name, state: parse_deletion_state(&state)?, - force_delete_dirty_workdirs: force, resources: serde_json::from_str(&resources) .map_err(|error| Error::Store(error.to_string()))?, child_operation_ids: serde_json::from_str(&children) @@ -474,6 +458,98 @@ fn owner_workspace( Ok(workspace) } +fn workspace_database_blockers( + conn: &rusqlite::Connection, + workspace_id: &str, +) -> Result> { + let mut blockers = Vec::new(); + for (sql, kind, resource_kind, message) in [ + ( + "SELECT workdir_id FROM worker_workdir_links WHERE workspace_id = ?1 AND unlinked_at IS NULL", + WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked, + "workdir", + "Release this active Worker–Workdir attachment before deleting the Workspace.", + ), + ( + "SELECT workdir_id FROM worker_workdir_attachment_reservations WHERE workspace_id = ?1", + WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked, + "workdir", + "Wait for or cancel this pending Workdir attachment reservation.", + ), + ( + "SELECT display_name FROM worker_registry WHERE workspace_id = ?1 AND retention_state = 'pinned'", + WorkspaceDeletionBlockerKind::RetentionHold, + "worker", + "Remove this Worker retention pin before deleting the Workspace.", + ), + ( + "SELECT workdir_id FROM workdir_registry WHERE workspace_id = ?1 AND COALESCE(cleanliness, '') != 'clean'", + WorkspaceDeletionBlockerKind::DirtyWorkdir, + "workdir", + "Clean this Workdir and refresh unknown cleanliness before deleting the Workspace.", + ), + ] { + for resource_key in query_resource_keys(conn, sql, workspace_id)? { + blockers.push(WorkspaceDeletionBlocker { + kind, + resource_kind: Some(resource_kind.to_string()), + resource_key: Some(resource_key), + message: message.to_string(), + }); + } + } + + for (sql, kind, resource_kind, message) in [ + ( + "SELECT COUNT(*) FROM ticket_current_worker_assignments WHERE workspace_id = ?1", + WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, + "ticket", + "Remove current Ticket assignments before deleting the Workspace.", + ), + ( + "SELECT COUNT(*) FROM worker_removal_operations WHERE workspace_id = ?1 AND state IN ('planned', 'blocked', 'executing', 'failed', 'stale')", + WorkspaceDeletionBlockerKind::CleanupUnavailable, + "worker", + "Resolve pending or failed Worker removal operations first.", + ), + ( + "SELECT COUNT(*) FROM workdir_removal_operations WHERE workspace_id = ?1 AND state IN ('pending', 'failed')", + WorkspaceDeletionBlockerKind::CleanupUnavailable, + "workdir", + "Resolve pending or failed Workdir removal operations first.", + ), + ( + "SELECT COUNT(*) FROM workdir_create_operations WHERE workspace_id = ?1 AND state = 'pending'", + WorkspaceDeletionBlockerKind::CleanupUnavailable, + "workdir", + "Wait for pending Workdir creation operations to finish.", + ), + ] { + let count: u64 = conn.query_row(sql, params![workspace_id], |row| row.get(0))?; + if count != 0 { + blockers.push(WorkspaceDeletionBlocker { + kind, + resource_kind: Some(resource_kind.to_string()), + resource_key: None, + message: format!("{message} ({count})"), + }); + } + } + Ok(blockers) +} + +fn query_resource_keys( + conn: &rusqlite::Connection, + sql: &str, + workspace_id: &str, +) -> Result> { + let mut statement = conn.prepare(sql)?; + statement + .query_map(params![workspace_id], |row| row.get(0))? + .collect::, _>>() + .map_err(Into::into) +} + fn resource_counts( conn: &rusqlite::Connection, workspace_id: &str, @@ -515,8 +591,8 @@ fn request_fingerprint( request: &WorkspaceDeletionRequest, ) -> String { let canonical = format!( - "workspace-delete-v1\0{actor_account_id}\0{workspace_id}\0{}\0{}\0{}", - request.expected_revision, request.confirmation, request.force_delete_dirty_workdirs + "workspace-delete-v1\0{actor_account_id}\0{workspace_id}\0{}\0{}", + request.expected_revision, request.confirmation ); encode_hex(&Sha256::digest(canonical.as_bytes())) } @@ -590,8 +666,7 @@ mod tests { let request = WorkspaceDeletionRequest { operation_id: "delete-workspace-a".to_string(), expected_revision: preflight.expected_revision, - confirmation: "delete Alpha".to_string(), - force_delete_dirty_workdirs: false, + confirmation: "Alpha".to_string(), }; let first = store .reserve_workspace_deletion(&owner, &workspace_id, &request) @@ -644,21 +719,54 @@ mod tests { let mut request = WorkspaceDeletionRequest { operation_id: "delete-alpha-guarded".to_string(), expected_revision: "stale".to_string(), - confirmation: "delete Alpha".to_string(), - force_delete_dirty_workdirs: false, + confirmation: "Alpha".to_string(), }; assert!(matches!( store.reserve_workspace_deletion(&owner, &workspace_id, &request), Err(Error::WorkspaceConfigConflict(_)) )); request.expected_revision = preflight.expected_revision; - request.confirmation = "Alpha".to_string(); + request.confirmation = "delete Alpha".to_string(); assert!(matches!( store.reserve_workspace_deletion(&owner, &workspace_id, &request), Err(Error::InvalidInput(_)) )); } + #[test] + fn pinned_worker_blocks_preflight_before_operation_reservation() { + let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) VALUES (?1, 'runtime-a', 'worker-a', 'Pinned worker', '1', '1', 'pinned')", + params![workspace_id], + )?; + Ok(()) + }) + .expect("worker"); + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + assert!(!preflight.can_delete); + assert!(preflight.blockers.iter().any(|blocker| { + blocker.kind == WorkspaceDeletionBlockerKind::RetentionHold + && blocker.resource_key.as_deref() == Some("Pinned worker") + })); + let request = WorkspaceDeletionRequest { + operation_id: "delete-pinned".to_string(), + expected_revision: preflight.expected_revision, + confirmation: "Alpha".to_string(), + }; + assert!(matches!( + store.reserve_workspace_deletion(&owner, &workspace_id, &request), + Err(Error::WorkspaceConfigConflict(_)) + )); + } + #[test] fn last_accessible_workspace_and_revision_conflicts_fail_closed() { let (store, owner, workspace_id) = setup(); @@ -675,8 +783,7 @@ mod tests { &WorkspaceDeletionRequest { operation_id: "delete-beta".to_string(), expected_revision: preflight.expected_revision, - confirmation: "delete Beta".to_string(), - force_delete_dirty_workdirs: false, + confirmation: "Beta".to_string(), }, ) .expect("reserve") diff --git a/web/workspace/src/lib/generated/workspace-api.ts b/web/workspace/src/lib/generated/workspace-api.ts index a48b3bb1..06cb5f1e 100644 --- a/web/workspace/src/lib/generated/workspace-api.ts +++ b/web/workspace/src/lib/generated/workspace-api.ts @@ -91,7 +91,6 @@ export type WorkspaceDeletionPreflightResponse = { */ expected_revision: string; can_delete: boolean; - force_delete_dirty_workdirs_available: boolean; resources: WorkspaceDeletionResourceCounts; blockers: Array; }; @@ -100,7 +99,6 @@ export type WorkspaceDeletionRequest = { operation_id: string; expected_revision: string; confirmation: string; - force_delete_dirty_workdirs: boolean; }; export type WorkspaceDeletionOperationResponse = { @@ -108,7 +106,6 @@ export type WorkspaceDeletionOperationResponse = { workspace_id: string; display_name: string; state: WorkspaceDeletionState; - force_delete_dirty_workdirs: boolean; resources: WorkspaceDeletionResourceCounts; child_operation_ids: Array; blockers: Array; diff --git a/web/workspace/src/lib/workspace/api/workspace-model.ts b/web/workspace/src/lib/workspace/api/workspace-model.ts index 5178fc73..d32305b9 100644 --- a/web/workspace/src/lib/workspace/api/workspace-model.ts +++ b/web/workspace/src/lib/workspace/api/workspace-model.ts @@ -665,7 +665,6 @@ export function parseWorkspaceDeletionPreflightResponse( "display_name", "expected_revision", "can_delete", - "force_delete_dirty_workdirs_available", "resources", "blockers", ], "Workspace deletion preflight"); @@ -686,10 +685,6 @@ export function parseWorkspaceDeletionPreflightResponse( item.can_delete, "Workspace deletion preflight.can_delete", ), - force_delete_dirty_workdirs_available: boolean( - item.force_delete_dirty_workdirs_available, - "Workspace deletion preflight.force_delete_dirty_workdirs_available", - ), resources: deletionResourceCounts( item.resources, "Workspace deletion preflight.resources", @@ -713,7 +708,6 @@ export function parseWorkspaceDeletionOperationResponse( "workspace_id", "display_name", "state", - "force_delete_dirty_workdirs", "resources", "child_operation_ids", "blockers", @@ -736,10 +730,6 @@ export function parseWorkspaceDeletionOperationResponse( "Workspace deletion operation.display_name", ), state: deletionState(item.state, "Workspace deletion operation.state"), - force_delete_dirty_workdirs: boolean( - item.force_delete_dirty_workdirs, - "Workspace deletion operation.force_delete_dirty_workdirs", - ), resources: deletionResourceCounts( item.resources, "Workspace deletion operation.resources", diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte index b5293f31..5b5df2cb 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte @@ -32,7 +32,6 @@ let deletionLoading = $state(false); let deletionSubmitting = $state(false); let deletionConfirmation = $state(''); - let forceDeleteDirtyWorkdirs = $state(false); let deletionPreflight = $state(null); let deletionOperation = $state(null); let deletionOperationId = $state(''); @@ -95,7 +94,6 @@ deletionOperation = null; deletionOperationId = crypto.randomUUID(); deletionConfirmation = ''; - forceDeleteDirtyWorkdirs = false; try { deletionPreflight = await preflightWorkspaceDeletion(workspaceId); } catch (err) { @@ -114,7 +112,6 @@ operation_id: deletionOperationId, expected_revision: deletionPreflight.expected_revision, confirmation: deletionConfirmation, - force_delete_dirty_workdirs: forceDeleteDirtyWorkdirs, }); deletionOperation = operation; while (operation.state === 'queued' || operation.state === 'running') { @@ -207,15 +204,9 @@

{blocker.message}

{/each} - {#if deletionPreflight.force_delete_dirty_workdirs_available} - - {/if} {/if} {#if deletionOperation}

Deletion state: {deletionOperation.state}

@@ -230,7 +221,7 @@ class="danger-button" type="button" onclick={() => void deleteWorkspace()} - disabled={deletionSubmitting || !deletionPreflight?.can_delete || deletionConfirmation !== `delete ${deletionPreflight?.display_name ?? ''}`} + disabled={deletionSubmitting || !deletionPreflight?.can_delete || deletionConfirmation !== (deletionPreflight?.display_name ?? '')} >{deletionSubmitting ? 'Deleting…' : 'Delete Workspace'} @@ -244,6 +235,5 @@ .modal-backdrop { position: fixed; inset: 0; z-index: 100; display: grid; place-items: center; padding: var(--space-4); background: rgb(0 0 0 / 0.55); } .deletion-dialog { width: min(34rem, 100%); max-height: calc(100vh - 2rem); overflow: auto; padding: var(--space-5); background: var(--color-surface, white); border: 1px solid var(--color-border); } .deletion-dialog label { display: grid; gap: var(--space-2); margin-block: var(--space-4); } - .force-delete-option { grid-template-columns: auto 1fr !important; align-items: start; } .dialog-actions { display: flex; justify-content: flex-end; gap: var(--space-2); margin-top: var(--space-5); } diff --git a/web/workspace/tests/workspace-model.test.ts b/web/workspace/tests/workspace-model.test.ts index 5374dfc4..eba48f0d 100644 --- a/web/workspace/tests/workspace-model.test.ts +++ b/web/workspace/tests/workspace-model.test.ts @@ -117,7 +117,6 @@ Deno.test("Workspace deletion DTOs fail closed and preserve durable operation st display_name: "Alpha", expected_revision: "2026-01-01T00:00:00Z", can_delete: true, - force_delete_dirty_workdirs_available: true, resources: { workers: 2, workdirs: 1, @@ -137,7 +136,6 @@ Deno.test("Workspace deletion DTOs fail closed and preserve durable operation st workspace_id: "workspace-a", display_name: "Alpha", state: "blocked", - force_delete_dirty_workdirs: false, resources: preflight.resources, child_operation_ids: ["worker-remove:arcadia/7"], blockers: [{ @@ -185,8 +183,7 @@ Deno.test("Workspace settings exposes owner-gated typed destructive confirmation "permissions.delete_workspace", "preflightWorkspaceDeletion", "startWorkspaceDeletion", - "delete ${", - "force_delete_dirty_workdirs", + "deletionConfirmation", "disposeWorkspaceMultiplexer(workspaceId)", ] ) { From 282a8d31b584c6cf55bd15804eb9741b22925b13 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 07:56:25 +0900 Subject: [PATCH 31/39] fix: enforce monotonic worker state projection --- crates/protocol/src/lib.rs | 99 +++++ crates/tui/src/app.rs | 99 ++++- crates/worker-runtime/src/worker_backend.rs | 352 ++++++++---------- crates/worker/src/controller.rs | 4 +- crates/worker/src/shared_state.rs | 44 ++- .../src/lib/workspace/console/model.test.ts | 60 +++ .../src/lib/workspace/console/model.ts | 68 +++- 7 files changed, 485 insertions(+), 241 deletions(-) diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index ec96e226..f66be5dc 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -203,6 +203,53 @@ impl WorkerStateSnapshot { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkerStateSnapshotApply { + Applied, + Duplicate, + Stale, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WorkerStateSnapshotConflict { + pub execution_generation: u64, + pub revision: u64, +} + +impl std::fmt::Display for WorkerStateSnapshotConflict { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "conflicting worker state snapshots at generation {} revision {}", + self.execution_generation, self.revision + ) + } +} + +impl std::error::Error for WorkerStateSnapshotConflict {} + +pub fn apply_worker_state_snapshot( + current: &mut WorkerStateSnapshot, + incoming: &WorkerStateSnapshot, +) -> Result { + use std::cmp::Ordering; + + let ordering = (incoming.execution_generation, incoming.revision) + .cmp(&(current.execution_generation, current.revision)); + match ordering { + Ordering::Greater => { + *current = incoming.clone(); + Ok(WorkerStateSnapshotApply::Applied) + } + Ordering::Less => Ok(WorkerStateSnapshotApply::Stale), + Ordering::Equal if incoming == current => Ok(WorkerStateSnapshotApply::Duplicate), + Ordering::Equal => Err(WorkerStateSnapshotConflict { + execution_generation: incoming.execution_generation, + revision: incoming.revision, + }), + } +} + impl From for WorkerStateSnapshot { fn from(status: WorkerStatus) -> Self { let state = match status { @@ -1574,6 +1621,58 @@ pub enum Permission { mod tests { use super::*; + #[test] + fn worker_state_snapshot_apply_is_monotonic_and_detects_conflicts() { + let mut current = WorkerStateSnapshot::initial(4); + let mut newer = current.clone(); + newer.revision = 1; + newer.state = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)); + + assert_eq!( + apply_worker_state_snapshot(&mut current, &newer), + Ok(WorkerStateSnapshotApply::Applied) + ); + assert_eq!( + apply_worker_state_snapshot(&mut current, &newer), + Ok(WorkerStateSnapshotApply::Duplicate) + ); + + let stale_revision = WorkerStateSnapshot::initial(4); + assert_eq!( + apply_worker_state_snapshot(&mut current, &stale_revision), + Ok(WorkerStateSnapshotApply::Stale) + ); + let stale_generation = WorkerStateSnapshot { + execution_generation: 3, + revision: u64::MAX, + ..newer.clone() + }; + assert_eq!( + apply_worker_state_snapshot(&mut current, &stale_generation), + Ok(WorkerStateSnapshotApply::Stale) + ); + + let conflicting = WorkerStateSnapshot { + state: WorkerState::Idle, + ..newer.clone() + }; + assert_eq!( + apply_worker_state_snapshot(&mut current, &conflicting), + Err(WorkerStateSnapshotConflict { + execution_generation: 4, + revision: 1, + }) + ); + assert_eq!(current, newer); + + let next_generation = WorkerStateSnapshot::initial(5); + assert_eq!( + apply_worker_state_snapshot(&mut current, &next_generation), + Ok(WorkerStateSnapshotApply::Applied) + ); + assert_eq!(current, next_generation); + } + #[test] fn method_submit_json_roundtrip_and_run_is_rejected() { let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#; diff --git a/crates/tui/src/app.rs b/crates/tui/src/app.rs index a76d32b5..c543f991 100644 --- a/crates/tui/src/app.rs +++ b/crates/tui/src/app.rs @@ -1128,6 +1128,22 @@ impl App { command } + fn apply_worker_state_snapshot(&mut self, snapshot: &WorkerStateSnapshot) { + match protocol::apply_worker_state_snapshot(&mut self.worker_state, snapshot) { + Ok(protocol::WorkerStateSnapshotApply::Applied) => { + self.set_worker_status(self.worker_state.catalog_status()); + } + Ok( + protocol::WorkerStateSnapshotApply::Duplicate + | protocol::WorkerStateSnapshotApply::Stale, + ) => {} + Err(error) => self.handle_error( + ErrorCode::Internal, + format!("worker state stream rejected: {error}"), + ), + } + } + pub fn handle_worker_event(&mut self, event: Event) -> Option { if self.rewind_refresh_fence && event_is_stale_after_rewind(&event) { return None; @@ -1465,8 +1481,7 @@ impl App { self.pending_submissions = session.pending_submissions.clone(); self.restore_snapshot(&session, greeting, in_flight); self.replace_internal_worker_snapshots(internal_workers); - self.worker_state = state.clone(); - self.set_worker_status(state.catalog_status()); + self.apply_worker_state_snapshot(&state); } Event::InternalWorker { worker, @@ -1478,12 +1493,10 @@ impl App { } Event::WorkerState { snapshot } => { self.rewind_refresh_fence = false; - self.worker_state = snapshot.clone(); - self.set_worker_status(snapshot.catalog_status()); + self.apply_worker_state_snapshot(&snapshot); } Event::CommandAcknowledged { acknowledgement } => { - self.worker_state = acknowledgement.state.clone(); - self.set_worker_status(acknowledgement.state.catalog_status()); + self.apply_worker_state_snapshot(&acknowledgement.state); } // Command telemetry is an operational Web Console surface. The // TUI continues to render the final Bash ToolResult from history. @@ -3559,7 +3572,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: public_session(vec![session_start_value]), - state: WorkerStatus::Running.into(), + state: test_worker_state(WorkerStatus::Running), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -3570,6 +3583,59 @@ mod completion_flow_tests { assert!(matches!(app.blocks.first(), Some(Block::Greeting(_)))); } + #[test] + fn worker_state_events_and_acknowledgements_share_monotonic_application() { + let mut app = App::new("test".into()); + let running = WorkerStateSnapshot { + execution_generation: 4, + revision: 3, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 2, + }; + app.handle_worker_event(Event::WorkerState { + snapshot: running.clone(), + }); + app.handle_worker_event(Event::WorkerState { + snapshot: WorkerStateSnapshot { + revision: 2, + state: protocol::WorkerState::Idle, + ..running.clone() + }, + }); + assert_eq!(app.worker_state, running); + + let paused = WorkerStateSnapshot { + revision: 4, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused, + )), + last_command_id: 3, + ..running.clone() + }; + app.handle_worker_event(Event::CommandAcknowledged { + acknowledgement: protocol::WorkerCommandAcknowledgement { + command_id: 3, + command: protocol::WorkerCommandKind::Pause, + disposition: protocol::WorkerCommandDisposition::Accepted, + state: paused.clone(), + }, + }); + assert_eq!(app.worker_state, paused); + + app.handle_worker_event(Event::WorkerState { + snapshot: WorkerStateSnapshot { + state: protocol::WorkerState::Idle, + ..paused.clone() + }, + }); + assert_eq!(app.worker_state, paused); + assert!(app.run_error_messages.iter().any(|message| { + message.contains("conflicting worker state snapshots at generation 4 revision 4") + })); + } + #[test] fn snapshot_replaces_live_error_with_one_durable_run_error_block() { let mut app = App::new("test".into()); @@ -3603,7 +3669,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: public_session(vec![serde_json::to_value(run_errored).unwrap()]), - state: WorkerStatus::Idle.into(), + state: test_worker_state(WorkerStatus::Idle), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -3667,7 +3733,7 @@ mod completion_flow_tests { pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - state: WorkerStatus::Running.into(), + state: test_worker_state(WorkerStatus::Running), in_flight: InFlightSnapshot { blocks: vec![ InFlightBlock::Thinking { @@ -3994,7 +4060,7 @@ mod completion_flow_tests { pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - state: WorkerStatus::Idle.into(), + state: test_worker_state(WorkerStatus::Idle), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -4046,7 +4112,7 @@ mod completion_flow_tests { pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - state: WorkerStatus::Idle.into(), + state: test_worker_state(WorkerStatus::Idle), in_flight: Default::default(), internal_workers: vec![InternalWorkerSnapshot { worker: InternalWorkerRef { @@ -4194,6 +4260,13 @@ mod completion_flow_tests { .count() } + fn test_worker_state(status: WorkerStatus) -> WorkerStateSnapshot { + let mut snapshot = WorkerStateSnapshot::from(status); + snapshot.execution_generation = 1; + snapshot.revision = 1; + snapshot + } + fn test_greeting() -> protocol::Greeting { protocol::Greeting { worker_name: "test".into(), @@ -4220,7 +4293,7 @@ mod completion_flow_tests { entries: Vec::new(), }, greeting, - state: WorkerStatus::Idle.into(), + state: test_worker_state(WorkerStatus::Idle), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -4419,7 +4492,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: public_session(assistant_item_entries), - state: WorkerStatus::Running.into(), + state: test_worker_state(WorkerStatus::Running), in_flight: Default::default(), internal_workers: Vec::new(), }); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 6c414874..20b05aed 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -10,7 +10,7 @@ use std::collections::HashMap; use std::future::Future; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, RwLock, mpsc}; use std::time::Duration; @@ -38,7 +38,9 @@ use crate::working_directory::{ WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer, }; use async_trait::async_trait; -use protocol::{Event, Method, Segment, WorkerCommandEnvelope, WorkerStatus}; +#[cfg(test)] +use protocol::WorkerStatus; +use protocol::{Event, Method, Segment, WorkerCommandEnvelope}; static NEXT_INTERNAL_COMMAND_ID: AtomicU64 = AtomicU64::new(1); @@ -1197,7 +1199,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { struct RuntimeWorkerExecution { handle: WorkerHandle, shutdown: Arc>>, - busy: Arc, worker_state: Arc>, workspace_client: Option>, } @@ -1296,7 +1297,6 @@ where ) -> Result< ( WorkerHandle, - Arc, Arc>, Option>, ), @@ -1323,7 +1323,6 @@ where .map(|execution| { ( execution.handle.clone(), - execution.busy.clone(), execution.worker_state.clone(), execution.workspace_client.clone(), ) @@ -1434,48 +1433,31 @@ where working_directory: Option, workspace_client: Option>, ) -> WorkerExecutionSpawnResult { - let busy = Arc::new(AtomicBool::new(false)); let worker_state = Arc::new(RwLock::new(handle.shared_state.snapshot())); #[cfg(feature = "ws-server")] { let streams = subscribe_worker_protocol_session(&handle); let mut events = streams.events; let mut entry_events = streams.log_entries; - let bridge_busy = busy.clone(); let bridge_worker_state = worker_state.clone(); if let Err(message) = self.spawn_on_adapter_runtime(async move { loop { tokio::select! { event = events.recv() => { match event { - Ok(event) => { - let next_state = match &event { - Event::WorkerState { snapshot } - | Event::Snapshot { state: snapshot, .. } => { - Some(snapshot.clone()) + Ok(mut event) => { + match apply_protocol_worker_state(&bridge_worker_state, &mut event) { + Ok(true) => { + let _ = bridge_context.publish_protocol_event(event); } - Event::CommandAcknowledged { acknowledgement } => { - Some(acknowledgement.state.clone()) + Ok(false) => {} + Err(message) => { + let _ = bridge_context.publish_protocol_event(Event::Error { + code: protocol::ErrorCode::Internal, + message: format!("worker state stream rejected: {message}"), + }); + break; } - _ => None, - }; - let next_busy = next_state - .as_ref() - .map(worker_state_is_executing) - .or_else(|| matches!(event, Event::Shutdown).then_some(false)); - let _ = bridge_context.publish_protocol_event(event); - if let Some(next_state) = next_state { - if let Ok(mut current) = bridge_worker_state.write() { - if next_state.execution_generation > current.execution_generation - || (next_state.execution_generation == current.execution_generation - && next_state.revision >= current.revision) - { - *current = next_state; - } - } - } - if let Some(next_busy) = next_busy { - bridge_busy.store(next_busy, Ordering::SeqCst); } } Err(broadcast::error::RecvError::Lagged(_)) => continue, @@ -1520,7 +1502,6 @@ where RuntimeWorkerExecution { handle, shutdown, - busy, worker_state, workspace_client, }, @@ -1543,32 +1524,28 @@ impl Drop for WorkerRuntimeExecutionBackend { } } -fn worker_state_is_executing(snapshot: &protocol::WorkerStateSnapshot) -> bool { - matches!( - snapshot.state, - protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( - protocol::WorkerRunState::Running - | protocol::WorkerRunState::Pausing - | protocol::WorkerRunState::Cancelling - )) | protocol::WorkerState::Busy(protocol::WorkerBusyState::Maintenance(_)) - ) -} - -fn method_starts_turn(method: &Method) -> bool { - matches!( - method, - Method::Submit { .. } - | Method::SubmitTracked { .. } - | Method::Notify { auto_run: true, .. } - | Method::NotifyTracked { auto_run: true, .. } - | Method::Resume { .. } - ) -} - -fn method_can_start_turn_from_status(method: &Method, status: WorkerStatus) -> bool { - match method { - Method::Resume { .. } => matches!(status, WorkerStatus::Idle | WorkerStatus::Paused), - _ => status == WorkerStatus::Idle, +fn apply_protocol_worker_state( + current: &Arc>, + event: &mut Event, +) -> Result { + let (incoming, replace_stale) = match event { + Event::WorkerState { snapshot } => (snapshot, false), + Event::Snapshot { state, .. } => (state, true), + Event::CommandAcknowledged { acknowledgement } => (&mut acknowledgement.state, true), + _ => return Ok(true), + }; + let mut current = current + .write() + .map_err(|_| "worker state projection lock is poisoned".to_string())?; + match protocol::apply_worker_state_snapshot(&mut current, incoming) { + Ok(protocol::WorkerStateSnapshotApply::Applied) + | Ok(protocol::WorkerStateSnapshotApply::Duplicate) => Ok(true), + Ok(protocol::WorkerStateSnapshotApply::Stale) if replace_stale => { + *incoming = current.clone(); + Ok(true) + } + Ok(protocol::WorkerStateSnapshotApply::Stale) => Ok(false), + Err(error) => Err(error.to_string()), } } @@ -1897,7 +1874,7 @@ where handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - let (worker, busy, worker_state, _workspace_client) = match self.get_execution(handle) { + let (worker, worker_state, _workspace_client) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::Input; @@ -1906,15 +1883,10 @@ where }; if input.kind == WorkerInputKind::Notify { - let status = worker.shared_state.catalog_status(); - let claimed_here = status == WorkerStatus::Idle - && busy - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok(); let notification_request_id = input .submission_request_id .unwrap_or_else(protocol::new_submission_request_id); - let result = self.send_method( + return self.send_method( WorkerExecutionOperation::Input, worker, Method::NotifyTracked { @@ -1926,11 +1898,6 @@ where }, }, ); - if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted - { - busy.store(false, Ordering::SeqCst); - } - return result; } if input.kind == WorkerInputKind::Compact { @@ -1947,26 +1914,12 @@ where ); } - let is_user_submit = input.kind == WorkerInputKind::User; - let status = worker.shared_state.catalog_status(); - let claimed_here = status == WorkerStatus::Idle - && busy - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok(); - if !is_user_submit && !claimed_here { - return WorkerExecutionResult::busy( - WorkerExecutionOperation::Input, - "Worker is already running", - ); - } - let (method, submission_request_id) = match input.kind { WorkerInputKind::User => { let Some(submission_id) = input .submission_request_id .filter(|submission_id| !submission_id.trim().is_empty()) else { - busy.store(false, Ordering::SeqCst); return WorkerExecutionResult::rejected( WorkerExecutionOperation::Input, "Runtime user input is missing its internal submission id", @@ -1986,7 +1939,7 @@ where ) } WorkerInputKind::Notify => { - unreachable!("Notify input is dispatched before the turn-start busy guard") + unreachable!("Notify input is dispatched before ordinary input mapping") } WorkerInputKind::Compact => unreachable!("compact input is dispatched above"), WorkerInputKind::ListRewindTargets => (Method::ListRewindTargets, None), @@ -1999,7 +1952,7 @@ where }; let waits_for_submission_acceptance = submission_request_id.is_some(); - let result = if waits_for_submission_acceptance { + if waits_for_submission_acceptance { self.send_submit_and_wait_for_acceptance( WorkerExecutionOperation::Input, worker, @@ -2008,11 +1961,7 @@ where ) } else { self.send_method(WorkerExecutionOperation::Input, worker, method) - }; - if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { - busy.store(false, Ordering::SeqCst); } - result } fn upload_file( @@ -2023,7 +1972,7 @@ where content: &[u8], context: Option<&session_store::UploadedFileUploadContext>, ) -> Result { - let (worker, _, _, _) = self.get_execution(handle).map_err(|mut result| { + let (worker, _, _) = self.get_execution(handle).map_err(|mut result| { result.operation = WorkerExecutionOperation::UploadFile; result })?; @@ -2046,7 +1995,7 @@ where handle: &WorkerExecutionHandle, artifact_id: &str, ) -> WorkerExecutionResult { - let (worker, _, _, _) = match self.get_execution(handle) { + let (worker, _, _) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::DeleteUploadedFile; @@ -2067,7 +2016,7 @@ where handle: &WorkerExecutionHandle, method: Method, ) -> WorkerExecutionResult { - let (worker, busy, _worker_state, _workspace_client) = match self.get_execution(handle) { + let (worker, _worker_state, _workspace_client) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::ProtocolMethod; @@ -2075,44 +2024,7 @@ where } }; - if let Some(auto_run) = match &method { - Method::Notify { auto_run, .. } | Method::NotifyTracked { auto_run, .. } => { - Some(*auto_run) - } - _ => None, - } { - let status = worker.shared_state.catalog_status(); - let claimed_here = status == WorkerStatus::Idle - && auto_run - && busy - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok(); - let result = self.send_method(WorkerExecutionOperation::ProtocolMethod, worker, method); - if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted - { - busy.store(false, Ordering::SeqCst); - } - return result; - } - - let starts_turn = method_starts_turn(&method); - if starts_turn - && (!method_can_start_turn_from_status(&method, worker.shared_state.catalog_status()) - || busy - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_err()) - { - return WorkerExecutionResult::busy( - WorkerExecutionOperation::ProtocolMethod, - "Worker is already running; runtime adapter v0 does not queue protocol methods", - ); - } - - let result = self.send_method(WorkerExecutionOperation::ProtocolMethod, worker, method); - if starts_turn && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { - busy.store(false, Ordering::SeqCst); - } - result + self.send_method(WorkerExecutionOperation::ProtocolMethod, worker, method) } fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { @@ -2193,7 +2105,7 @@ where } fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - let (worker, _busy, worker_state, _workspace_client) = match self.get_execution(handle) { + let (worker, worker_state, _workspace_client) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::Cancel; @@ -2297,6 +2209,56 @@ mod tests { WorkerCommandEnvelope::for_snapshot(state.last_command_id.saturating_add(1), &state) } + #[test] + fn protocol_bridge_applies_state_and_acknowledgement_monotonically() { + let running = protocol::WorkerStateSnapshot { + execution_generation: 4, + revision: 3, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 2, + }; + let current = Arc::new(RwLock::new(running.clone())); + let mut stale = Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + revision: 2, + state: protocol::WorkerState::Idle, + ..running.clone() + }, + }; + assert!(!apply_protocol_worker_state(¤t, &mut stale).unwrap()); + assert_eq!(*current.read().unwrap(), running); + + let paused = protocol::WorkerStateSnapshot { + revision: 4, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused, + )), + last_command_id: 3, + ..running.clone() + }; + let mut acknowledgement = Event::CommandAcknowledged { + acknowledgement: protocol::WorkerCommandAcknowledgement { + command_id: 3, + command: protocol::WorkerCommandKind::Pause, + disposition: protocol::WorkerCommandDisposition::Accepted, + state: paused.clone(), + }, + }; + assert!(apply_protocol_worker_state(¤t, &mut acknowledgement).unwrap()); + assert_eq!(*current.read().unwrap(), paused); + + let mut conflict = Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Idle, + ..paused.clone() + }, + }; + assert!(apply_protocol_worker_state(¤t, &mut conflict).is_err()); + assert_eq!(*current.read().unwrap(), paused); + } + #[test] fn workspace_prompt_projection_notification_advances_shared_cache() { let cache = WorkspacePromptProjectionCache::default(); @@ -2443,44 +2405,6 @@ mod tests { assert_eq!(after_restore_workspace_id.as_deref(), Some("workspace-a")); } - #[test] - fn compact_is_maintenance_not_a_turn_start() { - assert!(!method_starts_turn(&Method::Compact { - command: test_command(), - })); - assert!(method_starts_turn(&Method::Resume { - command: test_command(), - })); - } - - #[test] - fn resume_turn_claim_accepts_paused_and_idle_but_not_running_status() { - assert!(method_can_start_turn_from_status( - &Method::Resume { - command: test_command() - }, - WorkerStatus::Paused - )); - assert!(method_can_start_turn_from_status( - &Method::Resume { - command: test_command() - }, - WorkerStatus::Idle - )); - assert!(!method_can_start_turn_from_status( - &Method::Resume { - command: test_command() - }, - WorkerStatus::Running - )); - assert!(!method_can_start_turn_from_status( - &Method::Compact { - command: test_command() - }, - WorkerStatus::Paused - )); - } - #[derive(Clone)] enum MockResponse { Complete(Vec), @@ -2681,11 +2605,38 @@ mod tests { .collect() } + fn wait_for_adapter_command( + backend: &WorkerRuntimeExecutionBackend, + worker_ref: &WorkerRef, + expected_command_id: u64, + ) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let observed = { + let workers = backend.workers.lock().unwrap(); + workers + .get(worker_ref) + .expect("live Worker execution") + .worker_state + .read() + .unwrap() + .last_command_id + }; + if observed >= expected_command_id { + return; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for adapter command {expected_command_id}; last observed={observed}", + ); + std::thread::sleep(Duration::from_millis(10)); + } + } + fn wait_for_adapter_state( backend: &WorkerRuntimeExecutionBackend, worker_ref: &WorkerRef, expected_status: WorkerStatus, - expected_busy: bool, ) { let deadline = std::time::Instant::now() + Duration::from_secs(5); loop { @@ -2693,21 +2644,16 @@ mod tests { let workers = backend.workers.lock().unwrap(); let execution = workers.get(worker_ref).expect("live Worker execution"); let projected = execution.worker_state.read().unwrap().catalog_status(); - ( - execution.handle.shared_state.catalog_status(), - projected, - execution.busy.load(Ordering::SeqCst), - ) + (execution.handle.shared_state.catalog_status(), projected) }; - if observed == (expected_status, expected_status, expected_busy) { + if observed == (expected_status, expected_status) { return; } assert!( std::time::Instant::now() < deadline, - "timed out waiting for adapter state {expected_status:?}, busy={expected_busy}; last observed controller={:?}, projected={:?}, busy={}", + "timed out waiting for adapter state {expected_status:?}; last observed controller={:?}, projected={:?}", observed.0, observed.1, - observed.2, ); std::thread::sleep(Duration::from_millis(10)); } @@ -3694,22 +3640,18 @@ mod tests { runtime .send_input(&detail.worker_ref, WorkerInput::user("pause and resume")) .expect("start initial turn"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running); - let running_resume = runtime + let running_resume = adapter_command(&backend, &detail.worker_ref); + runtime .send_protocol_method( &detail.worker_ref, Method::Resume { - command: adapter_command(&backend, &detail.worker_ref), + command: running_resume, }, ) - .expect_err("Resume while Running must be rejected"); - assert!( - running_resume - .to_string() - .contains("does not queue protocol methods"), - "unexpected Running Resume error: {running_resume}" - ); + .expect("running Resume is forwarded for controller admission"); + wait_for_adapter_command(&backend, &detail.worker_ref, running_resume.command_id); runtime .send_protocol_method( @@ -3719,7 +3661,7 @@ mod tests { }, ) .expect("pause initial turn"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused); runtime .send_protocol_method( @@ -3729,22 +3671,18 @@ mod tests { }, ) .expect("resume paused turn"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running); - let duplicate_resume = runtime + let duplicate_resume = adapter_command(&backend, &detail.worker_ref); + runtime .send_protocol_method( &detail.worker_ref, Method::Resume { - command: adapter_command(&backend, &detail.worker_ref), + command: duplicate_resume, }, ) - .expect_err("duplicate Resume must be rejected"); - assert!( - duplicate_resume - .to_string() - .contains("does not queue protocol methods"), - "unexpected duplicate Resume error: {duplicate_resume}" - ); + .expect("duplicate Resume is forwarded for controller admission"); + wait_for_adapter_command(&backend, &detail.worker_ref, duplicate_resume.command_id); runtime .send_protocol_method( @@ -3754,7 +3692,7 @@ mod tests { }, ) .expect("pause resumed turn"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused); runtime .send_protocol_method( &detail.worker_ref, @@ -3763,18 +3701,20 @@ mod tests { }, ) .expect("resume paused turn a second time"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle); assert_eq!(call_count.load(Ordering::SeqCst), 3); + let idle_resume = adapter_command(&backend, &detail.worker_ref); runtime .send_protocol_method( &detail.worker_ref, Method::Resume { - command: adapter_command(&backend, &detail.worker_ref), + command: idle_resume, }, ) .expect("Idle Resume preserves controller NotPaused semantics"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false); + wait_for_adapter_command(&backend, &detail.worker_ref, idle_resume.command_id); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle); let events = runtime .read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero()) .expect("read protocol events"); diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index f9cc2388..beeb4791 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -3708,7 +3708,7 @@ mod tests { WorkerCommandEnvelope { command_id: 1, expected_execution_generation: 9, - expected_worker_state_revision: 0, + expected_worker_state_revision: 1, }, &shared, ), @@ -3719,7 +3719,7 @@ mod tests { WorkerCommandEnvelope { command_id: 2, expected_execution_generation: 9, - expected_worker_state_revision: 0, + expected_worker_state_revision: 1, }, &shared, ) diff --git a/crates/worker/src/shared_state.rs b/crates/worker/src/shared_state.rs index 58c5700b..573f2bb5 100644 --- a/crates/worker/src/shared_state.rs +++ b/crates/worker/src/shared_state.rs @@ -1,6 +1,6 @@ use std::sync::{ OnceLock, RwLock, - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, Ordering}, }; use protocol::{ @@ -23,7 +23,6 @@ pub struct WorkerSharedState { pub manifest_toml: String, pub greeting: protocol::Greeting, state: RwLock, - last_command_id: AtomicU64, /// Worker-from-the-inside view of the filesystem. Set once in /// `WorkerController::start` after the local WorkdirSession provider is /// materialised, and read from the IPC server layer to answer @@ -56,7 +55,6 @@ impl WorkerSharedState { manifest_toml, greeting, state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)), - last_command_id: AtomicU64::new(0), fs_view: OnceLock::new(), flow_transition_enabled: AtomicBool::new(false), } @@ -91,26 +89,27 @@ impl WorkerSharedState { snapshot.revision = snapshot.revision.saturating_add(1); snapshot.state = state; } - snapshot.last_command_id = self.last_command_id.load(Ordering::Acquire); snapshot.clone() } pub fn accept_command_id(&self, command_id: u64) -> bool { - self.last_command_id - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { - (command_id > current).then_some(command_id) - }) - .is_ok() + let mut snapshot = self + .state + .write() + .expect("worker state lock poisoned; refusing command admission"); + if command_id <= snapshot.last_command_id { + return false; + } + snapshot.last_command_id = command_id; + snapshot.revision = snapshot.revision.saturating_add(1); + true } pub fn snapshot(&self) -> WorkerStateSnapshot { - let mut snapshot = self - .state + self.state .read() .expect("worker state lock poisoned; refusing an inferred fallback state") - .clone(); - snapshot.last_command_id = self.last_command_id.load(Ordering::Acquire); - snapshot + .clone() } /// Runtime catalog projection. This must not be used as live command @@ -190,6 +189,23 @@ mod tests { assert_eq!(state.catalog_status(), WorkerStatus::Paused); } + #[test] + fn accepted_command_id_advances_the_snapshot_revision_atomically() { + let state = test_state(); + assert!(state.accept_command_id(9)); + assert_eq!( + state.snapshot(), + WorkerStateSnapshot { + execution_generation: 7, + revision: 1, + last_command_id: 9, + state: WorkerState::Idle, + } + ); + assert!(!state.accept_command_id(9)); + assert_eq!(state.snapshot().revision, 1); + } + #[test] fn status_json_contains_full_snapshot_and_catalog_projection() { let state = test_state(); diff --git a/web/workspace/src/lib/workspace/console/model.test.ts b/web/workspace/src/lib/workspace/console/model.test.ts index c1f1b72a..de1e46b3 100644 --- a/web/workspace/src/lib/workspace/console/model.test.ts +++ b/web/workspace/src/lib/workspace/console/model.test.ts @@ -218,6 +218,66 @@ Deno.test("console routing projects live errors but not completion replies", () ); }); +Deno.test("Worker state events and acknowledgements apply monotonically", () => { + const projector = createConsoleProjector(); + const running: WorkerStateSnapshot = { + execution_generation: 4, + revision: 3, + last_command_id: 2, + state: { kind: "busy", state: { kind: "run", state: "running" } }, + }; + const paused: WorkerStateSnapshot = { + ...running, + revision: 4, + last_command_id: 3, + state: { kind: "busy", state: { kind: "run", state: "paused" } }, + }; + let projection = projector.append([ + { + eventId: "running", + event: { event: "worker_state", data: { snapshot: running } }, + }, + { + eventId: "stale", + event: { + event: "worker_state", + data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } }, + }, + }, + { + eventId: "pause-ack", + event: { + event: "command_acknowledged", + data: { + acknowledgement: { + command_id: 3, + command: "pause", + disposition: "accepted", + state: paused, + }, + }, + }, + }, + ]); + assertEquals(projection.workerState, paused); + assertEquals(projection.status, "paused"); + + projection = projector.append([{ + eventId: "conflict", + event: { + event: "worker_state", + data: { snapshot: { ...paused, state: { kind: "idle" } } }, + }, + }]); + assertEquals(projection.workerState, paused); + assert( + projection.lines.some((line) => + line.eventId === "conflict:worker-state-conflict" && line.error + ), + "conflicting equal-version snapshots must fail closed", + ); +}); + Deno.test("snapshot replaces a live error with one durable run_errored row", () => { const projector = createConsoleProjector(); let projection = projector.append([ diff --git a/web/workspace/src/lib/workspace/console/model.ts b/web/workspace/src/lib/workspace/console/model.ts index 857e1b30..95578cbd 100644 --- a/web/workspace/src/lib/workspace/console/model.ts +++ b/web/workspace/src/lib/workspace/console/model.ts @@ -10,6 +10,7 @@ import type { InternalWorkerRef, InternalWorkerSnapshot, Segment, + WorkerState, WorkerStateSnapshot, WorkerStatus, } from "$lib/generated/protocol"; @@ -796,6 +797,60 @@ function refreshCompactionActivity( return changed ? { ...projection, lines } : projection; } +function workerStateEqual(left: WorkerState, right: WorkerState): boolean { + if (left.kind !== right.kind) return false; + if (left.kind === "idle" || right.kind === "idle") return true; + return left.state.kind === right.state.kind && + left.state.state === right.state.state; +} + +function workerStateSnapshotEqual( + left: WorkerStateSnapshot, + right: WorkerStateSnapshot, +): boolean { + return left.execution_generation === right.execution_generation && + left.revision === right.revision && + left.last_command_id === right.last_command_id && + workerStateEqual(left.state, right.state); +} + +function applyWorkerStateSnapshot( + projection: ConsoleProjection, + incoming: WorkerStateSnapshot, + eventId: string, +): void { + const current = projection.workerState; + if (!current) { + projection.workerState = incoming; + projection.status = workerStatusFromState(incoming); + return; + } + const generationOrder = incoming.execution_generation - + current.execution_generation; + const revisionOrder = incoming.revision - current.revision; + if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) { + projection.workerState = incoming; + projection.status = workerStatusFromState(incoming); + return; + } + if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) { + return; + } + if (!workerStateSnapshotEqual(current, incoming)) { + projection.lines.push( + line( + `${eventId}:worker-state-conflict`, + "error", + "error · internal", + `worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`, + undefined, + false, + true, + ), + ); + } +} + export function applyProtocolEvent( projection: ConsoleProjection, envelope: ConsoleEventInput, @@ -917,8 +972,6 @@ export function applyProtocolEvent( ); break; case "snapshot": { - next.workerState = event.data.state; - next.status = workerStatusFromState(event.data.state); next.cwd = event.data.greeting.cwd; const snapshot = snapshotProjectionFromSession( envelope.eventId, @@ -968,6 +1021,7 @@ export function applyProtocolEvent( }; } } + applyWorkerStateSnapshot(next, event.data.state, envelope.eventId); break; } case "internal_worker": { @@ -1016,12 +1070,14 @@ export function applyProtocolEvent( break; } case "worker_state": - next.workerState = event.data.snapshot; - next.status = workerStatusFromState(event.data.snapshot); + applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId); break; case "command_acknowledged": - next.workerState = event.data.acknowledgement.state; - next.status = workerStatusFromState(event.data.acknowledgement.state); + applyWorkerStateSnapshot( + next, + event.data.acknowledgement.state, + envelope.eventId, + ); break; case "command": applyCommandEvent(next, envelope.eventId, event.data.event); From 38627c498b942125eb007bd807ce3bc16c169cd2 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 08:57:54 +0900 Subject: [PATCH 32/39] fix: harden workspace deletion recovery --- crates/workspace-api/src/lib.rs | 289 +++++++++++++++++- crates/workspace-server/src/server.rs | 186 ++++++++--- .../src/workspace_deletion.rs | 176 ++++++++++- .../src/lib/workspace/api/workspace-model.ts | 88 +++++- .../settings/workspace-deletion-api.ts | 55 ++-- .../workspace/sidebar/worker-subscription.ts | 4 + .../src/routes/w/[workspaceId]/+layout.svelte | 6 +- .../settings/workspace/+page.svelte | 93 ++++-- web/workspace/tests/workspace-model.test.ts | 20 ++ 9 files changed, 814 insertions(+), 103 deletions(-) diff --git a/crates/workspace-api/src/lib.rs b/crates/workspace-api/src/lib.rs index 14a25295..8ab5dad7 100644 --- a/crates/workspace-api/src/lib.rs +++ b/crates/workspace-api/src/lib.rs @@ -607,6 +607,124 @@ pub struct WorkspaceMetadataMutationResponse { pub diagnostics: Vec, } +pub const WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES: usize = 128; +pub const WORKSPACE_DELETION_MAX_REVISION_BYTES: usize = 128; +pub const WORKSPACE_DELETION_MAX_CONFIRMATION_BYTES: usize = 256; +pub const WORKSPACE_DELETION_MAX_BLOCKERS: usize = 1024; +pub const WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS: usize = 4096; +pub const WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES: usize = 128; +pub const WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES: usize = 512; + +fn deserialize_workspace_deletion_operation_id<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES { + return Err(serde::de::Error::custom( + "Workspace deletion operation_id is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_revision<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_REVISION_BYTES { + return Err(serde::de::Error::custom( + "Workspace deletion revision is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_confirmation<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_CONFIRMATION_BYTES { + return Err(serde::de::Error::custom( + "Workspace deletion confirmation is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_resource_value<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + if value + .as_ref() + .is_some_and(|value| value.len() > WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES) + { + return Err(serde::de::Error::custom( + "Workspace deletion resource value is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_blocker_message<'de, D>( + deserializer: D, +) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES { + return Err(serde::de::Error::custom( + "Workspace deletion blocker message is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_blockers<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Vec::::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_BLOCKERS { + return Err(serde::de::Error::custom( + "too many Workspace deletion blockers", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_child_operation_ids<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Vec::::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS { + return Err(serde::de::Error::custom( + "too many Workspace deletion child operations", + )); + } + if value + .iter() + .any(|operation_id| operation_id.len() > WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES) + { + return Err(serde::de::Error::custom( + "Workspace deletion child operation_id is too long", + )); + } + Ok(value) +} + /// Lifecycle state for one durable Workspace deletion operation. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] @@ -634,9 +752,8 @@ pub enum WorkspaceDeletionBlockerKind { } /// One bounded, user-actionable blocker returned by preflight or execution. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] -#[serde(deny_unknown_fields)] pub struct WorkspaceDeletionBlocker { pub kind: WorkspaceDeletionBlockerKind, pub resource_kind: Option, @@ -664,9 +781,8 @@ pub struct WorkspaceDeletionResourceCounts { } /// Owner-only impact preview for deleting one Workspace. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] -#[serde(deny_unknown_fields)] pub struct WorkspaceDeletionPreflightResponse { pub workspace_id: String, pub display_name: String, @@ -678,9 +794,8 @@ pub struct WorkspaceDeletionPreflightResponse { } /// Idempotent request to start or resume Workspace deletion. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] -#[serde(deny_unknown_fields)] pub struct WorkspaceDeletionRequest { pub operation_id: String, pub expected_revision: String, @@ -688,9 +803,8 @@ pub struct WorkspaceDeletionRequest { } /// Durable deletion operation projection used by request responses and polling. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] -#[serde(deny_unknown_fields)] pub struct WorkspaceDeletionOperationResponse { pub operation_id: String, pub workspace_id: String, @@ -705,6 +819,129 @@ pub struct WorkspaceDeletionOperationResponse { pub completed_at: Option, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceDeletionBlockerWire { + kind: WorkspaceDeletionBlockerKind, + #[serde(deserialize_with = "deserialize_workspace_deletion_resource_value")] + resource_kind: Option, + #[serde(deserialize_with = "deserialize_workspace_deletion_resource_value")] + resource_key: Option, + #[serde(deserialize_with = "deserialize_workspace_deletion_blocker_message")] + message: String, +} + +impl<'de> Deserialize<'de> for WorkspaceDeletionBlocker { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = WorkspaceDeletionBlockerWire::deserialize(deserializer)?; + Ok(Self { + kind: wire.kind, + resource_kind: wire.resource_kind, + resource_key: wire.resource_key, + message: wire.message, + }) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceDeletionPreflightResponseWire { + workspace_id: String, + display_name: String, + #[serde(deserialize_with = "deserialize_workspace_deletion_revision")] + expected_revision: String, + can_delete: bool, + resources: WorkspaceDeletionResourceCounts, + #[serde(deserialize_with = "deserialize_workspace_deletion_blockers")] + blockers: Vec, +} + +impl<'de> Deserialize<'de> for WorkspaceDeletionPreflightResponse { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = WorkspaceDeletionPreflightResponseWire::deserialize(deserializer)?; + Ok(Self { + workspace_id: wire.workspace_id, + display_name: wire.display_name, + expected_revision: wire.expected_revision, + can_delete: wire.can_delete, + resources: wire.resources, + blockers: wire.blockers, + }) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceDeletionRequestWire { + #[serde(deserialize_with = "deserialize_workspace_deletion_operation_id")] + operation_id: String, + #[serde(deserialize_with = "deserialize_workspace_deletion_revision")] + expected_revision: String, + #[serde(deserialize_with = "deserialize_workspace_deletion_confirmation")] + confirmation: String, +} + +impl<'de> Deserialize<'de> for WorkspaceDeletionRequest { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = WorkspaceDeletionRequestWire::deserialize(deserializer)?; + Ok(Self { + operation_id: wire.operation_id, + expected_revision: wire.expected_revision, + confirmation: wire.confirmation, + }) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceDeletionOperationResponseWire { + #[serde(deserialize_with = "deserialize_workspace_deletion_operation_id")] + operation_id: String, + workspace_id: String, + display_name: String, + state: WorkspaceDeletionState, + resources: WorkspaceDeletionResourceCounts, + #[serde(deserialize_with = "deserialize_workspace_deletion_child_operation_ids")] + child_operation_ids: Vec, + #[serde(deserialize_with = "deserialize_workspace_deletion_blockers")] + blockers: Vec, + failure_category: Option, + created_at: String, + updated_at: String, + completed_at: Option, +} + +impl<'de> Deserialize<'de> for WorkspaceDeletionOperationResponse { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = WorkspaceDeletionOperationResponseWire::deserialize(deserializer)?; + Ok(Self { + operation_id: wire.operation_id, + workspace_id: wire.workspace_id, + display_name: wire.display_name, + state: wire.state, + resources: wire.resources, + child_operation_ids: wire.child_operation_ids, + blockers: wire.blockers, + failure_category: wire.failure_category, + created_at: wire.created_at, + updated_at: wire.updated_at, + completed_at: wire.completed_at, + }) + } +} + /// Read-only Profile catalog projected from one active Workspace config revision. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] @@ -3288,6 +3525,42 @@ mod tests { })) .is_err() ); + assert!( + serde_json::from_value::(serde_json::json!({ + "operation_id": "x".repeat(WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES + 1), + "expected_revision": "revision-7", + "confirmation": "Test" + })) + .is_err() + ); + assert!( + serde_json::from_value::(serde_json::json!({ + "operation_id": "delete-test", + "workspace_id": "workspace-test", + "display_name": "Test", + "state": "blocked", + "resources": { + "workers": 0, + "workdirs": 0, + "repositories": 0, + "runtime_bindings": 0, + "secrets": 0, + "artifacts": 0 + }, + "child_operation_ids": [], + "blockers": (0..=WORKSPACE_DELETION_MAX_BLOCKERS).map(|_| serde_json::json!({ + "kind": "cleanup_unavailable", + "resource_kind": null, + "resource_key": null, + "message": "blocked" + })).collect::>(), + "failure_category": null, + "created_at": "1", + "updated_at": "1", + "completed_at": null + })) + .is_err() + ); } #[test] diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 01be9ad8..74830e5c 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1008,6 +1008,20 @@ pub struct WorkspaceServerApi { catalog: WorkspaceCatalogService, routers: Arc>>, apis: Arc>>, + mutation_locks: Arc>>>>, + running_deletions: Arc>>, + hook_handles: Arc>>, +} + +async fn workspace_mutation_lock( + locks: &Arc>>>>, + workspace_id: &str, +) -> Arc> { + let mut locks = locks.lock().await; + locks + .entry(workspace_id.to_string()) + .or_insert_with(|| Arc::new(AsyncMutex::new(()))) + .clone() } impl WorkspaceServerApi { @@ -1018,9 +1032,16 @@ impl WorkspaceServerApi { store, routers: Arc::new(AsyncMutex::new(HashMap::new())), apis: Arc::new(AsyncMutex::new(HashMap::new())), + mutation_locks: Arc::new(AsyncMutex::new(HashMap::new())), + running_deletions: Arc::new(AsyncMutex::new(HashSet::new())), + hook_handles: Arc::new(AsyncMutex::new(HashMap::new())), } } + async fn mutation_lock(&self, workspace_id: &str) -> Arc> { + workspace_mutation_lock(&self.mutation_locks, workspace_id).await + } + async fn api_for_workspace(&self, workspace_id: &str) -> Result> { let mut apis = self.apis.lock().await; if let Some(api) = apis.get(workspace_id).cloned() { @@ -1038,6 +1059,47 @@ impl WorkspaceServerApi { Ok(Some(api)) } + async fn schedule_workspace_deletion(&self, operation_id: String) { + let mut running = self.running_deletions.lock().await; + if !running.insert(operation_id.clone()) { + return; + } + drop(running); + let api = self.clone(); + tokio::spawn(async move { + if api.execute_workspace_deletion(&operation_id).await.is_err() { + let operation = api + .store + .workspace_deletion_operation_for_recovery(&operation_id) + .ok() + .flatten(); + let child_operation_ids = operation + .as_ref() + .map(|operation| operation.child_operation_ids.as_slice()) + .unwrap_or_default(); + let blockers = operation + .as_ref() + .map(|operation| operation.blockers.as_slice()) + .unwrap_or_default(); + let _ = api.store.update_workspace_deletion_operation( + &operation_id, + WorkspaceDeletionState::Failed, + child_operation_ids, + blockers, + Some("workspace_deletion_execution_failed"), + ); + } + api.running_deletions.lock().await.remove(&operation_id); + }); + } + + async fn recover_workspace_deletions(&self) -> Result<()> { + for operation_id in self.store.resumable_workspace_deletion_operation_ids()? { + self.schedule_workspace_deletion(operation_id).await; + } + Ok(()) + } + async fn workspace_deletion_preflight( &self, actor_account_id: &str, @@ -1049,7 +1111,10 @@ impl WorkspaceServerApi { let Some(api) = self.api_for_workspace(workspace_id).await? else { return Err(Error::InvalidInput("Workspace does not exist".to_string())); }; - for registry_worker in self.store.list_worker_registry(workspace_id, 10_000)? { + for registry_worker in self + .store + .list_worker_registry(workspace_id, i64::MAX as usize)? + { let worker_key = registry_worker.display_name; match api.runtime.worker(®istry_worker.worker) { Ok(worker) if worker.state == "stopped" && worker.singleton_key.is_none() => {} @@ -1078,6 +1143,7 @@ impl WorkspaceServerApi { }), } } + crate::workspace_deletion::bound_workspace_deletion_blockers(&mut preflight.blockers); preflight.can_delete = preflight.blockers.is_empty(); Ok(preflight) } @@ -1102,7 +1168,7 @@ impl WorkspaceServerApi { let mut blockers = Vec::new(); for worker in self .store - .list_worker_registry(&operation.workspace_id, 10_000)? + .list_worker_registry(&operation.workspace_id, i64::MAX as usize)? { let worker_key = worker.display_name.clone(); let target = worker.worker; @@ -1130,7 +1196,7 @@ impl WorkspaceServerApi { if blockers.is_empty() { for workdir in self .store - .list_workdir_registry(&operation.workspace_id, 10_000)? + .list_workdir_registry(&operation.workspace_id, i64::MAX as usize)? { match execute_workdir_removal_for_workspace_deletion( &api, @@ -1183,7 +1249,19 @@ impl WorkspaceServerApi { } let completed = self.store.finalize_workspace_deletion(operation_id)?; self.routers.lock().await.remove(&completed.workspace_id); + if let Some(handle) = self + .hook_handles + .lock() + .await + .remove(&completed.workspace_id) + { + handle.abort(); + } self.apis.lock().await.remove(&completed.workspace_id); + self.mutation_locks + .lock() + .await + .remove(&completed.workspace_id); Ok(completed) } @@ -1194,12 +1272,22 @@ impl WorkspaceServerApi { let Some(api) = self.api_for_workspace(workspace_id).await? else { return Ok(None); }; - tokio::spawn(run_orchestrator_turn_end_hook(api.clone())); + let mut routers = self.routers.lock().await; + if let Some(router) = routers.get(workspace_id).cloned() { + return Ok(Some(router)); + } + let Some(workspace) = self.store.get_workspace(workspace_id).await? else { + return Ok(None); + }; + if workspace.state == "active" { + let hook = tokio::spawn(run_orchestrator_turn_end_hook(api.clone())); + self.hook_handles + .lock() + .await + .insert(workspace_id.to_string(), hook.abort_handle()); + } let router = build_inner_router(api); - self.routers - .lock() - .await - .insert(workspace_id.to_string(), router.clone()); + routers.insert(workspace_id.to_string(), router.clone()); Ok(Some(router)) } @@ -1324,6 +1412,8 @@ async fn start_server_workspace_deletion( Ok(None) => return forbidden_server_response("Workspace deletion requires its owner"), Err(error) => return server_error_response(error), }; + let mutation_lock = api.mutation_lock(&workspace_id).await; + let _mutation_guard = mutation_lock.lock().await; let existing = match api .store .workspace_deletion_operation(&actor_account_id, &request.operation_id) @@ -1351,24 +1441,13 @@ async fn start_server_workspace_deletion( Ok(reservation) => reservation, Err(error) => return server_error_response(error), }; - let operation = - if reservation.replay && reservation.operation.state == WorkspaceDeletionState::Succeeded { - reservation.operation - } else { - match api.execute_workspace_deletion(&request.operation_id).await { - Ok(operation) => operation, - Err(error) => { - let _ = api.store.update_workspace_deletion_operation( - &request.operation_id, - WorkspaceDeletionState::Failed, - &reservation.operation.child_operation_ids, - &[], - Some("workspace_deletion_execution_failed"), - ); - return server_error_response(error); - } - } - }; + if let Some(handle) = api.hook_handles.lock().await.remove(&workspace_id) { + handle.abort(); + } + let operation = reservation.operation; + if operation.state != WorkspaceDeletionState::Succeeded { + api.schedule_workspace_deletion(request.operation_id).await; + } let status = if operation.state == WorkspaceDeletionState::Succeeded { StatusCode::OK } else { @@ -1393,17 +1472,6 @@ async fn get_server_workspace_deletion( .store .workspace_deletion_operation(&actor_account_id, &operation_id) { - Ok(Some(operation)) - if matches!( - operation.state, - WorkspaceDeletionState::Queued | WorkspaceDeletionState::Running - ) => - { - match api.execute_workspace_deletion(&operation_id).await { - Ok(operation) => Json(operation).into_response(), - Err(error) => server_error_response(error), - } - } Ok(Some(operation)) => Json(operation).into_response(), Ok(None) => ( StatusCode::NOT_FOUND, @@ -1730,6 +1798,15 @@ async fn dispatch_workspace_request( ) -> Response { let path = request.uri().path().to_owned(); let workspace_id = scoped_workspace_id(&path); + let _mutation_guard = if let Some(workspace_id) = workspace_id + && !matches!( + *request.method(), + Method::GET | Method::HEAD | Method::OPTIONS + ) { + Some(api.mutation_lock(workspace_id).await.lock_owned().await) + } else { + None + }; if let Some(workspace_id) = workspace_id && (path.starts_with("/api/w/") || path.starts_with("/api/runtime/v1/workspaces/")) && let Err(response) = @@ -1835,6 +1912,7 @@ pub async fn build_workspace_server_router( }); let api = WorkspaceServerApi::new(template, store); api.preload().await?; + api.recover_workspace_deletions().await?; let catalog = Router::new() .route( "/api/workspaces", @@ -17007,6 +17085,40 @@ mod tests { SqliteWorkspaceStore, UserRecord, WorkspaceRecord, WorkspaceRuntimeBinding, }; + #[tokio::test] + async fn workspace_mutation_gate_serializes_deletion_with_active_mutations() { + let locks = Arc::new(AsyncMutex::new(HashMap::new())); + let active_mutation = workspace_mutation_lock(&locks, "workspace-a").await; + let deletion = workspace_mutation_lock(&locks, "workspace-a").await; + assert!(Arc::ptr_eq(&active_mutation, &deletion)); + + let active_guard = active_mutation.lock_owned().await; + let (acquired_tx, mut acquired_rx) = tokio::sync::oneshot::channel(); + let waiter = tokio::spawn(async move { + let _deletion_guard = deletion.lock_owned().await; + let _ = acquired_tx.send(()); + }); + tokio::task::yield_now().await; + assert!(matches!( + acquired_rx.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + drop(active_guard); + acquired_rx.await.expect("deletion acquires after mutation"); + waiter.await.expect("waiter joins"); + } + + #[test] + fn workspace_deletion_execution_is_server_owned_and_polling_is_read_only() { + let source = include_str!("server.rs"); + let start = handler_source(source, "start_server_workspace_deletion"); + assert!(start.contains("schedule_workspace_deletion")); + assert!(!start.contains("execute_workspace_deletion(&request")); + let poll = handler_source(source, "get_server_workspace_deletion"); + assert!(!poll.contains("execute_workspace_deletion")); + assert!(source.contains("api.recover_workspace_deletions().await?")); + } + fn handler_source<'a>(source: &'a str, name: &str) -> &'a str { let start = source .find(&format!("async fn {name}")) diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index f5777cba..506d6690 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -3,6 +3,9 @@ use rusqlite::{OptionalExtension, params}; use sha2::{Digest, Sha256}; use std::collections::BTreeSet; use workspace_api::{ + WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES, WORKSPACE_DELETION_MAX_BLOCKERS, + WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS, WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES, + WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES, WORKSPACE_DELETION_MAX_REVISION_BYTES, WorkspaceDeletionBlocker, WorkspaceDeletionBlockerKind, WorkspaceDeletionOperationResponse, WorkspaceDeletionPreflightResponse, WorkspaceDeletionRequest, WorkspaceDeletionResourceCounts, WorkspaceDeletionState, @@ -11,8 +14,6 @@ use workspace_api::{ use crate::store::{SqliteWorkspaceStore, WorkspaceRecord}; use crate::{Error, Result}; -const MAX_OPERATION_ID_BYTES: usize = 128; - #[derive(Debug, Clone)] pub struct WorkspaceDeletionReservation { pub operation: WorkspaceDeletionOperationResponse, @@ -46,6 +47,13 @@ pub trait WorkspaceDeletionStore: Send + Sync { worker_id: &str, ) -> Result>; + fn workspace_deletion_operation_for_recovery( + &self, + operation_id: &str, + ) -> Result>; + + fn resumable_workspace_deletion_operation_ids(&self) -> Result>; + fn update_workspace_deletion_operation( &self, operation_id: &str, @@ -84,6 +92,19 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { message: "You cannot delete your last accessible Workspace.".to_string(), }); } + if resources.workers.saturating_add(resources.workdirs) + > WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS as u64 + { + blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::CleanupUnavailable, + resource_kind: None, + resource_key: None, + message: + "Workspace cleanup exceeds the supported durable child-operation bound." + .to_string(), + }); + } + bound_workspace_deletion_blockers(&mut blockers); Ok(WorkspaceDeletionPreflightResponse { workspace_id: workspace.workspace_id, display_name: workspace.display_name, @@ -102,6 +123,13 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { request: &WorkspaceDeletionRequest, ) -> Result { validate_operation_id(&request.operation_id)?; + if request.expected_revision.len() > WORKSPACE_DELETION_MAX_REVISION_BYTES + || request.confirmation.len() > workspace_api::WORKSPACE_DELETION_MAX_CONFIRMATION_BYTES + { + return Err(Error::InvalidInput( + "Workspace deletion request exceeds bounded field limits".to_string(), + )); + } self.with_transaction(|tx| { if let Some(existing) = read_operation(tx, &request.operation_id)? { if existing.actor_account_id != actor_account_id { @@ -228,6 +256,26 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { }) } + fn workspace_deletion_operation_for_recovery( + &self, + operation_id: &str, + ) -> Result> { + self.with_conn(|conn| Ok(read_operation(conn, operation_id)?.map(|stored| stored.response))) + } + + fn resumable_workspace_deletion_operation_ids(&self) -> Result> { + self.with_conn(|conn| { + let mut statement = conn.prepare( + "SELECT operation_id FROM workspace_deletion_operations + WHERE state IN ('queued', 'running') ORDER BY created_at, operation_id", + )?; + statement + .query_map([], |row| row.get(0))? + .collect::, _>>() + .map_err(Into::into) + }) + } + fn update_workspace_deletion_operation( &self, operation_id: &str, @@ -236,6 +284,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { blockers: &[WorkspaceDeletionBlocker], failure_category: Option<&str>, ) -> Result { + validate_operation_projection(child_operation_ids, blockers)?; self.with_transaction(|tx| { let now = Utc::now().to_rfc3339(); let completed_at = @@ -458,6 +507,45 @@ fn owner_workspace( Ok(workspace) } +pub(crate) fn bound_workspace_deletion_blockers(blockers: &mut Vec) { + for blocker in blockers.iter_mut() { + blocker.resource_kind = blocker + .resource_kind + .take() + .map(|value| truncate_utf8(value, WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES)); + blocker.resource_key = blocker + .resource_key + .take() + .map(|value| truncate_utf8(value, WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES)); + blocker.message = truncate_utf8( + std::mem::take(&mut blocker.message), + WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES, + ); + } + if blockers.len() > WORKSPACE_DELETION_MAX_BLOCKERS { + blockers.truncate(WORKSPACE_DELETION_MAX_BLOCKERS - 1); + blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::CleanupUnavailable, + resource_kind: None, + resource_key: None, + message: "Additional deletion blockers exist; reduce Workspace resources and run preflight again." + .to_string(), + }); + } +} + +fn truncate_utf8(mut value: String, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value; + } + let mut end = max_bytes; + while !value.is_char_boundary(end) { + end -= 1; + } + value.truncate(end); + value +} + fn workspace_database_blockers( conn: &rusqlite::Connection, workspace_id: &str, @@ -573,9 +661,36 @@ fn table_count(conn: &rusqlite::Connection, table: &str, workspace_id: &str) -> .map_err(Into::into) } +fn validate_operation_projection( + child_operation_ids: &[String], + blockers: &[WorkspaceDeletionBlocker], +) -> Result<()> { + let invalid_child_ids = child_operation_ids.len() > WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS + || child_operation_ids + .iter() + .any(|value| value.len() > WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES); + let invalid_blockers = + blockers.len() > WORKSPACE_DELETION_MAX_BLOCKERS + || blockers.iter().any(|blocker| { + blocker.message.len() > WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES + || blocker.resource_kind.as_ref().is_some_and(|value| { + value.len() > WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES + }) + || blocker.resource_key.as_ref().is_some_and(|value| { + value.len() > WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES + }) + }); + if invalid_child_ids || invalid_blockers { + return Err(Error::Store( + "Workspace deletion operation projection exceeds bounded limits".to_string(), + )); + } + Ok(()) +} + fn validate_operation_id(operation_id: &str) -> Result<()> { if operation_id.is_empty() - || operation_id.len() > MAX_OPERATION_ID_BYTES + || operation_id.len() > WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES || !operation_id .bytes() .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) @@ -672,10 +787,26 @@ mod tests { .reserve_workspace_deletion(&owner, &workspace_id, &request) .expect("reserve"); assert!(!first.replay); + assert_eq!( + store + .resumable_workspace_deletion_operation_ids() + .expect("resumable operations"), + vec![request.operation_id.clone()] + ); let replay = store .reserve_workspace_deletion(&owner, &workspace_id, &request) .expect("replay"); assert!(replay.replay); + assert!(matches!( + store.update_workspace_deletion_operation( + &request.operation_id, + WorkspaceDeletionState::Running, + &vec!["child".to_string(); WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS + 1], + &[], + None, + ), + Err(Error::Store(_)) + )); let completed = store .finalize_workspace_deletion(&request.operation_id) .expect("finalize"); @@ -684,6 +815,12 @@ mod tests { .finalize_workspace_deletion(&request.operation_id) .expect("finalize replay"); assert_eq!(completed, replayed); + assert!( + store + .resumable_workspace_deletion_operation_ids() + .expect("terminal operations") + .is_empty() + ); let workspace_count: u64 = store .with_conn(|conn| { conn.query_row( @@ -767,6 +904,39 @@ mod tests { )); } + #[test] + fn preflight_counts_complete_inventory_and_bounds_blocker_projection() { + let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "WITH RECURSIVE seq(value) AS ( + SELECT 1 UNION ALL SELECT value + 1 FROM seq WHERE value < 10001 + ) + INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) + SELECT ?1, 'runtime-a', 'worker-' || value, 'Pinned ' || value, + '1', '1', 'pinned' + FROM seq", + params![workspace_id], + )?; + Ok(()) + }) + .expect("worker inventory"); + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + assert_eq!(preflight.resources.workers, 10_001); + assert_eq!(preflight.blockers.len(), WORKSPACE_DELETION_MAX_BLOCKERS); + assert!(preflight.blockers.iter().any(|blocker| { + blocker + .message + .contains("Additional deletion blockers exist") + })); + } + #[test] fn last_accessible_workspace_and_revision_conflicts_fail_closed() { let (store, owner, workspace_id) = setup(); diff --git a/web/workspace/src/lib/workspace/api/workspace-model.ts b/web/workspace/src/lib/workspace/api/workspace-model.ts index d32305b9..2a62b3bb 100644 --- a/web/workspace/src/lib/workspace/api/workspace-model.ts +++ b/web/workspace/src/lib/workspace/api/workspace-model.ts @@ -583,6 +583,37 @@ export function parseRepositoryDetailResponse( }; } +const WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES = 128; +const WORKSPACE_DELETION_MAX_REVISION_BYTES = 128; +const WORKSPACE_DELETION_MAX_BLOCKERS = 1024; +const WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS = 4096; +const WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES = 128; +const WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES = 512; + +function deletionBoundedString( + value: unknown, + path: string, + maxBytes: number, +): string { + const candidate = string(value, path); + if (new TextEncoder().encode(candidate).length > maxBytes) { + throw new Error(`${path} is too long`); + } + return candidate; +} + +function deletionBoundedArray( + value: unknown, + path: string, + maxItems: number, +): unknown[] { + const candidate = array(value, path); + if (candidate.length > maxItems) { + throw new Error(`${path} has too many items`); + } + return candidate; +} + const deletionStates = new Set([ "queued", "running", @@ -619,14 +650,35 @@ function deletionBlocker( if (!deletionBlockerKinds.has(kind)) { throw new Error(`${path}.kind is invalid`); } + const resourceKind = optionalNullableString( + item.resource_kind, + `${path}.resource_kind`, + ); + const resourceKey = optionalNullableString( + item.resource_key, + `${path}.resource_key`, + ); return { kind, - resource_kind: - optionalNullableString(item.resource_kind, `${path}.resource_kind`) ?? - null, - resource_key: - optionalNullableString(item.resource_key, `${path}.resource_key`) ?? null, - message: string(item.message, `${path}.message`), + resource_kind: resourceKind === undefined || resourceKind === null + ? null + : deletionBoundedString( + resourceKind, + `${path}.resource_kind`, + WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES, + ), + resource_key: resourceKey === undefined || resourceKey === null + ? null + : deletionBoundedString( + resourceKey, + `${path}.resource_key`, + WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES, + ), + message: deletionBoundedString( + item.message, + `${path}.message`, + WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES, + ), }; } @@ -677,9 +729,10 @@ export function parseWorkspaceDeletionPreflightResponse( item.display_name, "Workspace deletion preflight.display_name", ), - expected_revision: string( + expected_revision: deletionBoundedString( item.expected_revision, "Workspace deletion preflight.expected_revision", + WORKSPACE_DELETION_MAX_REVISION_BYTES, ), can_delete: boolean( item.can_delete, @@ -689,7 +742,11 @@ export function parseWorkspaceDeletionPreflightResponse( item.resources, "Workspace deletion preflight.resources", ), - blockers: array(item.blockers, "Workspace deletion preflight.blockers").map( + blockers: deletionBoundedArray( + item.blockers, + "Workspace deletion preflight.blockers", + WORKSPACE_DELETION_MAX_BLOCKERS, + ).map( (entry, index) => deletionBlocker( entry, @@ -717,9 +774,10 @@ export function parseWorkspaceDeletionOperationResponse( "completed_at", ], "Workspace deletion operation"); return { - operation_id: string( + operation_id: deletionBoundedString( item.operation_id, "Workspace deletion operation.operation_id", + WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES, ), workspace_id: string( item.workspace_id, @@ -734,16 +792,22 @@ export function parseWorkspaceDeletionOperationResponse( item.resources, "Workspace deletion operation.resources", ), - child_operation_ids: array( + child_operation_ids: deletionBoundedArray( item.child_operation_ids, "Workspace deletion operation.child_operation_ids", + WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS, ).map((entry, index) => - string( + deletionBoundedString( entry, `Workspace deletion operation.child_operation_ids[${index}]`, + WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES, ) ), - blockers: array(item.blockers, "Workspace deletion operation.blockers").map( + blockers: deletionBoundedArray( + item.blockers, + "Workspace deletion operation.blockers", + WORKSPACE_DELETION_MAX_BLOCKERS, + ).map( (entry, index) => deletionBlocker( entry, diff --git a/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts b/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts index 8106f1f7..449bff2e 100644 --- a/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts +++ b/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts @@ -3,34 +3,44 @@ import type { WorkspaceDeletionPreflightResponse, WorkspaceDeletionRequest, } from "$lib/generated/workspace-api"; +import { loadJson } from "$lib/workspace/api/http"; import { parseWorkspaceDeletionOperationResponse, parseWorkspaceDeletionPreflightResponse, } from "$lib/workspace/api/workspace-model"; -async function responseJson( - response: Response, - context: string, -): Promise { - const value: unknown = await response.json().catch(() => null); - if (!response.ok) { - const message = typeof value === "object" && value !== null && - "error" in value && typeof value.error === "string" - ? value.error - : `${context} failed (${response.status})`; - throw new Error(message); +const deletionResponsePolicy = { + maxResponseBytes: 2 * 1024 * 1024, + diagnosticLabel: "Workspace deletion", +} as const; + +async function deletionJson( + path: string, + init: RequestInit | undefined, + parse: (value: unknown) => T, +): Promise { + const result = await loadJson( + fetch, + path, + init, + parse, + deletionResponsePolicy, + ); + if (result.error !== null || result.data === null) { + throw new Error( + result.error ?? "Workspace deletion response is unavailable", + ); } - return value; + return result.data; } export async function preflightWorkspaceDeletion( workspaceId: string, ): Promise { - const response = await fetch( + return await deletionJson( `/api/workspaces/${encodeURIComponent(workspaceId)}/deletion`, - ); - return parseWorkspaceDeletionPreflightResponse( - await responseJson(response, "Workspace deletion preflight"), + undefined, + parseWorkspaceDeletionPreflightResponse, ); } @@ -38,26 +48,23 @@ export async function startWorkspaceDeletion( workspaceId: string, request: WorkspaceDeletionRequest, ): Promise { - const response = await fetch( + return await deletionJson( `/api/workspaces/${encodeURIComponent(workspaceId)}/deletion`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(request), }, - ); - return parseWorkspaceDeletionOperationResponse( - await responseJson(response, "Workspace deletion"), + parseWorkspaceDeletionOperationResponse, ); } export async function getWorkspaceDeletion( operationId: string, ): Promise { - const response = await fetch( + return await deletionJson( `/api/workspace-deletions/${encodeURIComponent(operationId)}`, - ); - return parseWorkspaceDeletionOperationResponse( - await responseJson(response, "Workspace deletion status"), + undefined, + parseWorkspaceDeletionOperationResponse, ); } diff --git a/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts b/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts index a1eed656..916c4693 100644 --- a/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts +++ b/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts @@ -22,6 +22,10 @@ export type WorkspaceWorkersState = { const stores = new Map>(); +export function disposeWorkspaceWorkersStore(workspaceId: string): void { + stores.delete(workspaceId); +} + export function workspaceWorkersStore(workspaceId: string): Readable { const cached = stores.get(workspaceId); if (cached) return cached; diff --git a/web/workspace/src/routes/w/[workspaceId]/+layout.svelte b/web/workspace/src/routes/w/[workspaceId]/+layout.svelte index 6adaeb73..f5e8587f 100644 --- a/web/workspace/src/routes/w/[workspaceId]/+layout.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/+layout.svelte @@ -12,6 +12,7 @@ } from '$lib/workspace/sidebar/context'; import { createOverrideStack } from '$lib/workspace/sidebar/override-stack'; import { disposeWorkspaceMultiplexer } from '$lib/workspace/multiplexer'; + import { disposeWorkspaceWorkersStore } from '$lib/workspace/sidebar/worker-subscription'; import WorkspaceSidebar from '$lib/workspace/sidebar/WorkspaceSidebar.svelte'; import '$lib/workspace/styles/workspace-pages.css'; import '$lib/workspace/styles/tickets.css'; @@ -32,7 +33,10 @@ $effect(() => { const workspaceId = data.workspace?.workspace_id; if (!workspaceId) return; - return () => disposeWorkspaceMultiplexer(workspaceId); + return () => { + disposeWorkspaceMultiplexer(workspaceId); + disposeWorkspaceWorkersStore(workspaceId); + }; }); diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte index 5b5df2cb..26ec7005 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte @@ -3,10 +3,13 @@ Diagnostic, WorkspaceDeletionOperationResponse, WorkspaceDeletionPreflightResponse, + WorkspaceDeletionRequest, WorkspaceMetadataSettingsResponse, } from '$lib/generated/workspace-api'; import { goto } from '$app/navigation'; + import { onMount } from 'svelte'; import { disposeWorkspaceMultiplexer } from '$lib/workspace/multiplexer'; + import { disposeWorkspaceWorkersStore } from '$lib/workspace/sidebar/worker-subscription'; import { getWorkspaceDeletion, preflightWorkspaceDeletion, @@ -34,8 +37,11 @@ let deletionConfirmation = $state(''); let deletionPreflight = $state(null); let deletionOperation = $state(null); - let deletionOperationId = $state(''); + let deletionRequest = $state(null); let deletionError = $state(null); + function deletionStorageKey(): string { + return `yoi:workspace-deletion:${workspaceId}`; + } $effect(() => { if (!workspaceId) { @@ -92,7 +98,8 @@ deletionLoading = true; deletionError = null; deletionOperation = null; - deletionOperationId = crypto.randomUUID(); + deletionRequest = null; + sessionStorage.removeItem(deletionStorageKey()); deletionConfirmation = ''; try { deletionPreflight = await preflightWorkspaceDeletion(workspaceId); @@ -103,26 +110,76 @@ } } + async function trackDeletion(operationId: string) { + let operation = await getWorkspaceDeletion(operationId); + deletionOperation = operation; + while (operation.state === 'queued' || operation.state === 'running') { + await new Promise((resolve) => setTimeout(resolve, 500)); + operation = await getWorkspaceDeletion(operation.operation_id); + deletionOperation = operation; + } + if (operation.state === 'succeeded') { + sessionStorage.removeItem(deletionStorageKey()); + disposeWorkspaceMultiplexer(workspaceId); + disposeWorkspaceWorkersStore(workspaceId); + await goto('/'); + } + } + + function storedDeletionRequest(): WorkspaceDeletionRequest | null { + try { + const value: unknown = JSON.parse(sessionStorage.getItem(deletionStorageKey()) ?? 'null'); + if (typeof value !== 'object' || value === null) return null; + const record = value as Record; + if ( + Object.keys(record).sort().join(',') !== 'confirmation,expected_revision,operation_id' || + typeof record.operation_id !== 'string' || record.operation_id.length === 0 || record.operation_id.length > 128 || + !/^[A-Za-z0-9_-]+$/.test(record.operation_id) || + typeof record.expected_revision !== 'string' || record.expected_revision.length > 128 || + typeof record.confirmation !== 'string' || record.confirmation !== data.workspace?.display_name || record.confirmation.length > 256 + ) return null; + return { + operation_id: record.operation_id, + expected_revision: record.expected_revision, + confirmation: record.confirmation, + }; + } catch { + return null; + } + } + + onMount(() => { + if (!data.workspace?.permissions.delete_workspace) return; + const request = storedDeletionRequest(); + if (!request) return; + deletionRequest = request; + deletionConfirmation = request.confirmation; + deletionOpen = true; + deletionSubmitting = true; + void trackDeletion(request.operation_id) + .catch((err) => { + deletionError = err instanceof Error ? err.message : 'Workspace deletion status failed'; + }) + .finally(() => { + deletionSubmitting = false; + }); + }); + async function deleteWorkspace() { - if (!deletionPreflight) return; + if (!deletionPreflight && !deletionRequest) return; deletionSubmitting = true; deletionError = null; try { - let operation = await startWorkspaceDeletion(workspaceId, { - operation_id: deletionOperationId, - expected_revision: deletionPreflight.expected_revision, + const request = deletionRequest ?? { + operation_id: crypto.randomUUID(), + expected_revision: deletionPreflight!.expected_revision, confirmation: deletionConfirmation, - }); + }; + deletionRequest = request; + sessionStorage.setItem(deletionStorageKey(), JSON.stringify(request)); + const operation = await startWorkspaceDeletion(workspaceId, request); deletionOperation = operation; - while (operation.state === 'queued' || operation.state === 'running') { - await new Promise((resolve) => setTimeout(resolve, 500)); - operation = await getWorkspaceDeletion(operation.operation_id); - deletionOperation = operation; - } - if (operation.state === 'succeeded') { - disposeWorkspaceMultiplexer(workspaceId); - await goto('/'); - } + await trackDeletion(operation.operation_id); } catch (err) { deletionError = err instanceof Error ? err.message : 'Workspace deletion failed'; } finally { @@ -187,7 +244,7 @@ {#if deletionOpen} diff --git a/web/workspace/tests/workspace-model.test.ts b/web/workspace/tests/workspace-model.test.ts index eba48f0d..223d4b17 100644 --- a/web/workspace/tests/workspace-model.test.ts +++ b/web/workspace/tests/workspace-model.test.ts @@ -169,6 +169,22 @@ Deno.test("Workspace deletion DTOs fail closed and preserve durable operation st }), ".state is invalid", ); + assertThrows( + () => + parseWorkspaceDeletionOperationResponse({ + ...operation, + operation_id: "x".repeat(129), + }), + ".operation_id is too long", + ); + assertThrows( + () => + parseWorkspaceDeletionOperationResponse({ + ...operation, + blockers: Array.from({ length: 1025 }, () => operation.blockers[0]), + }), + ".blockers has too many items", + ); }); Deno.test("Workspace settings exposes owner-gated typed destructive confirmation", async () => { @@ -185,6 +201,10 @@ Deno.test("Workspace settings exposes owner-gated typed destructive confirmation "startWorkspaceDeletion", "deletionConfirmation", "disposeWorkspaceMultiplexer(workspaceId)", + "disposeWorkspaceWorkersStore(workspaceId)", + "sessionStorage.setItem(deletionStorageKey", + "storedDeletionRequest()", + "trackDeletion(request.operation_id)", ] ) { if (!source.includes(token)) { From 4631b9514400e090e66d2d828a0e3d5ecf4e3283 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 09:21:37 +0900 Subject: [PATCH 33/39] fix: fence workspace deletion side effects --- crates/workspace-server/src/server.rs | 21 +- .../src/workspace_deletion.rs | 293 ++++++++++++++++-- 2 files changed, 282 insertions(+), 32 deletions(-) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 74830e5c..85600c08 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1152,10 +1152,14 @@ impl WorkspaceServerApi { &self, operation_id: &str, ) -> Result { + let prior = self + .store + .workspace_deletion_operation_for_recovery(operation_id)? + .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; let operation = self.store.update_workspace_deletion_operation( operation_id, WorkspaceDeletionState::Running, - &[], + &prior.child_operation_ids, &[], None, )?; @@ -1164,7 +1168,7 @@ impl WorkspaceServerApi { .await? .ok_or_else(|| Error::InvalidInput("Workspace no longer exists".to_string()))?; - let mut child_operation_ids = Vec::new(); + let mut child_operation_ids = operation.child_operation_ids.clone(); let mut blockers = Vec::new(); for worker in self .store @@ -1181,7 +1185,10 @@ impl WorkspaceServerApi { &target.runtime_id, &target.worker_id, )? { - child_operation_ids.push(child_operation_id); + child_operation_ids = self + .store + .append_workspace_deletion_child_operation(operation_id, &child_operation_id)? + .child_operation_ids; } if response.status != 200 { blockers.push(WorkspaceDeletionBlocker { @@ -1204,7 +1211,13 @@ impl WorkspaceServerApi { operation_id, ) { Ok(child) => { - child_operation_ids.push(child.operation_id.clone()); + child_operation_ids = self + .store + .append_workspace_deletion_child_operation( + operation_id, + &child.operation_id, + )? + .child_operation_ids; if child.state != WorkdirRemovalOperationState::Completed || child.disposition != Some(WorkdirRemovalDisposition::Removed) { diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index 506d6690..d8e52808 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -1,7 +1,6 @@ use chrono::Utc; use rusqlite::{OptionalExtension, params}; use sha2::{Digest, Sha256}; -use std::collections::BTreeSet; use workspace_api::{ WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES, WORKSPACE_DELETION_MAX_BLOCKERS, WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS, WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES, @@ -14,6 +13,77 @@ use workspace_api::{ use crate::store::{SqliteWorkspaceStore, WorkspaceRecord}; use crate::{Error, Result}; +/// Explicit domain-owned purge inventory. The deletion operation tombstone is intentionally +/// excluded so retries and audit remain available after the Workspace row is gone. +const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ + "artifacts", + "audit_events", + "flow_source_revisions", + "flow_sources", + "memory_staging_records", + "memory_staging_resolutions", + "merge_request_review_grants", + "merge_request_reviewer_child_sessions", + "merge_request_thread_events", + "merge_request_ticket_relations", + "merge_requests", + "objective_events", + "objective_resources", + "objective_ticket_links", + "objectives", + "repositories", + "repository_secret_audit_events", + "repository_secret_operations", + "repository_ssh_credential_revisions", + "repository_ssh_credentials", + "repository_ssh_host_trust_revisions", + "repository_ssh_host_trusts", + "server_secret_versions", + "ticket_assignment_operations", + "ticket_assignment_ticket_tombstones", + "ticket_assignment_worker_tombstones", + "ticket_current_worker_assignments", + "ticket_worker_assignment_events", + "ticket_worker_assignments", + "typed_ticket_artifacts", + "typed_ticket_event_attributes", + "typed_ticket_event_references", + "typed_ticket_events", + "typed_ticket_labels", + "typed_ticket_orchestration_plans", + "typed_ticket_raw_frontmatter", + "typed_ticket_relations", + "typed_ticket_risk_flags", + "typed_tickets", + "workdir_create_operations", + "workdir_registry", + "workdir_removal_operations", + "worker_control_grants", + "worker_create_reservations", + "worker_diagnostics_archives", + "worker_mutation_source_proof_jtis", + "worker_orphan_diagnostics", + "worker_registry", + "worker_removal_operations", + "worker_retention_audit_events", + "worker_session_archives", + "worker_tombstones", + "worker_workdir_attachment_reservations", + "worker_workdir_links", + "workspace_config_entries", + "workspace_config_tree_revisions", + "workspace_config_trees", + "workspace_create_operations", + "workspace_memory_documents", + "workspace_memory_settings", + "workspace_resource_key_counters", + "workspace_resource_keys", + "workspace_runtime_binding_audit", + "workspace_runtime_bindings", + "workspace_worker_retention_policies", + "workspace_worker_retention_policy_revisions", +]; + #[derive(Debug, Clone)] pub struct WorkspaceDeletionReservation { pub operation: WorkspaceDeletionOperationResponse, @@ -54,6 +124,12 @@ pub trait WorkspaceDeletionStore: Send + Sync { fn resumable_workspace_deletion_operation_ids(&self) -> Result>; + fn append_workspace_deletion_child_operation( + &self, + operation_id: &str, + child_operation_id: &str, + ) -> Result; + fn update_workspace_deletion_operation( &self, operation_id: &str, @@ -276,6 +352,45 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { }) } + fn append_workspace_deletion_child_operation( + &self, + operation_id: &str, + child_operation_id: &str, + ) -> Result { + validate_operation_id(child_operation_id)?; + self.with_transaction(|tx| { + let operation = read_operation(tx, operation_id)? + .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? + .response; + if !operation + .child_operation_ids + .iter() + .any(|existing| existing == child_operation_id) + { + let mut child_operation_ids = operation.child_operation_ids; + child_operation_ids.push(child_operation_id.to_string()); + validate_operation_projection(&child_operation_ids, &operation.blockers)?; + let now = Utc::now().to_rfc3339(); + tx.execute( + "UPDATE workspace_deletion_operations + SET child_operation_ids_json = ?2, updated_at = ?3 + WHERE operation_id = ?1", + params![ + operation_id, + serde_json::to_string(&child_operation_ids) + .map_err(|error| Error::Store(error.to_string()))?, + now, + ], + )?; + } + Ok(read_operation(tx, operation_id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + })? + .response) + }) + } + fn update_workspace_deletion_operation( &self, operation_id: &str, @@ -332,31 +447,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { } let workspace_id = operation.response.workspace_id.clone(); - let mut scoped_tables = Vec::new(); - let mut statement = tx.prepare( - "SELECT m.name - FROM sqlite_master m - WHERE m.type = 'table' AND m.name NOT LIKE 'sqlite_%' - ORDER BY m.name", - )?; - let names = statement - .query_map([], |row| row.get::<_, String>(0))? - .collect::, _>>()?; - drop(statement); - for table in names { - if table == "workspaces" || table == "workspace_deletion_operations" { - continue; - } - let escaped = table.replace('"', "\"\""); - let mut info = tx.prepare(&format!("PRAGMA table_info(\"{escaped}\")"))?; - let columns = info - .query_map([], |row| row.get::<_, String>(1))? - .collect::, _>>()?; - if columns.contains("workspace_id") { - scoped_tables.push(escaped); - } - } - for table in scoped_tables { + for table in WORKSPACE_DELETION_PURGE_TABLES { tx.execute( &format!("DELETE FROM \"{table}\" WHERE workspace_id = ?1"), params![workspace_id], @@ -594,6 +685,12 @@ fn workspace_database_blockers( "ticket", "Remove current Ticket assignments before deleting the Workspace.", ), + ( + "SELECT COUNT(*) FROM worker_create_reservations WHERE workspace_id = ?1 AND state = 'reserved'", + WorkspaceDeletionBlockerKind::CleanupUnavailable, + "worker", + "Wait for or cancel pending Worker creation reservations.", + ), ( "SELECT COUNT(*) FROM worker_removal_operations WHERE workspace_id = ?1 AND state IN ('planned', 'blocked', 'executing', 'failed', 'stale')", WorkspaceDeletionBlockerKind::CleanupUnavailable, @@ -706,8 +803,8 @@ fn request_fingerprint( request: &WorkspaceDeletionRequest, ) -> String { let canonical = format!( - "workspace-delete-v1\0{actor_account_id}\0{workspace_id}\0{}\0{}", - request.expected_revision, request.confirmation + "workspace-delete-v1\0{actor_account_id}\0{workspace_id}\0{}\0{}\0{}", + request.operation_id, request.expected_revision, request.confirmation ); encode_hex(&Sha256::digest(canonical.as_bytes())) } @@ -748,6 +845,7 @@ fn parse_deletion_state(value: &str) -> Result { #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeSet; use tempfile::tempdir; fn setup() -> (SqliteWorkspaceStore, String, String) { @@ -772,9 +870,56 @@ mod tests { (store, "owner".to_string(), "workspace-a".to_string()) } + #[test] + fn explicit_purge_inventory_covers_every_workspace_scoped_table() { + let (store, _, _) = setup(); + store + .with_conn(|conn| { + let mut statement = conn.prepare( + "SELECT name FROM sqlite_master + WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + )?; + let tables = statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + drop(statement); + let mut scoped = BTreeSet::new(); + for table in tables { + let escaped = table.replace('"', "\"\""); + let mut info = conn.prepare(&format!("PRAGMA table_info(\"{escaped}\")"))?; + let columns = info + .query_map([], |row| row.get::<_, String>(1))? + .collect::, _>>()?; + if columns.iter().any(|column| column == "workspace_id") { + scoped.insert(table); + } + } + let expected = WORKSPACE_DELETION_PURGE_TABLES + .iter() + .copied() + .chain(["workspace_deletion_operations", "workspaces"]) + .map(str::to_string) + .collect::>(); + assert_eq!(scoped, expected); + Ok(()) + }) + .expect("purge inventory"); + } + #[test] fn deletion_is_idempotent_and_removes_workspace_scoped_rows() { let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_mutation_source_proof_jtis ( + workspace_id, runtime_id, jti, expires_at, consumed_at + ) VALUES (?1, 'runtime-a', 'jti-a', 1, '1')", + params![workspace_id], + )?; + Ok(()) + }) + .expect("non-FK scoped audit fixture"); let preflight = store .workspace_deletion_preflight(&owner, &workspace_id) .expect("preflight"); @@ -807,6 +952,20 @@ mod tests { ), Err(Error::Store(_)) )); + let with_child = store + .append_workspace_deletion_child_operation(&request.operation_id, "child-operation-1") + .expect("append child operation"); + assert_eq!( + with_child.child_operation_ids, + vec!["child-operation-1".to_string()] + ); + let duplicate = store + .append_workspace_deletion_child_operation(&request.operation_id, "child-operation-1") + .expect("append child operation replay"); + assert_eq!( + duplicate.child_operation_ids, + with_child.child_operation_ids + ); let completed = store .finalize_workspace_deletion(&request.operation_id) .expect("finalize"); @@ -832,6 +991,17 @@ mod tests { }) .expect("read"); assert_eq!(workspace_count, 0); + let proof_count: u64 = store + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM worker_mutation_source_proof_jtis WHERE workspace_id = ?1", + params![workspace_id], + |row| row.get(0), + ) + .map_err(Into::into) + }) + .expect("scoped audit read"); + assert_eq!(proof_count, 0); } #[test] @@ -862,6 +1032,12 @@ mod tests { store.reserve_workspace_deletion(&owner, &workspace_id, &request), Err(Error::WorkspaceConfigConflict(_)) )); + let mut other_operation = request.clone(); + other_operation.operation_id = "delete-alpha-other-operation".to_string(); + assert_ne!( + request_fingerprint(&owner, &workspace_id, &request), + request_fingerprint(&owner, &workspace_id, &other_operation) + ); request.expected_revision = preflight.expected_revision; request.confirmation = "delete Alpha".to_string(); assert!(matches!( @@ -870,6 +1046,67 @@ mod tests { )); } + #[test] + fn pending_worker_and_workdir_creation_block_reservation_without_orphans() { + let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_create_reservations ( + workspace_id, allocation_key, worker_id, runtime_id, + create_fingerprint, state, created_at, updated_at + ) VALUES (?1, 'allocation', 'worker-pending', 'runtime-a', + 'fingerprint', 'reserved', '1', '1')", + params![workspace_id], + )?; + conn.execute( + "INSERT INTO workdir_create_operations ( + workspace_id, operation_id, request_fingerprint, repository_id, + selector, requested_runtime_id, resolved_runtime_id, config_revision, + config_projection_digest, working_directory_id, state, created_at, updated_at + ) VALUES ( + ?1, 'workdir-create', 'fingerprint', 'repository-pending', + 'develop', 'runtime-a', 'runtime-a', 1, + 'projection', 'workdir-pending', 'pending', '1', '1' + )", + params![workspace_id], + )?; + Ok(()) + }) + .expect("pending creation fixtures"); + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + assert!(!preflight.can_delete); + assert!( + preflight + .blockers + .iter() + .any(|blocker| blocker.message.contains("Worker creation reservations")) + ); + assert!( + preflight + .blockers + .iter() + .any(|blocker| blocker.message.contains("Workdir creation operations")) + ); + let request = WorkspaceDeletionRequest { + operation_id: "delete-with-pending-creates".to_string(), + expected_revision: preflight.expected_revision, + confirmation: "Alpha".to_string(), + }; + assert!(matches!( + store.reserve_workspace_deletion(&owner, &workspace_id, &request), + Err(Error::WorkspaceConfigConflict(_)) + )); + assert!( + store + .workspace_deletion_operation_for_recovery(&request.operation_id) + .expect("operation lookup") + .is_none() + ); + } + #[test] fn pinned_worker_blocks_preflight_before_operation_reservation() { let (store, owner, workspace_id) = setup(); From 7b25b767f8f694b521e197a53f0b81d1c29c55e0 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 09:35:44 +0900 Subject: [PATCH 34/39] fix: checkpoint workspace cleanup children --- crates/workspace-server/src/server.rs | 75 ++++++++++++++++--- .../src/workspace_deletion.rs | 30 +------- 2 files changed, 67 insertions(+), 38 deletions(-) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 85600c08..00f1a757 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -164,6 +164,7 @@ use crate::workdir_removal::{ WorkdirRemovalOperationState, workdir_removal_intent, }; use crate::workspace_catalog::{WorkspaceCatalogService, WorkspaceCreateRequest}; +use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; use worker_runtime::catalog::{ ConfigBundleRef, ProfileSelector, RepositoryMaterializationContext, RepositoryRefObservation, @@ -746,7 +747,8 @@ impl WorkspaceWorkerRemoveExecutor { )); } - self.execute_target_removal(&runtime, &target, reason).await + self.execute_target_removal(&runtime, &target, reason, None) + .await } async fn execute_target_removal( @@ -754,6 +756,7 @@ impl WorkspaceWorkerRemoveExecutor { runtime: &RuntimeRegistry, target: &RuntimeWorkerRef, reason: &str, + parent_workspace_deletion_operation_id: Option<&str>, ) -> std::result::Result { let remove_lock = { let mut locks = self @@ -802,6 +805,14 @@ impl WorkspaceWorkerRemoveExecutor { } else { prepared }; + if let Some(parent_operation_id) = parent_workspace_deletion_operation_id { + self.store + .append_workspace_deletion_child_operation( + parent_operation_id, + &prepared.plan.operation_id, + ) + .map_err(|error| error.to_string())?; + } if close_worker_workdir_sessions(&self.workdir_sessions, &target) .await .is_err() @@ -898,6 +909,14 @@ impl WorkspaceWorkerRemoveExecutor { Err(error) => return Ok(worker_retention_error_response(error)), }; + if let Some(parent_operation_id) = parent_workspace_deletion_operation_id { + self.store + .append_workspace_deletion_child_operation( + parent_operation_id, + &prepared.plan.operation_id, + ) + .map_err(|error| error.to_string())?; + } if close_worker_workdir_sessions(&self.workdir_sessions, &target) .await .is_err() @@ -1177,19 +1196,21 @@ impl WorkspaceServerApi { let worker_key = worker.display_name.clone(); let target = worker.worker; let response = WorkspaceWorkerRemoveExecutor::new(&api) - .execute_target_removal(api.runtime.as_ref(), &target, "Workspace deletion") + .execute_target_removal( + api.runtime.as_ref(), + &target, + "Workspace deletion", + Some(operation_id), + ) .await .map_err(Error::Store)?; - if let Some(child_operation_id) = self.store.latest_worker_removal_operation_id( - &operation.workspace_id, - &target.runtime_id, - &target.worker_id, - )? { - child_operation_ids = self - .store - .append_workspace_deletion_child_operation(operation_id, &child_operation_id)? - .child_operation_ids; - } + child_operation_ids = self + .store + .workspace_deletion_operation_for_recovery(operation_id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + })? + .child_operation_ids; if response.status != 200 { blockers.push(WorkspaceDeletionBlocker { kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, @@ -10873,6 +10894,8 @@ fn execute_workdir_removal_for_workspace_deletion( api.config_store .reserve_workdir_removal_operation(&intent)? }; + api.config_store + .append_workspace_deletion_child_operation(parent_operation_id, &operation.operation_id)?; execute_reserved_workdir_removal(api, operation, false) } @@ -17132,6 +17155,34 @@ mod tests { assert!(source.contains("api.recover_workspace_deletions().await?")); } + #[test] + fn workspace_deletion_checkpoints_child_operations_before_external_cleanup() { + let source = include_str!("server.rs"); + let worker_start = source + .find("async fn execute_target_removal") + .expect("Worker removal helper"); + let worker_source = &source[worker_start..]; + let checkpoint = worker_source + .find("append_workspace_deletion_child_operation") + .expect("Worker child checkpoint"); + let cleanup = worker_source + .find("close_worker_workdir_sessions") + .expect("Worker cleanup side effect"); + assert!(checkpoint < cleanup); + + let workdir_start = source + .find("fn execute_workdir_removal_for_workspace_deletion") + .expect("Workdir removal helper"); + let workdir_source = &source[workdir_start..]; + let checkpoint = workdir_source + .find("append_workspace_deletion_child_operation") + .expect("Workdir child checkpoint"); + let cleanup = workdir_source + .find("execute_reserved_workdir_removal") + .expect("Workdir cleanup side effect"); + assert!(checkpoint < cleanup); + } + fn handler_source<'a>(source: &'a str, name: &str) -> &'a str { let start = source .find(&format!("async fn {name}")) diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index d8e52808..94ae304c 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -110,13 +110,6 @@ pub trait WorkspaceDeletionStore: Send + Sync { operation_id: &str, ) -> Result>; - fn latest_worker_removal_operation_id( - &self, - workspace_id: &str, - runtime_id: &str, - worker_id: &str, - ) -> Result>; - fn workspace_deletion_operation_for_recovery( &self, operation_id: &str, @@ -313,25 +306,6 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { }) } - fn latest_worker_removal_operation_id( - &self, - workspace_id: &str, - runtime_id: &str, - worker_id: &str, - ) -> Result> { - self.with_conn(|conn| { - conn.query_row( - "SELECT operation_id FROM worker_removal_operations - WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 - ORDER BY created_at DESC LIMIT 1", - params![workspace_id, runtime_id, worker_id], - |row| row.get(0), - ) - .optional() - .map_err(Into::into) - }) - } - fn workspace_deletion_operation_for_recovery( &self, operation_id: &str, @@ -970,6 +944,10 @@ mod tests { .finalize_workspace_deletion(&request.operation_id) .expect("finalize"); assert_eq!(completed.state, WorkspaceDeletionState::Succeeded); + assert_eq!( + completed.child_operation_ids, + vec!["child-operation-1".to_string()] + ); let replayed = store .finalize_workspace_deletion(&request.operation_id) .expect("finalize replay"); From b7bba8b53aa61a5c3214e563ddc45e8ea3c6a289 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 10:02:18 +0900 Subject: [PATCH 35/39] fix: preserve workspace deletion checkpoints --- .../src/workspace_deletion.rs | 76 +++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index 94ae304c..c28105b9 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -375,6 +375,19 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { ) -> Result { validate_operation_projection(child_operation_ids, blockers)?; self.with_transaction(|tx| { + let current = read_operation(tx, operation_id)? + .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? + .response; + let mut merged_child_operation_ids = current.child_operation_ids; + for child_operation_id in child_operation_ids { + if !merged_child_operation_ids + .iter() + .any(|existing| existing == child_operation_id) + { + merged_child_operation_ids.push(child_operation_id.clone()); + } + } + validate_operation_projection(&merged_child_operation_ids, blockers)?; let now = Utc::now().to_rfc3339(); let completed_at = matches!(state, WorkspaceDeletionState::Succeeded).then_some(now.as_str()); @@ -386,7 +399,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { params![ operation_id, deletion_state_label(state), - serde_json::to_string(child_operation_ids) + serde_json::to_string(&merged_child_operation_ids) .map_err(|error| Error::Store(error.to_string()))?, serde_json::to_string(blockers) .map_err(|error| Error::Store(error.to_string()))?, @@ -660,10 +673,24 @@ fn workspace_database_blockers( "Remove current Ticket assignments before deleting the Workspace.", ), ( - "SELECT COUNT(*) FROM worker_create_reservations WHERE workspace_id = ?1 AND state = 'reserved'", + "SELECT COUNT(*) + FROM worker_create_reservations reservation + WHERE reservation.workspace_id = ?1 + AND ( + reservation.state = 'reserved' + OR ( + reservation.state = 'created' + AND NOT EXISTS ( + SELECT 1 FROM worker_registry worker + WHERE worker.workspace_id = reservation.workspace_id + AND worker.runtime_id = reservation.runtime_id + AND worker.worker_id = reservation.worker_id + ) + ) + )", WorkspaceDeletionBlockerKind::CleanupUnavailable, "worker", - "Wait for or cancel pending Worker creation reservations.", + "Reconcile pending or incompletely finalized Worker creation reservations.", ), ( "SELECT COUNT(*) FROM worker_removal_operations WHERE workspace_id = ?1 AND state IN ('planned', 'blocked', 'executing', 'failed', 'stale')", @@ -940,6 +967,19 @@ mod tests { duplicate.child_operation_ids, with_child.child_operation_ids ); + let stale_failure_update = store + .update_workspace_deletion_operation( + &request.operation_id, + WorkspaceDeletionState::Blocked, + &[], + &[], + Some("retryable_failure"), + ) + .expect("stale failure update"); + assert_eq!( + stale_failure_update.child_operation_ids, + with_child.child_operation_ids + ); let completed = store .finalize_workspace_deletion(&request.operation_id) .expect("finalize"); @@ -1025,7 +1065,7 @@ mod tests { } #[test] - fn pending_worker_and_workdir_creation_block_reservation_without_orphans() { + fn incomplete_worker_create_and_pending_workdir_create_block_without_orphans() { let (store, owner, workspace_id) = setup(); store .with_conn(|conn| { @@ -1034,7 +1074,7 @@ mod tests { workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, state, created_at, updated_at ) VALUES (?1, 'allocation', 'worker-pending', 'runtime-a', - 'fingerprint', 'reserved', '1', '1')", + 'fingerprint', 'created', '1', '1')", params![workspace_id], )?; conn.execute( @@ -1083,6 +1123,32 @@ mod tests { .expect("operation lookup") .is_none() ); + + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) VALUES (?1, 'runtime-a', 'worker-pending', 'Created worker', '1', '1', 'normal')", + params![workspace_id], + )?; + conn.execute( + "DELETE FROM workdir_create_operations WHERE operation_id = 'workdir-create'", + [], + )?; + Ok(()) + }) + .expect("finalized worker creation"); + let reconciled = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("reconciled preflight"); + assert!( + !reconciled + .blockers + .iter() + .any(|blocker| blocker.message.contains("Worker creation reservations")) + ); } #[test] From 2d1956b653a4fca802a003cac73392787161f6be Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 10:05:54 +0900 Subject: [PATCH 36/39] fix: separate worker state from runtime lifecycle --- crates/protocol/src/lib.rs | 1 + crates/protocol/src/subscription.rs | 6 + crates/tui/src/app.rs | 53 +++-- crates/tui/src/backend_worker_picker.rs | 34 ++- crates/worker-runtime/src/catalog.rs | 6 + crates/worker-runtime/src/http_server.rs | 11 +- crates/worker-runtime/src/runtime.rs | 209 +++++++++++------- crates/worker/src/controller.rs | 138 +++++++++--- crates/worker/src/shared_state.rs | 149 ++++++++++++- crates/workspace-api/src/lib.rs | 5 + crates/workspace-server/src/hosts.rs | 10 + .../src/runtime_subscription_tests.rs | 51 ++++- crates/workspace-server/src/server.rs | 3 + web/workspace/src/lib/generated/protocol.ts | 13 +- .../src/lib/workspace/sidebar/types.ts | 2 + .../src/lib/workspace/sidebar/worker-state.ts | 12 + .../sidebar/worker-subscription.test.ts | 17 ++ .../workspace/sidebar/worker-subscription.ts | 4 +- .../workers/[workerId]/console/+page.svelte | 60 ++--- .../w/[workspaceId]/workers/+page.svelte | 3 +- .../workers/[workerRef]/+page.svelte | 3 +- 21 files changed, 602 insertions(+), 188 deletions(-) create mode 100644 web/workspace/src/lib/workspace/sidebar/worker-state.ts diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index f66be5dc..d54c9232 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -125,6 +125,7 @@ pub enum WorkerCommandDisposition { StaleExecutionGeneration, StaleWorkerStateRevision, StaleCommandId, + Conflict, InvalidState, } diff --git a/crates/protocol/src/subscription.rs b/crates/protocol/src/subscription.rs index 468e241c..bf4a8635 100644 --- a/crates/protocol/src/subscription.rs +++ b/crates/protocol/src/subscription.rs @@ -573,6 +573,11 @@ pub struct SubscriptionWorker { pub resource_key: Option, /// Producer-owned monotonic revision for this Worker subject. pub subject_revision: u64, + /// Latest revisioned foreground state observed from the Worker. This remains + /// absent until an authoritative Worker snapshot/event has been applied. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, + /// Runtime catalog lifecycle compatibility projection; not foreground-state authority. pub state: SubscriptionWorkerState, #[serde(default)] pub has_running_internal_workers: bool, @@ -874,6 +879,7 @@ mod tests { runtime_id: None, resource_key: None, subject_revision: 0, + worker_state: None, state: SubscriptionWorkerState::Idle, has_running_internal_workers: false, workspace_id: Some("workspace-1".to_string()), diff --git a/crates/tui/src/app.rs b/crates/tui/src/app.rs index c543f991..9858dd39 100644 --- a/crates/tui/src/app.rs +++ b/crates/tui/src/app.rs @@ -1180,18 +1180,14 @@ impl App { self.assistant_streaming = false; } Event::TurnStart { .. } => { - self.set_worker_status(WorkerStatus::Running); self.run_requests += 1; self.current_tool = None; self.latest_llm_wait_event = None; self.assistant_streaming = false; } - Event::InvokeStart { .. } => { - self.set_worker_status(WorkerStatus::Running); - } + Event::InvokeStart { .. } => {} // UI consumers of per-attempt LlmCall semantics remain out of scope; - // the run-level status starts at InvokeStart and TurnStart counts each - // LLM request within that run. + // authoritative run state comes only from WorkerStateSnapshot. Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => { self.latest_llm_wait_event = None; } @@ -1398,12 +1394,7 @@ impl App { output_tokens: self.run_output_tokens, }); self.pending_submit_rollback = None; - self.reset_run_state(match result { - RunResult::Paused => WorkerStatus::Paused, - RunResult::Finished | RunResult::LimitReached | RunResult::RolledBack => { - WorkerStatus::Idle - } - }); + self.reset_run_state(); } } Event::CompactStart { .. } => { @@ -1536,7 +1527,7 @@ impl App { }; self.completion = None; self.close_rewind_picker(); - self.reset_run_state(self.worker_status); + self.reset_run_state(); let mut message = if restored_composer { format!( "Rewound session: discarded {} log entries; restored selected input to composer.", @@ -1584,8 +1575,7 @@ impl App { None } - fn reset_run_state(&mut self, status: WorkerStatus) { - self.set_worker_status(status); + fn reset_run_state(&mut self) { self.run_requests = 0; self.run_upload_tokens = 0; self.run_output_tokens = 0; @@ -1615,7 +1605,7 @@ impl App { "Rolled back empty assistant turn; no local submitted input was available to restore." .to_owned() }; - self.reset_run_state(WorkerStatus::Idle); + self.reset_run_state(); self.blocks.push(Block::Alert { level: AlertLevel::Warn, source: AlertSource::Worker, @@ -3583,6 +3573,37 @@ mod completion_flow_tests { assert!(matches!(app.blocks.first(), Some(Block::Greeting(_)))); } + #[test] + fn occurrence_events_do_not_infer_foreground_worker_state() { + let mut app = App::new("test".into()); + app.handle_worker_event(Event::TurnStart { turn: 1 }); + app.handle_worker_event(Event::InvokeStart { + kind: protocol::InvokeKind::UserSend, + }); + app.handle_worker_event(Event::RunEnd { + result: RunResult::Paused, + }); + assert_eq!(app.worker_state.state, protocol::WorkerState::Idle); + assert_eq!(app.worker_status, WorkerStatus::Idle); + + let running = WorkerStateSnapshot { + execution_generation: 1, + revision: 1, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 0, + }; + app.handle_worker_event(Event::WorkerState { + snapshot: running.clone(), + }); + app.handle_worker_event(Event::RunEnd { + result: RunResult::Finished, + }); + assert_eq!(app.worker_state, running); + assert_eq!(app.worker_status, WorkerStatus::Running); + } + #[test] fn worker_state_events_and_acknowledgements_share_monotonic_application() { let mut app = App::new("test".into()); diff --git a/crates/tui/src/backend_worker_picker.rs b/crates/tui/src/backend_worker_picker.rs index e9a37271..473ca5eb 100644 --- a/crates/tui/src/backend_worker_picker.rs +++ b/crates/tui/src/backend_worker_picker.rs @@ -321,7 +321,7 @@ fn row_line( Span::raw(" "), Span::styled( pad_column(&worker_state(worker), widths.state), - state_style(worker.state.as_str()), + state_style(worker_state_label(worker)), ), Span::raw(" "), Span::styled( @@ -341,8 +341,20 @@ fn worker_name(worker: &BackendWorkerSummary) -> &str { } } +fn worker_state_label(worker: &BackendWorkerSummary) -> &str { + match worker.worker_state.as_ref().map(|state| &state.state) { + Some(protocol::WorkerState::Idle) => "idle", + Some(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused, + ))) => "paused", + Some(protocol::WorkerState::Busy(_)) => "running", + None if worker.state == "stopped" => "stopped", + None => "unknown", + } +} + fn worker_state(worker: &BackendWorkerSummary) -> String { - format!("[{}]", worker.state) + format!("[{}]", worker_state_label(worker)) } fn text_width(value: &str) -> usize { @@ -413,7 +425,15 @@ mod tests { identity: "ws".to_string(), workspace_id: Some("ws".to_string()), }, - state: "running".to_string(), + state: "idle".to_string(), + worker_state: Some(protocol::WorkerStateSnapshot { + execution_generation: 1, + revision: 1, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 0, + }), last_seen_at: None, pinned: false, retention_state: String::new(), @@ -450,6 +470,7 @@ mod tests { worker.display_name = "Coder".to_string(); worker.label = "Coder · T-585".to_string(); worker.state = "stopped".to_string(); + worker.worker_state = None; worker.working_directory = Some( serde_json::from_value(serde_json::json!({ "working_directory_id": "001a06a9f0202000000", @@ -478,12 +499,19 @@ mod tests { short.label = "Coder".to_string(); short.display_name = short.label.clone(); short.state = "idle".to_string(); + short.worker_state = Some(protocol::WorkerStateSnapshot { + execution_generation: 1, + revision: 2, + state: protocol::WorkerState::Idle, + last_command_id: 0, + }); let mut long = worker("runtime-a", "worker-b", None); long.resource_key = "W-100".to_string(); long.label = "Longer worker · T-9".to_string(); long.display_name = long.label.clone(); long.state = "stopped".to_string(); + long.worker_state = None; for worker in [&mut short, &mut long] { worker.working_directory = Some( diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index 3cc27d01..96bb2beb 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -307,6 +307,8 @@ pub struct WorkerSummary { pub worker_id: WorkerId, pub status: WorkerStatus, #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub workspace_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub working_directory: Option, @@ -325,6 +327,8 @@ pub struct WorkerDetail { pub worker_id: WorkerId, pub status: WorkerStatus, #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub workspace_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub working_directory: Option, @@ -341,6 +345,8 @@ pub struct WorkerDetail { pub struct WorkerLifecycleAck { pub worker_ref: WorkerRef, pub status: WorkerStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, } #[cfg(test)] diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 28cf26aa..c05bdb91 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -3486,7 +3486,16 @@ mod ws_tests { .. }) if delivered_subscription_id == subscription_id && worker.worker_id.as_str() == worker_ref.worker_id.to_string() - && worker.state == protocol::subscription::SubscriptionWorkerState::Running + && worker.state == protocol::subscription::SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running + )), + .. + }) + ) )); let unsubscribe_request_id = diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 808788f4..76066336 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -700,6 +700,7 @@ impl Runtime { worker_ref: worker_ref.clone(), worker_id: worker_id.clone(), status: WorkerStatus::Stopped, + worker_state: None, workspace_id: scope.map(|scope| scope.workspace_id.clone()), request: durable_request, run_generation: 1, @@ -787,7 +788,6 @@ impl Runtime { let detail = self.commit_created_worker( &worker_ref, handle, - WorkerStatus::Running, working_directory, dispatch_result, )?; @@ -797,7 +797,6 @@ impl Runtime { self.commit_created_worker( &worker_ref, handle, - WorkerStatus::Idle, working_directory, WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn), ) @@ -1220,17 +1219,7 @@ impl Runtime { state.ensure_running()?; let worker = state.worker_mut(worker_ref)?; if let Some(snapshot) = dispatch_result.worker_state.as_ref() { - worker.status = match snapshot.catalog_status() { - protocol::WorkerStatus::Idle => WorkerStatus::Idle, - protocol::WorkerStatus::Running => WorkerStatus::Running, - protocol::WorkerStatus::Paused => WorkerStatus::Paused, - protocol::WorkerStatus::Stopped => WorkerStatus::Stopped, - }; - } else if matches!( - submission.as_ref().map(|ack| ack.disposition), - Some(protocol::SubmissionDisposition::Started) - ) { - worker.status = WorkerStatus::Running; + let _ = worker.apply_worker_state(snapshot); } let status = worker.status; #[cfg(feature = "ws-server")] @@ -1490,16 +1479,19 @@ impl Runtime { &self, worker_ref: &WorkerRef, handle: WorkerExecutionHandle, - status: WorkerStatus, working_directory: Option, - _result: WorkerExecutionResult, + result: WorkerExecutionResult, ) -> Result { let mut state = self.lock()?; let detail = { let worker = state.worker_mut(worker_ref)?; worker.execution_handle = Some(handle); worker.execution_bound = true; - worker.status = status; + worker.status = WorkerStatus::Idle; + worker.worker_state = None; + if let Some(snapshot) = result.worker_state.as_ref() { + let _ = worker.apply_worker_state(snapshot); + } worker.restore_intent = restore_intent_for_status(worker.status); worker.working_directory = working_directory; worker.detail() @@ -1536,16 +1528,14 @@ impl Runtime { let Some(snapshot) = result.worker_state else { return Ok(()); }; - let status = match snapshot.catalog_status() { - protocol::WorkerStatus::Idle => WorkerStatus::Idle, - protocol::WorkerStatus::Running => WorkerStatus::Running, - protocol::WorkerStatus::Paused => WorkerStatus::Paused, - protocol::WorkerStatus::Stopped => WorkerStatus::Stopped, - }; let mut state = self.lock()?; let worker = state.worker_mut(worker_ref)?; - worker.status = status; - worker.restore_intent = restore_intent_for_status(status); + let applied = worker + .apply_worker_state(&snapshot) + .is_ok_and(|result| matches!(result, protocol::WorkerStateSnapshotApply::Applied)); + if !applied { + return Ok(()); + } state.publish_worker_upsert(worker_ref.worker_id)?; state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; @@ -1636,20 +1626,26 @@ impl Runtime { worker_ref: &WorkerRef, reason: Option, ) -> Result { - let current = { + { let state = self.lock()?; state.ensure_running()?; - state.worker(worker_ref)?.status - }; - if matches!(current, WorkerStatus::Idle | WorkerStatus::Stopped) { - return Ok(WorkerLifecycleAck { - worker_ref: worker_ref.clone(), - status: current, - }); + if state.worker(worker_ref)?.status == WorkerStatus::Stopped { + return Ok(WorkerLifecycleAck { + worker_ref: worker_ref.clone(), + status: WorkerStatus::Stopped, + worker_state: None, + }); + } } self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?; let _ = reason; - self.transition_worker_preserving_execution(worker_ref, WorkerStatus::Idle) + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + Ok(WorkerLifecycleAck { + worker_ref: worker_ref.clone(), + status: worker.status, + worker_state: worker.worker_state.clone(), + }) } /// Delete a non-running Worker through a workspace-scoped Runtime authorization context. @@ -1795,12 +1791,13 @@ impl Runtime { ) -> Result { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; - let status_changed = state.project_protocol_event_to_status(worker_ref, &payload); + let worker_state_changed = + state.project_protocol_event_to_worker_state(worker_ref, &payload); let activity_changed = state.project_internal_worker_activity(worker_ref, &payload); - if status_changed || activity_changed { + if worker_state_changed || activity_changed { state.publish_worker_upsert(worker_ref.worker_id)?; } - if status_changed { + if worker_state_changed { state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; } @@ -1836,26 +1833,6 @@ impl Runtime { Ok(()) } - fn transition_worker_preserving_execution( - &self, - worker_ref: &WorkerRef, - status: WorkerStatus, - ) -> Result { - let mut state = self.lock()?; - state.ensure_running()?; - let worker = state.worker_mut(worker_ref)?; - worker.status = status; - worker.restore_intent = restore_intent_for_status(status); - let status = worker.status; - state.publish_worker_upsert(worker_ref.worker_id)?; - state.persist_runtime_snapshot()?; - state.persist_worker(&worker_ref.worker_id)?; - Ok(WorkerLifecycleAck { - worker_ref: worker_ref.clone(), - status, - }) - } - fn transition_worker( &self, worker_ref: &WorkerRef, @@ -1867,6 +1844,7 @@ impl Runtime { let worker = state.worker_mut(worker_ref)?; worker.status = status; + worker.worker_state = None; worker.restore_intent = restore_intent_for_status(status); worker.execution_handle = None; worker.internal_workers.clear(); @@ -1877,6 +1855,7 @@ impl Runtime { Ok(WorkerLifecycleAck { worker_ref: worker_ref.clone(), status, + worker_state: None, }) } @@ -2301,6 +2280,7 @@ impl RuntimeState { worker_ref: worker.worker_ref, worker_id: worker.worker_id, status: worker.status, + worker_state: None, workspace_id: worker.workspace_id, request: worker.request, run_generation, @@ -2630,6 +2610,7 @@ impl RuntimeState { .get(&worker.worker_id) .copied() .unwrap_or(0), + worker_state: worker.worker_state.clone(), state: subscription_worker_state(worker.status), has_running_internal_workers: worker .internal_workers @@ -2965,7 +2946,7 @@ impl RuntimeState { Self::update_internal_worker_activity(&mut worker.internal_workers, event) } - fn project_protocol_event_to_status( + fn project_protocol_event_to_worker_state( &mut self, worker_ref: &WorkerRef, event: &protocol::Event, @@ -2973,7 +2954,7 @@ impl RuntimeState { let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else { return false; }; - let next_status = match event { + let incoming = match event { protocol::Event::WorkerState { snapshot } | protocol::Event::Snapshot { state: snapshot, .. @@ -2983,21 +2964,16 @@ impl RuntimeState { protocol::WorkerCommandAcknowledgement { state: snapshot, .. }, - } => Some(match snapshot.catalog_status() { - protocol::WorkerStatus::Idle => WorkerStatus::Idle, - protocol::WorkerStatus::Running => WorkerStatus::Running, - protocol::WorkerStatus::Paused => WorkerStatus::Paused, - protocol::WorkerStatus::Stopped => WorkerStatus::Stopped, - }), - _ => None, + } => snapshot, + _ => return false, }; - if let Some(next_status) = next_status { - let changed = worker.status != next_status; - worker.status = next_status; - worker.restore_intent = restore_intent_for_status(next_status); - changed - } else { - false + match worker.apply_worker_state(incoming) { + Ok(protocol::WorkerStateSnapshotApply::Applied) => true, + Ok( + protocol::WorkerStateSnapshotApply::Duplicate + | protocol::WorkerStateSnapshotApply::Stale, + ) + | Err(_) => false, } } } @@ -3013,6 +2989,7 @@ struct WorkerRecord { worker_ref: WorkerRef, worker_id: WorkerId, status: WorkerStatus, + worker_state: Option, workspace_id: Option, request: CreateWorkerRequest, run_generation: u64, @@ -3024,6 +3001,19 @@ struct WorkerRecord { } impl WorkerRecord { + fn apply_worker_state( + &mut self, + incoming: &protocol::WorkerStateSnapshot, + ) -> Result { + match self.worker_state.as_mut() { + Some(current) => protocol::apply_worker_state_snapshot(current, incoming), + None => { + self.worker_state = Some(incoming.clone()); + Ok(protocol::WorkerStateSnapshotApply::Applied) + } + } + } + fn belongs_to_workspace(&self, workspace_id: &str) -> bool { self.workspace_id.as_deref() == Some(workspace_id) } @@ -3033,6 +3023,7 @@ impl WorkerRecord { worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, status: self.status, + worker_state: self.worker_state.clone(), workspace_id: self.workspace_id.clone(), working_directory: self.working_directory.clone(), profile: self.request.profile.clone(), @@ -3047,6 +3038,7 @@ impl WorkerRecord { worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, status: self.status, + worker_state: self.worker_state.clone(), workspace_id: self.workspace_id.clone(), working_directory: self.working_directory.clone(), profile: self.request.profile.clone(), @@ -4719,7 +4711,7 @@ mod tests { } #[test] - fn create_worker_uses_started_submission_ack_for_initial_running_status() { + fn create_worker_does_not_infer_state_from_started_submission_ack() { let (runtime, backend) = runtime_and_backend(); backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, @@ -4732,7 +4724,69 @@ mod tests { let detail = runtime.create_worker(request).unwrap(); - assert_eq!(detail.status, WorkerStatus::Running); + assert_eq!(detail.status, WorkerStatus::Idle); + assert_eq!(detail.worker_state, None); + } + + #[test] + fn runtime_applies_only_newer_worker_state_snapshots() { + let (runtime, _) = runtime_and_backend(); + let detail = runtime + .create_worker(task_request("state ordering")) + .unwrap(); + let running = protocol::WorkerStateSnapshot { + execution_generation: 7, + revision: 3, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 2, + }; + assert!({ + let mut state = runtime.lock().unwrap(); + state.project_protocol_event_to_worker_state( + &detail.worker_ref, + &protocol::Event::WorkerState { + snapshot: running.clone(), + }, + ) + }); + assert_eq!( + runtime + .worker_detail(&detail.worker_ref) + .unwrap() + .worker_state, + Some(running.clone()) + ); + + assert!({ + let mut state = runtime.lock().unwrap(); + !state.project_protocol_event_to_worker_state( + &detail.worker_ref, + &protocol::Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + revision: 2, + state: protocol::WorkerState::Idle, + ..running.clone() + }, + }, + ) + }); + assert!({ + let mut state = runtime.lock().unwrap(); + !state.project_protocol_event_to_worker_state( + &detail.worker_ref, + &protocol::Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Idle, + ..running.clone() + }, + }, + ) + }); + let after = runtime.worker_detail(&detail.worker_ref).unwrap(); + assert_eq!(after.status, WorkerStatus::Idle); + assert_eq!(after.worker_state, Some(running)); } #[test] @@ -5027,10 +5081,9 @@ mod tests { assert_eq!(*backend.restore_count.lock().unwrap(), 1); assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]); - assert_eq!( - runtime.worker_detail(&detail.worker_ref).unwrap().status, - WorkerStatus::Running - ); + let restored = runtime.worker_detail(&detail.worker_ref).unwrap(); + assert_eq!(restored.status, WorkerStatus::Idle); + assert_eq!(restored.worker_state, None); } #[test] diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index beeb4791..1655a387 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -17,7 +17,7 @@ use crate::ipc::notify_buffer::NotifyBuffer; use crate::ipc::server::SocketServer; use crate::runtime::dir::RuntimeDir; use crate::segment_log_sink::SegmentLogSink; -use crate::shared_state::WorkerSharedState; +use crate::shared_state::{WorkerCommandAdmission, WorkerSharedState}; use crate::shutdown_after_idle::{ ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role, take_shutdown_request_after_status, @@ -181,21 +181,40 @@ impl WorkerHandle { } } +fn command_admission_disposition( + admission: WorkerCommandAdmission, +) -> Result<(), WorkerCommandDisposition> { + match admission { + WorkerCommandAdmission::Accepted => Ok(()), + WorkerCommandAdmission::Retry | WorkerCommandAdmission::StaleCommandId => { + Err(WorkerCommandDisposition::StaleCommandId) + } + WorkerCommandAdmission::Conflict => Err(WorkerCommandDisposition::Conflict), + WorkerCommandAdmission::ExecutionGenerationMismatch => { + Err(WorkerCommandDisposition::StaleExecutionGeneration) + } + WorkerCommandAdmission::StateRevisionMismatch => { + Err(WorkerCommandDisposition::StaleWorkerStateRevision) + } + } +} + fn validate_command( + envelope: WorkerCommandEnvelope, + kind: WorkerCommandKind, + shared_state: &WorkerSharedState, +) -> Result<(), WorkerCommandDisposition> { + command_admission_disposition(shared_state.admit_command(envelope, kind, true)) +} + +fn validate_shutdown_command( envelope: WorkerCommandEnvelope, shared_state: &WorkerSharedState, ) -> Result<(), WorkerCommandDisposition> { - let snapshot = shared_state.snapshot(); - if envelope.expected_execution_generation != snapshot.execution_generation { - return Err(WorkerCommandDisposition::StaleExecutionGeneration); + match shared_state.admit_command(envelope, WorkerCommandKind::Shutdown, false) { + WorkerCommandAdmission::Accepted | WorkerCommandAdmission::Retry => Ok(()), + admission => command_admission_disposition(admission), } - if envelope.expected_worker_state_revision != snapshot.revision { - return Err(WorkerCommandDisposition::StaleWorkerStateRevision); - } - if !shared_state.accept_command_id(envelope.command_id) { - return Err(WorkerCommandDisposition::StaleCommandId); - } - Ok(()) } fn acknowledge_command( @@ -205,6 +224,7 @@ fn acknowledge_command( command: WorkerCommandKind, disposition: WorkerCommandDisposition, ) { + shared_state.complete_command(command_id, command, disposition); let _ = working_event_tx.send(Event::CommandAcknowledged { acknowledgement: WorkerCommandAcknowledgement { command_id, @@ -1913,7 +1933,9 @@ async fn controller_loop( } } Method::Resume { command } => { - if let Err(disposition) = validate_command(command, &shared_state) { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Resume, &shared_state) + { acknowledge_command( &working_event_tx, &shared_state, @@ -1953,7 +1975,9 @@ async fn controller_loop( } Method::Cancel { command } => { - if let Err(disposition) = validate_command(command, &shared_state) { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Cancel, &shared_state) + { acknowledge_command( &working_event_tx, &shared_state, @@ -2017,7 +2041,9 @@ async fn controller_loop( } Method::Pause { command } => { - if let Err(disposition) = validate_command(command, &shared_state) { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Pause, &shared_state) + { acknowledge_command( &working_event_tx, &shared_state, @@ -2036,7 +2062,9 @@ async fn controller_loop( } Method::Compact { command } => { - if let Err(disposition) = validate_command(command, &shared_state) { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Compact, &shared_state) + { acknowledge_command( &working_event_tx, &shared_state, @@ -2081,7 +2109,11 @@ async fn controller_loop( method = method_rx.recv() => { match method { Some(Method::Cancel { command }) => { - if let Err(disposition) = validate_command(command, &shared_state) { + if let Err(disposition) = validate_command( + command, + WorkerCommandKind::Cancel, + &shared_state, + ) { acknowledge_command( &working_event_tx, &shared_state, @@ -2101,7 +2133,18 @@ async fn controller_loop( let _ = cancel_tx.send(true); } Some(Method::Shutdown { command }) => { - shared_state.accept_command_id(command.command_id); + if let Err(disposition) = + validate_shutdown_command(command, &shared_state) + { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + disposition, + ); + continue; + } shutdown_after_compaction = true; acknowledge_command( &working_event_tx, @@ -2196,9 +2239,18 @@ async fn controller_loop( }, Method::Shutdown { command } => { - // Shutdown remains unconditional/retryable even when the caller's - // live-state fence is stale. - shared_state.accept_command_id(command.command_id); + // Shutdown ignores the state-revision fence but remains bound to the + // current execution generation and command payload identity. + if let Err(disposition) = validate_shutdown_command(command, &shared_state) { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + disposition, + ); + continue; + } acknowledge_command( &working_event_tx, &shared_state, @@ -2544,7 +2596,9 @@ where method = method_rx.recv(), if input_commit.is_none() => { match method { Some(Method::Cancel { command }) => { - if let Err(disposition) = validate_command(command, shared_state) { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Cancel, shared_state) + { acknowledge_command( working_event_tx, shared_state, @@ -2583,7 +2637,9 @@ where let _ = cancel_tx.try_send(()); } Some(Method::Pause { command }) => { - if let Err(disposition) = validate_command(command, shared_state) { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Pause, shared_state) + { acknowledge_command( working_event_tx, shared_state, @@ -2623,7 +2679,16 @@ where let _ = pause_tx.try_send(()); } Some(Method::Shutdown { command }) => { - shared_state.accept_command_id(command.command_id); + if let Err(disposition) = validate_shutdown_command(command, shared_state) { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + disposition, + ); + continue; + } shutdown_requested = true; set_controller_state( shared_state, @@ -2705,7 +2770,9 @@ where } } Some(Method::Resume { command }) => { - if let Err(disposition) = validate_command(command, shared_state) { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Resume, shared_state) + { acknowledge_command( working_event_tx, shared_state, @@ -2763,7 +2830,9 @@ where } } Some(Method::Compact { command }) => { - if let Err(disposition) = validate_command(command, shared_state) { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Compact, shared_state) + { acknowledge_command( working_event_tx, shared_state, @@ -3677,6 +3746,7 @@ mod tests { expected_execution_generation: 8, expected_worker_state_revision: 0, }, + WorkerCommandKind::Pause, &shared, ), Err(WorkerCommandDisposition::StaleExecutionGeneration) @@ -3688,6 +3758,7 @@ mod tests { expected_execution_generation: 9, expected_worker_state_revision: 1, }, + WorkerCommandKind::Pause, &shared, ), Err(WorkerCommandDisposition::StaleWorkerStateRevision) @@ -3699,6 +3770,7 @@ mod tests { expected_execution_generation: 9, expected_worker_state_revision: 0, }, + WorkerCommandKind::Pause, &shared, ) .is_ok() @@ -3708,12 +3780,25 @@ mod tests { WorkerCommandEnvelope { command_id: 1, expected_execution_generation: 9, - expected_worker_state_revision: 1, + expected_worker_state_revision: 0, }, + WorkerCommandKind::Pause, &shared, ), Err(WorkerCommandDisposition::StaleCommandId) ); + assert_eq!( + validate_command( + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 9, + expected_worker_state_revision: 0, + }, + WorkerCommandKind::Cancel, + &shared, + ), + Err(WorkerCommandDisposition::Conflict) + ); assert!( validate_command( WorkerCommandEnvelope { @@ -3721,6 +3806,7 @@ mod tests { expected_execution_generation: 9, expected_worker_state_revision: 1, }, + WorkerCommandKind::Pause, &shared, ) .is_ok() diff --git a/crates/worker/src/shared_state.rs b/crates/worker/src/shared_state.rs index 573f2bb5..810b59f4 100644 --- a/crates/worker/src/shared_state.rs +++ b/crates/worker/src/shared_state.rs @@ -1,17 +1,37 @@ +use std::collections::VecDeque; use std::sync::{ OnceLock, RwLock, atomic::{AtomicBool, Ordering}, }; use protocol::{ - WorkerBusyState, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, - WorkerStatus, + WorkerBusyState, WorkerCommandDisposition, WorkerCommandEnvelope, WorkerCommandKind, + WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerStatus, }; use serde_json::json; use session_store::SegmentId; use crate::fs_view::WorkerFsView; +const COMPLETED_COMMAND_RETENTION: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AcceptedWorkerCommand { + envelope: WorkerCommandEnvelope, + kind: WorkerCommandKind, + disposition: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum WorkerCommandAdmission { + Accepted, + Retry, + Conflict, + StaleCommandId, + ExecutionGenerationMismatch, + StateRevisionMismatch, +} + /// Shared state between WorkerController and runtime directory. /// /// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime @@ -23,6 +43,7 @@ pub struct WorkerSharedState { pub manifest_toml: String, pub greeting: protocol::Greeting, state: RwLock, + accepted_commands: RwLock>, /// Worker-from-the-inside view of the filesystem. Set once in /// `WorkerController::start` after the local WorkdirSession provider is /// materialised, and read from the IPC server layer to answer @@ -55,6 +76,7 @@ impl WorkerSharedState { manifest_toml, greeting, state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)), + accepted_commands: RwLock::new(VecDeque::new()), fs_view: OnceLock::new(), flow_transition_enabled: AtomicBool::new(false), } @@ -92,17 +114,99 @@ impl WorkerSharedState { snapshot.clone() } - pub fn accept_command_id(&self, command_id: u64) -> bool { + pub(crate) fn admit_command( + &self, + envelope: WorkerCommandEnvelope, + kind: WorkerCommandKind, + require_state_revision: bool, + ) -> WorkerCommandAdmission { let mut snapshot = self .state .write() .expect("worker state lock poisoned; refusing command admission"); - if command_id <= snapshot.last_command_id { - return false; + let mut accepted = self + .accepted_commands + .write() + .expect("worker command ledger lock poisoned; refusing command admission"); + if let Some(existing) = accepted + .iter() + .find(|accepted| accepted.envelope.command_id == envelope.command_id) + { + return if existing.envelope == envelope && existing.kind == kind { + WorkerCommandAdmission::Retry + } else { + WorkerCommandAdmission::Conflict + }; } - snapshot.last_command_id = command_id; + if envelope.expected_execution_generation != snapshot.execution_generation { + return WorkerCommandAdmission::ExecutionGenerationMismatch; + } + if require_state_revision && envelope.expected_worker_state_revision != snapshot.revision { + return WorkerCommandAdmission::StateRevisionMismatch; + } + if envelope.command_id <= snapshot.last_command_id { + return WorkerCommandAdmission::StaleCommandId; + } + + snapshot.last_command_id = envelope.command_id; snapshot.revision = snapshot.revision.saturating_add(1); - true + accepted.push_back(AcceptedWorkerCommand { + envelope, + kind, + disposition: None, + }); + WorkerCommandAdmission::Accepted + } + + pub(crate) fn complete_command( + &self, + command_id: u64, + kind: WorkerCommandKind, + disposition: WorkerCommandDisposition, + ) { + if !matches!( + disposition, + WorkerCommandDisposition::Accepted | WorkerCommandDisposition::InvalidState + ) { + return; + } + let mut accepted = self + .accepted_commands + .write() + .expect("worker command ledger lock poisoned; refusing command completion"); + if let Some(command) = accepted + .iter_mut() + .find(|command| command.envelope.command_id == command_id && command.kind == kind) + { + command.disposition.get_or_insert(disposition); + } + while accepted + .iter() + .filter(|command| command.disposition.is_some()) + .count() + > COMPLETED_COMMAND_RETENTION + { + let Some(index) = accepted + .iter() + .position(|command| command.disposition.is_some()) + else { + break; + }; + accepted.remove(index); + } + } + + #[cfg(test)] + pub(crate) fn command_result( + &self, + command_id: u64, + ) -> Option> { + self.accepted_commands + .read() + .expect("worker command ledger lock poisoned") + .iter() + .find(|command| command.envelope.command_id == command_id) + .map(|command| command.disposition) } pub fn snapshot(&self) -> WorkerStateSnapshot { @@ -190,9 +294,17 @@ mod tests { } #[test] - fn accepted_command_id_advances_the_snapshot_revision_atomically() { + fn accepted_command_identity_advances_revision_and_detects_reuse_conflicts() { let state = test_state(); - assert!(state.accept_command_id(9)); + let envelope = WorkerCommandEnvelope { + command_id: 9, + expected_execution_generation: 7, + expected_worker_state_revision: 0, + }; + assert_eq!( + state.admit_command(envelope, WorkerCommandKind::Pause, true), + WorkerCommandAdmission::Accepted + ); assert_eq!( state.snapshot(), WorkerStateSnapshot { @@ -202,7 +314,24 @@ mod tests { state: WorkerState::Idle, } ); - assert!(!state.accept_command_id(9)); + assert_eq!(state.command_result(9), Some(None)); + state.complete_command( + 9, + WorkerCommandKind::Pause, + WorkerCommandDisposition::Accepted, + ); + assert_eq!( + state.command_result(9), + Some(Some(WorkerCommandDisposition::Accepted)) + ); + assert_eq!( + state.admit_command(envelope, WorkerCommandKind::Pause, true), + WorkerCommandAdmission::Retry + ); + assert_eq!( + state.admit_command(envelope, WorkerCommandKind::Cancel, true), + WorkerCommandAdmission::Conflict + ); assert_eq!(state.snapshot().revision, 1); } diff --git a/crates/workspace-api/src/lib.rs b/crates/workspace-api/src/lib.rs index a2cf8f87..9bdd7bce 100644 --- a/crates/workspace-api/src/lib.rs +++ b/crates/workspace-api/src/lib.rs @@ -1568,7 +1568,12 @@ pub struct WorkerSummary { #[serde(default)] pub tags: Vec, pub workspace: WorkerWorkspaceSummary, + /// Runtime catalog lifecycle compatibility state. Live foreground state, when + /// available, is carried separately in `worker_state`. pub state: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "typescript", ts(optional))] + pub worker_state: Option, pub last_seen_at: Option, #[serde(default)] pub pinned: bool, diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index f615685d..a6b065b2 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -243,7 +243,10 @@ pub struct WorkerSummary { #[serde(default)] pub tags: Vec, pub workspace: WorkerWorkspaceSummary, + /// Runtime catalog lifecycle compatibility state. pub state: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, pub last_seen_at: Option, #[serde(default)] pub pinned: bool, @@ -335,6 +338,7 @@ pub(crate) fn workspace_worker_summary( workspace_id: summary.workspace.workspace_id, }, state: summary.state, + worker_state: summary.worker_state, last_seen_at: summary.last_seen_at, pinned: summary.pinned, retention_state: summary.retention_state, @@ -1998,6 +2002,7 @@ impl EmbeddedWorkerRuntime { workspace_id: summary.workspace_id.clone(), }, state: embedded_worker_status_label(summary.status).to_string(), + worker_state: summary.worker_state.clone(), last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -2037,6 +2042,7 @@ impl EmbeddedWorkerRuntime { workspace_id: detail.workspace_id.clone(), }, state: embedded_worker_status_label(detail.status).to_string(), + worker_state: detail.worker_state.clone(), last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -3340,6 +3346,7 @@ impl RemoteWorkerRuntime { workspace_id: summary.workspace_id.clone(), }, state: embedded_worker_status_label(summary.status).to_string(), + worker_state: summary.worker_state.clone(), last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -3383,6 +3390,7 @@ impl RemoteWorkerRuntime { workspace_id: detail.workspace_id.clone(), }, state: embedded_worker_status_label(detail.status).to_string(), + worker_state: detail.worker_state.clone(), last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -4730,6 +4738,7 @@ pub fn placeholder_worker(host_id: impl Into) -> WorkerSummary { workspace_id: None, }, state: "unsupported".to_string(), + worker_state: None, last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -5251,6 +5260,7 @@ mod tests { workspace_id: None, }, state: "available".to_string(), + worker_state: None, last_seen_at: None, pinned: false, retention_state: "transient".to_string(), diff --git a/crates/workspace-server/src/runtime_subscription_tests.rs b/crates/workspace-server/src/runtime_subscription_tests.rs index 5c67f983..ab5d65ac 100644 --- a/crates/workspace-server/src/runtime_subscription_tests.rs +++ b/crates/workspace-server/src/runtime_subscription_tests.rs @@ -202,7 +202,16 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() { BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { ref worker }, .. - } if worker.state == SubscriptionWorkerState::Running + } if worker.state == SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running + )), + .. + }) + ) )); } let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap(); @@ -212,7 +221,18 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() { assert!(matches!( snapshot, SubscriptionSnapshot::Workers { workers } - if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running) + if workers.iter().any(|worker| { + worker.state == SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running) + ), + .. + }) + ) + }) )); drop(late); @@ -337,8 +357,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() { ) .unwrap(); assert!(matches!(next_event(&mut subscription).await, - BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. } - if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") && worker.state == SubscriptionWorkerState::Running)); + BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. } + if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") + && worker.state == SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running + )), + .. + }) + ))); let mut late = broker .subscribe( "embedded-worker-runtime", @@ -351,7 +381,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() { assert!(matches!( snapshot, SubscriptionSnapshot::Workers { workers } - if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running) + if workers.iter().any(|worker| { + worker.state == SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running) + ), + .. + }) + ) + }) )); runtime diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 39206e5d..9cbeb899 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -15252,6 +15252,7 @@ fn worker_summary_from_registry(record: &WorkerRegistryRecord) -> WorkerSummary singleton_key: None, tags: Vec::new(), state: "missing".to_string(), + worker_state: None, last_seen_at: Some(record.updated_at.clone()), pinned: record.retention_state == "pinned", retention_state: record.retention_state.clone(), @@ -24990,6 +24991,7 @@ mod tests { workspace_id: Some(TEST_WORKSPACE_ID.to_string()), }, state: "idle".to_string(), + worker_state: None, last_seen_at: None, pinned: false, retention_state: "normal".to_string(), @@ -25086,6 +25088,7 @@ mod tests { workspace_id: Some(TEST_WORKSPACE_ID.to_string()), }, state: "idle".to_string(), + worker_state: None, last_seen_at: None, pinned: false, retention_state: "normal".to_string(), diff --git a/web/workspace/src/lib/generated/protocol.ts b/web/workspace/src/lib/generated/protocol.ts index c15fbdca..b3482873 100644 --- a/web/workspace/src/lib/generated/protocol.ts +++ b/web/workspace/src/lib/generated/protocol.ts @@ -19,7 +19,7 @@ command_id: number, expected_execution_generation: number, expected_worker_state export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown"; -export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "invalid_state"; +export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state"; export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition, /** @@ -233,7 +233,16 @@ resource_key?: string | null, /** * Producer-owned monotonic revision for this Worker subject. */ -subject_revision: number, state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null, +subject_revision: number, +/** + * Latest revisioned foreground state observed from the Worker. This remains + * absent until an authoritative Worker snapshot/event has been applied. + */ +worker_state?: WorkerStateSnapshot | null, +/** + * Runtime catalog lifecycle compatibility projection; not foreground-state authority. + */ +state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null, /** * Workspace-facing Repository key. Runtime producers leave this unset and * Workspace Server projections replace `repository_id` with this field. diff --git a/web/workspace/src/lib/workspace/sidebar/types.ts b/web/workspace/src/lib/workspace/sidebar/types.ts index 67d7aa10..856076fd 100644 --- a/web/workspace/src/lib/workspace/sidebar/types.ts +++ b/web/workspace/src/lib/workspace/sidebar/types.ts @@ -19,6 +19,7 @@ import type { Event as PodProtocolEvent, Method as PodProtocolMethod, Segment as PodProtocolSegment, + WorkerStateSnapshot, } from "$lib/generated/protocol"; import type { GitCommitSummary as SharedGitCommitSummary, @@ -99,6 +100,7 @@ export type Worker = { tags: string[]; workspace: { visibility: string; identity: string }; state: string; + worker_state?: WorkerStateSnapshot | null; pinned?: boolean; retention_state?: string; last_seen_at?: string | null; diff --git a/web/workspace/src/lib/workspace/sidebar/worker-state.ts b/web/workspace/src/lib/workspace/sidebar/worker-state.ts new file mode 100644 index 00000000..0f45e9f3 --- /dev/null +++ b/web/workspace/src/lib/workspace/sidebar/worker-state.ts @@ -0,0 +1,12 @@ +import type { WorkerStateSnapshot } from "$lib/generated/protocol"; + +export function liveWorkerState(worker: { + state: string; + worker_state?: WorkerStateSnapshot | null; +}): string { + const state = worker.worker_state?.state; + if (!state) return worker.state === "stopped" ? "stopped" : "unknown"; + if (state.kind === "idle") return "idle"; + if (state.state.kind === "maintenance") return "running"; + return state.state.state === "paused" ? "paused" : "running"; +} diff --git a/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts b/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts index 279afa0d..84e81598 100644 --- a/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts +++ b/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts @@ -5,6 +5,7 @@ function assertEquals(actual: unknown, expected: unknown): void { throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); } } +import { liveWorkerState } from './worker-state'; import { applyWorkspaceWorkersFrame, createWorkspaceWorkersProjection, @@ -33,6 +34,22 @@ function worker( }; } +Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => { + const active = worker('runtime-a', 'worker-1', 1); + active.worker_state = { + execution_generation: 4, + revision: 2, + last_command_id: 1, + state: { kind: 'busy', state: { kind: 'run', state: 'paused' } }, + }; + assertEquals(liveWorkerState(active), 'paused'); + + const unavailable = worker('runtime-a', 'worker-2', 1); + assertEquals(liveWorkerState(unavailable), 'unknown'); + unavailable.state = 'stopped'; + assertEquals(liveWorkerState(unavailable), 'stopped'); +}); + Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => { const projection = createWorkspaceWorkersProjection(); const frame: SubscriptionFrame = { diff --git a/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts b/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts index a1eed656..8c9b7557 100644 --- a/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts +++ b/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts @@ -5,6 +5,7 @@ import { applyWorkspaceWorkersFrame, createWorkspaceWorkersProjection, } from './worker-subscription-model'; +import { liveWorkerState } from './worker-state'; import { compareWorkersForSidebar } from './workers'; import type { Worker } from './types'; @@ -86,7 +87,8 @@ function projectWorker(worker: SubscriptionWorker): SidebarWorker { profile: worker.profile ?? null, tags: [], workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' }, - state: worker.state, + state: liveWorkerState(worker), + worker_state: worker.worker_state, pinned: false, retention_state: 'transient', implementation: { diff --git a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte index cf28f294..ed8d1df4 100644 --- a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte @@ -52,11 +52,7 @@ import { pushWorkspaceAlert } from "$lib/workspace/alerts/store"; import { workspaceApiPath } from "$lib/workspace/api/http"; import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer"; - import type { - Diagnostic, - Worker, - PodProtocolEvent, - } from "$lib/workspace/sidebar/types"; + import type { Diagnostic, Worker } from "$lib/workspace/sidebar/types"; type Props = { data: { @@ -207,7 +203,6 @@ ); let pendingObservationEvents: ConsoleEventInput[] = []; let protocolEventSequence = 0; - let pendingObservedStates: Array = []; let pendingStreamDiagnostics: Diagnostic[] = []; let observationFlushHandle: number | null = null; let nextReloadToken = 0; @@ -249,7 +244,9 @@ const diagnostics = $derived( mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics), ); - const workerState = $derived(liveWorkerState ?? worker?.state ?? "loading"); + const workerState = $derived( + liveWorkerState ?? (worker?.state === "stopped" ? "stopped" : "loading"), + ); const workerRunning = $derived(workerState === "running"); const workerPaused = $derived(workerState === "paused"); const composerEditable = $derived(protocolState === "open" && !sending); @@ -343,7 +340,6 @@ observationFlushHandle = null; } pendingObservationEvents = []; - pendingObservedStates = []; pendingStreamDiagnostics = []; } @@ -359,18 +355,15 @@ function flushObservationBatch() { observationFlushHandle = null; const eventBatch = pendingObservationEvents; - const stateBatch = pendingObservedStates; const diagnosticBatch = pendingStreamDiagnostics; pendingObservationEvents = []; - pendingObservedStates = []; pendingStreamDiagnostics = []; if (eventBatch.length > 0) { - const latestState = stateBatch.findLast((state) => state !== null); - if (latestState) { - liveWorkerState = latestState; - } consoleProjection = consoleProjector.append(eventBatch); + liveWorkerState = consoleProjection.status === "shutdown" + ? "shutdown" + : workerStateFromSnapshot(consoleProjection.workerState); advanceEventObservedAtVersion(); } @@ -407,7 +400,6 @@ event: payload, observedAtMs, }); - pendingObservedStates.push(workerStateFromProtocolEvent(payload)); scheduleObservationFlush(); } @@ -925,36 +917,16 @@ handleComposerSubmit(); } - function workerStateFromProtocolEvent( - event: PodProtocolEvent, + function workerStateFromSnapshot( + snapshot: ConsoleProjection["workerState"], ): string | null { - switch (event.event) { - case "snapshot": - return event.data.state.state.kind === "idle" - ? "idle" - : event.data.state.state.state.kind === "run" && - event.data.state.state.state.state === "paused" - ? "paused" - : "running"; - case "worker_state": - return event.data.snapshot.state.kind === "idle" - ? "idle" - : event.data.snapshot.state.state.kind === "run" && - event.data.snapshot.state.state.state === "paused" - ? "paused" - : "running"; - case "command_acknowledged": - return event.data.acknowledgement.state.state.kind === "idle" - ? "idle" - : event.data.acknowledgement.state.state.state.kind === "run" && - event.data.acknowledgement.state.state.state.state === "paused" - ? "paused" - : "running"; - case "shutdown": - return "shutdown"; - default: - return null; - } + if (!snapshot) return null; + return snapshot.state.kind === "idle" + ? "idle" + : snapshot.state.state.kind === "run" && + snapshot.state.state.state === "paused" + ? "paused" + : "running"; } function connectProtocolTransport( diff --git a/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte index e734def2..e3a0d542 100644 --- a/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte @@ -4,6 +4,7 @@ import { workerHref } from '$lib/workspace/resource-links'; import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision'; import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers'; + import { liveWorkerState } from '$lib/workspace/sidebar/worker-state'; import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types'; import type { PageProps } from './$types'; @@ -136,7 +137,7 @@ } function workerStatus(worker: Worker): string { - return worker.state; + return liveWorkerState(worker); } function workerProfile(worker: Worker): string { diff --git a/web/workspace/src/routes/w/[workspaceId]/workers/[workerRef]/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/workers/[workerRef]/+page.svelte index ef3d04bb..15d3ffe4 100644 --- a/web/workspace/src/routes/w/[workspaceId]/workers/[workerRef]/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/workers/[workerRef]/+page.svelte @@ -1,5 +1,6 @@ @@ -24,7 +25,7 @@ >Open console
-
Status
{data.worker.state}
+
Status
{liveWorkerState(data.worker)}
Profile
{data.worker.profile}
Internal ID
{data.worker.worker_id}
From 101a0acb6bd58a19ea1618a29a829fe6106e9d45 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 10:32:45 +0900 Subject: [PATCH 37/39] fix: terminalize removed worker reservations --- crates/workspace-server/src/latest_schema.sql | 2 +- crates/workspace-server/src/retention.rs | 11 ++ crates/workspace-server/src/store.rs | 167 +++++++++++++++++- .../src/workspace_deletion.rs | 68 +++++++ 4 files changed, 244 insertions(+), 4 deletions(-) diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 919e0957..7a7c2199 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -659,7 +659,7 @@ CREATE TABLE worker_create_reservations ( worker_id TEXT NOT NULL, runtime_id TEXT NOT NULL, create_fingerprint TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('reserved', 'created')), + state TEXT NOT NULL CHECK (state IN ('reserved', 'created', 'removed')), created_at TEXT NOT NULL, updated_at TEXT NOT NULL, request_fingerprint TEXT, memory_settings_revision INTEGER, memory_language TEXT, PRIMARY KEY (workspace_id, allocation_key), diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index 474abeac..19c22ccb 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -464,6 +464,7 @@ impl SqliteWorkspaceStore { tx.execute("INSERT OR IGNORE INTO worker_tombstones(workspace_id,runtime_id,worker_id,display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_revision,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,worker.display_name,worker.profile,worker.created_at,now,plan.archive_id,plan.policy_id,plan.policy_revision,operation_id])?; } let deleted=tx.execute("DELETE FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND updated_at=?4",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.worker_revision])?; + tx.execute("UPDATE worker_create_reservations SET state='removed',updated_at=?4 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='created'",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,now])?; if deleted!=1{return Err(StoreError::InvalidInput(format!("stale:{}:removal fence changed",plan.plan_id)));} tx.execute("UPDATE worker_removal_operations SET state='succeeded',failure_category=NULL,updated_at=?1 WHERE operation_id=?2",params![now,operation_id])?; tx.execute("INSERT OR IGNORE INTO worker_retention_audit_events(event_id,operation_id,workspace_id,event_kind,detail,created_at) VALUES(?1,?2,?3,'worker_removed',?4,?5)",params![stable("wre",operation_id),operation_id,workspace_id,format!("runtime_id={} worker_id={} session={} metadata={} diagnostics={}",plan.worker.runtime_id,plan.worker.worker_id,sess(plan.session_disposition),meta(plan.metadata_disposition),diag(plan.diagnostics_disposition)),now])?; @@ -1166,6 +1167,7 @@ mod tests { s.with_conn(|conn| { conn.execute("INSERT INTO typed_tickets(workspace_id,ticket_id,slug,title,status,kind,priority,body,workflow_state,workflow_state_explicit) VALUES('w','ticket-old','ticket-old','Old Ticket','open','task','normal','','planning',1)", [])?; conn.execute("INSERT INTO worker_registry(workspace_id,worker_id,runtime_id,display_name,profile,retention_state,created_at,updated_at) VALUES('w','1','r','old worker','builtin:coder','normal','created','rev1')", [])?; + conn.execute("INSERT INTO worker_create_reservations(workspace_id,allocation_key,worker_id,runtime_id,create_fingerprint,state,created_at,updated_at) VALUES('w','allocation-old',?1,'r','fingerprint','created','created','created')", [worker_id().to_string()])?; conn.execute("INSERT INTO ticket_worker_assignments(workspace_id,ticket_id,assignment_id,runtime_id,worker_id,assigned_by,assigned_at) VALUES('w','ticket-old','assignment-old','r','1','test','t')", [])?; conn.execute("DELETE FROM worker_registry WHERE workspace_id='w' AND runtime_id='r' AND worker_id='1'", [])?; conn.execute("DELETE FROM typed_tickets WHERE workspace_id='w' AND ticket_id='ticket-old'", [])?; @@ -1212,6 +1214,15 @@ mod tests { WorkerRemovalPlanState::Succeeded ); assert!(s.worker_tombstone("w", &p.worker).unwrap().is_some()); + let reservation_state: String = s.with_conn(|conn| { + conn.query_row( + "SELECT state FROM worker_create_reservations WHERE workspace_id='w' AND allocation_key='allocation-old'", + [], + |row| row.get(0), + ) + .map_err(StoreError::from) + }).unwrap(); + assert_eq!(reservation_state, "removed"); assert_eq!( s.commit_worker_removal("w", &p.operation_id, &p.input_fingerprint, &result) .unwrap() diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index aa0d7b85..bd934788 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -1264,7 +1264,7 @@ impl SqliteWorkspaceStore { let existing = tx .query_row( "SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \ - memory_settings_revision, memory_language \ + memory_settings_revision, memory_language, state \ FROM worker_create_reservations \ WHERE workspace_id = ?1 AND allocation_key = ?2", params![workspace_id, allocation_key], @@ -1276,11 +1276,17 @@ impl SqliteWorkspaceStore { row.get::<_, String>(3)?, row.get::<_, Option>(4)?, row.get::<_, Option>(5)?, + row.get::<_, String>(6)?, )) }, ) .optional()?; - if let Some((worker_id, reserved_runtime_id, stored_request_fingerprint, create_fingerprint, revision, language)) = existing { + if let Some((worker_id, reserved_runtime_id, stored_request_fingerprint, create_fingerprint, revision, language, state)) = existing { + if state == "removed" { + return Err(Error::InvalidInput(format!( + "Worker create allocation {allocation_key} was terminally removed" + ))); + } if reserved_runtime_id != runtime_id || stored_request_fingerprint.as_deref() != Some(request_fingerprint) { @@ -1391,7 +1397,7 @@ impl SqliteWorkspaceStore { let changed = conn.execute( "UPDATE worker_create_reservations \ SET state = 'created', updated_at = ?3 \ - WHERE workspace_id = ?1 AND worker_id = ?2", + WHERE workspace_id = ?1 AND worker_id = ?2 AND state IN ('reserved', 'created')", params![ workspace_id, worker_id.to_string(), @@ -3746,6 +3752,20 @@ impl ControlPlaneStore for SqliteWorkspaceStore { "DELETE FROM worker_registry WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", params![workspace_id, worker.runtime_id, worker.worker_id], )?; + if changed != 0 { + tx.execute( + "UPDATE worker_create_reservations + SET state = 'removed', updated_at = ?4 + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + AND state = 'created'", + params![ + workspace_id, + worker.runtime_id, + worker.worker_id, + chrono::Utc::now().to_rfc3339(), + ], + )?; + } tx.commit()?; Ok(changed > 0) }) @@ -6430,6 +6450,39 @@ fn migrate_workspace_deletion_v52_to_v53(conn: &Connection) -> Result<()> { let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; tx.execute_batch( r#" + CREATE TABLE worker_create_reservations_v53 ( + workspace_id TEXT NOT NULL, + allocation_key TEXT NOT NULL, + worker_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + create_fingerprint TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('reserved', 'created', 'removed')), + request_fingerprint TEXT, + memory_settings_revision INTEGER, + memory_language TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, allocation_key), + UNIQUE (workspace_id, worker_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + CHECK ( + (request_fingerprint IS NULL AND memory_settings_revision IS NULL AND memory_language IS NULL) + OR + (request_fingerprint IS NOT NULL AND memory_settings_revision IS NOT NULL AND memory_settings_revision > 0 AND memory_language IS NOT NULL AND length(trim(memory_language)) > 0) + ) + ); + INSERT INTO worker_create_reservations_v53 ( + workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, + state, request_fingerprint, memory_settings_revision, memory_language, + created_at, updated_at + ) + SELECT workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, + state, request_fingerprint, memory_settings_revision, memory_language, + created_at, updated_at + FROM worker_create_reservations; + DROP TABLE worker_create_reservations; + ALTER TABLE worker_create_reservations_v53 RENAME TO worker_create_reservations; + CREATE TABLE workspace_deletion_operations ( operation_id TEXT PRIMARY KEY, request_fingerprint TEXT NOT NULL, @@ -6491,6 +6544,16 @@ fn verify_workspace_deletion_schema(conn: &Connection) -> Result<()> { "workspace_deletion_operations schema does not match schema-53".to_string(), )); } + let reservation_sql: String = conn.query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'worker_create_reservations'", + [], + |row| row.get(0), + )?; + if !reservation_sql.contains("'removed'") { + return Err(Error::Store( + "worker_create_reservations schema lacks terminal removed state".to_string(), + )); + } Ok(()) } @@ -7217,6 +7280,38 @@ mod tests { r#" DROP INDEX workspace_deletion_operations_workspace_recent; DROP TABLE workspace_deletion_operations; + CREATE TABLE worker_create_reservations_v52 ( + workspace_id TEXT NOT NULL, + allocation_key TEXT NOT NULL, + worker_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + create_fingerprint TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('reserved', 'created')), + request_fingerprint TEXT, + memory_settings_revision INTEGER, + memory_language TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, allocation_key), + UNIQUE (workspace_id, worker_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + CHECK ( + (request_fingerprint IS NULL AND memory_settings_revision IS NULL AND memory_language IS NULL) + OR + (request_fingerprint IS NOT NULL AND memory_settings_revision IS NOT NULL AND memory_settings_revision > 0 AND memory_language IS NOT NULL AND length(trim(memory_language)) > 0) + ) + ); + INSERT INTO worker_create_reservations_v52 ( + workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, + state, request_fingerprint, memory_settings_revision, memory_language, + created_at, updated_at + ) + SELECT workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, + state, request_fingerprint, memory_settings_revision, memory_language, + created_at, updated_at + FROM worker_create_reservations; + DROP TABLE worker_create_reservations; + ALTER TABLE worker_create_reservations_v52 RENAME TO worker_create_reservations; DELETE FROM __yoi_schema_migrations; INSERT INTO __yoi_schema_migrations(version, name) VALUES (52, 'workspace schema baseline'); @@ -7230,12 +7325,30 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("server.db"); prepare_schema_v52(&path); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) + VALUES('owner','user','owner','Owner','1','1'); + INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) + VALUES('workspace-a','owner','Workspace A','active','1','1'); + INSERT INTO worker_create_reservations( + workspace_id,allocation_key,worker_id,runtime_id,create_fingerprint,state,created_at,updated_at + ) VALUES('workspace-a','allocation','worker-a','runtime-a','fingerprint','created','1','1');", + ) + .unwrap(); + drop(conn); let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); verify_workspace_deletion_schema(conn)?; + let reservation_state: String = conn.query_row( + "SELECT state FROM worker_create_reservations WHERE workspace_id='workspace-a' AND allocation_key='allocation'", + [], + |row| row.get(0), + )?; + assert_eq!(reservation_state, "created"); let violations: i64 = conn.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |row| { row.get(0) @@ -7269,6 +7382,14 @@ mod tests { vec!["unexpected".to_string()] ); assert_eq!(current_schema_version(&conn).unwrap(), 52); + let reservation_sql: String = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='worker_create_reservations'", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(!reservation_sql.contains("'removed'")); } #[test] @@ -7875,6 +7996,46 @@ mod tests { }) .unwrap(); assert_eq!(state, "created"); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) VALUES ('workspace-a', 'arcadia', ?1, 'Worker', '1', '1', 'normal')", + [reserved.worker_id.to_string()], + )?; + Ok(()) + }) + .unwrap(); + assert!( + store + .delete_worker_registry("workspace-a", &reserved_worker) + .unwrap() + ); + let removed_state: String = store + .with_conn(|conn| { + conn.query_row( + "SELECT state FROM worker_create_reservations + WHERE workspace_id = 'workspace-a' AND worker_id = ?1", + [reserved.worker_id.to_string()], + |row| row.get(0), + ) + .map_err(Error::from) + }) + .unwrap(); + assert_eq!(removed_state, "removed"); + assert!( + store + .reserve_worker_create( + "workspace-a", + "arcadia", + "operation-1", + "sha256:one", + &updated_memory_settings, + ) + .is_err() + ); store .with_conn(|conn| { diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index c28105b9..1382edcf 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -846,6 +846,7 @@ fn parse_deletion_state(value: &str) -> Result { #[cfg(test)] mod tests { use super::*; + use crate::store::ControlPlaneStore; use std::collections::BTreeSet; use tempfile::tempdir; @@ -1151,6 +1152,73 @@ mod tests { ); } + #[test] + fn worker_registry_removal_terminalizes_created_reservation() { + let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_create_reservations ( + workspace_id, allocation_key, worker_id, runtime_id, + create_fingerprint, state, created_at, updated_at + ) VALUES (?1, 'allocation', 'worker-created', 'runtime-a', + 'fingerprint', 'created', '1', '1')", + params![workspace_id], + )?; + conn.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) VALUES (?1, 'runtime-a', 'worker-created', 'Created worker', '1', '1', 'normal')", + params![workspace_id], + )?; + Ok(()) + }) + .expect("created worker fixture"); + let before = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight before removal"); + assert!( + !before + .blockers + .iter() + .any(|blocker| blocker.message.contains("Worker creation reservations")) + ); + + assert!( + store + .delete_worker_registry( + &workspace_id, + &worker_runtime::identity::RuntimeWorkerRef { + runtime_id: "runtime-a".to_string(), + worker_id: "worker-created".to_string(), + }, + ) + .expect("remove worker registry") + ); + let state: String = store + .with_conn(|conn| { + conn.query_row( + "SELECT state FROM worker_create_reservations + WHERE workspace_id = ?1 AND allocation_key = 'allocation'", + params![workspace_id], + |row| row.get(0), + ) + .map_err(Into::into) + }) + .expect("reservation state"); + assert_eq!(state, "removed"); + let after = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight after removal"); + assert!( + !after + .blockers + .iter() + .any(|blocker| blocker.message.contains("Worker creation reservations")) + ); + } + #[test] fn pinned_worker_blocks_preflight_before_operation_reservation() { let (store, owner, workspace_id) = setup(); From 0c48c5dee31877273c36e9b03a4f4306a1fa5ac1 Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 10:33:45 +0900 Subject: [PATCH 38/39] fix: fail closed without worker snapshots --- crates/worker-runtime/src/runtime.rs | 44 ++++++++++++++------------- crates/workspace-server/src/server.rs | 28 +++++++++++++++++ 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 76066336..e44e9f7f 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1732,27 +1732,9 @@ impl Runtime { return Ok(snapshot); } } - Ok(protocol::Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: worker_ref.worker_id.to_string(), - cwd: String::new(), - provider: "worker-runtime".to_string(), - model: "worker-runtime".to_string(), - scope_summary: "runtime worker observation".to_string(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - }, - state: protocol::WorkerStateSnapshot::initial(1), - in_flight: protocol::InFlightSnapshot { - blocks: Vec::new(), - commands: Vec::new(), - }, - internal_workers: Vec::new(), + Err(RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "authoritative Worker snapshot is unavailable".to_string(), }) } @@ -4974,6 +4956,26 @@ mod tests { } } + #[cfg(feature = "ws-server")] + #[test] + fn observation_snapshot_fails_closed_when_backend_snapshot_is_unavailable() { + let runtime = runtime_with_backend(); + let detail = runtime + .create_worker(task_request("snapshot unavailable")) + .unwrap(); + + assert!(matches!( + runtime + .worker_observation_snapshot(&detail.worker_ref) + .unwrap_err(), + RuntimeError::WorkerExecutionUnavailable { + worker_id, + message, + } if worker_id == detail.worker_ref.worker_id + && message == "authoritative Worker snapshot is unavailable" + )); + } + struct InputOnlyBackend; impl WorkerExecutionBackend for InputOnlyBackend { diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 9cbeb899..8b70d092 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -18865,6 +18865,34 @@ mod tests { ) } + fn worker_snapshot( + &self, + handle: &worker_runtime::execution::WorkerExecutionHandle, + ) -> Option { + Some(protocol::Event::Snapshot { + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: Vec::new(), + }, + greeting: protocol::Greeting { + worker_name: handle.worker_ref().worker_id.to_string(), + cwd: String::new(), + provider: "deterministic-workspace-server-test".to_string(), + model: "deterministic-workspace-server-test".to_string(), + scope_summary: "test execution snapshot".to_string(), + tools: Vec::new(), + context_window: 0, + context_tokens: 0, + }, + state: protocol::WorkerStateSnapshot::initial(1), + in_flight: protocol::InFlightSnapshot { + blocks: Vec::new(), + commands: Vec::new(), + }, + internal_workers: Vec::new(), + }) + } + fn stop_worker( &self, _handle: &worker_runtime::execution::WorkerExecutionHandle, From 88683a8d8fcbad75e9639914f0ab8888f7fd50eb Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 6 Sep 2026 10:41:20 +0900 Subject: [PATCH 39/39] test: provide authoritative websocket snapshot --- crates/worker-runtime/src/http_server.rs | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index c05bdb91..a35eaab0 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -3246,6 +3246,31 @@ mod ws_tests { } } + fn worker_snapshot(&self, handle: &WorkerExecutionHandle) -> Option { + Some(protocol::Event::Snapshot { + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: Vec::new(), + }, + greeting: protocol::Greeting { + worker_name: handle.worker_ref().worker_id.to_string(), + cwd: String::new(), + provider: "ws-test".to_string(), + model: "ws-test".to_string(), + scope_summary: "WebSocket test execution snapshot".to_string(), + tools: Vec::new(), + context_window: 0, + context_tokens: 0, + }, + state: protocol::WorkerStateSnapshot::initial(1), + in_flight: protocol::InFlightSnapshot { + blocks: Vec::new(), + commands: Vec::new(), + }, + internal_workers: Vec::new(), + }) + } + fn dispatch_method( &self, _handle: &WorkerExecutionHandle,