feat: persist worker submit activation queue

This commit is contained in:
2026-09-05 22:06:59 +09:00
parent aa96bbedbc
commit bb56283063
41 changed files with 2336 additions and 811 deletions
Generated
+1
View File
@@ -6650,6 +6650,7 @@ dependencies = [
"serial_test", "serial_test",
"session-metrics", "session-metrics",
"session-store", "session-store",
"sha2 0.11.0",
"tempfile", "tempfile",
"thiserror 2.0.18", "thiserror 2.0.18",
"ticket", "ticket",
+5 -2
View File
@@ -120,12 +120,15 @@ mod tests {
let mut client = Client::new(socket); let mut client = Client::new(socket);
client client
.send(&Method::run_text("hello")) .send(&Method::submit_text(
protocol::new_submission_request_id(),
"hello",
))
.await .await
.expect("send method"); .expect("send method");
assert!(matches!( assert!(matches!(
decode_method(&client.socket.sent[0]), decode_method(&client.socket.sent[0]),
Ok(Method::Run { .. }) Ok(Method::Submit { .. })
)); ));
assert!(matches!( assert!(matches!(
client.next_event().await, client.next_event().await,
+5 -2
View File
@@ -89,12 +89,15 @@ mod tests {
let mut client = Client::new(socket); let mut client = Client::new(socket);
client client
.send(&Method::run_text("hello")) .send(&Method::submit_text(
protocol::new_submission_request_id(),
"hello",
))
.await .await
.expect("send method"); .expect("send method");
assert!(matches!( assert!(matches!(
peer.next().await.as_deref().map(decode_method), peer.next().await.as_deref().map(decode_method),
Some(Ok(Method::Run { .. })) Some(Ok(Method::Submit { .. }))
)); ));
peer.send( peer.send(
+8 -2
View File
@@ -147,12 +147,18 @@ mod tests {
let mut client = Client::new(Socket::connect(&socket_path).await.unwrap()); let mut client = Client::new(Socket::connect(&socket_path).await.unwrap());
client client
.send(&Method::run_text("hello")) .send(&Method::submit_text(
protocol::new_submission_request_id(),
"hello",
))
.await .await
.expect("send method"); .expect("send method");
let received = server.await.unwrap().expect("method message"); let received = server.await.unwrap().expect("method message");
assert!(matches!(decode_method(&received), Ok(Method::Run { .. }))); assert!(matches!(
decode_method(&received),
Ok(Method::Submit { .. })
));
} }
#[tokio::test] #[tokio::test]
+5 -2
View File
@@ -114,7 +114,7 @@ mod tests {
assert!(matches!( assert!(matches!(
message, message,
Message::Text(ref text) Message::Text(ref text)
if matches!(decode_method(text), Ok(Method::Run { .. })) if matches!(decode_method(text), Ok(Method::Submit { .. }))
)); ));
let event = encode_event(&Event::Status { let event = encode_event(&Event::Status {
status: WorkerStatus::Idle, status: WorkerStatus::Idle,
@@ -126,7 +126,10 @@ mod tests {
let request = format!("ws://{address}").into_client_request().unwrap(); let request = format!("ws://{address}").into_client_request().unwrap();
let mut client = Client::new(Socket::connect(request).await.unwrap()); let mut client = Client::new(Socket::connect(request).await.unwrap());
client client
.send(&Method::run_text("hello")) .send(&Method::submit_text(
protocol::new_submission_request_id(),
"hello",
))
.await .await
.expect("send method"); .expect("send method");
assert!(matches!( assert!(matches!(
+124 -48
View File
@@ -11,6 +11,11 @@ use serde::{Deserialize, Serialize};
pub use identity::{WorkerId, WorkerIdParseError}; pub use identity::{WorkerId, WorkerIdParseError};
/// Allocate an opaque idempotency key for one client Submit request.
pub fn new_submission_request_id() -> String {
uuid::Uuid::now_v7().to_string()
}
fn default_true() -> bool { fn default_true() -> bool {
true true
} }
@@ -31,17 +36,22 @@ fn is_false(value: &bool) -> bool {
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(tag = "method", content = "params", rename_all = "snake_case")] #[serde(tag = "method", content = "params", rename_all = "snake_case")]
pub enum Method { pub enum Method {
Run { /// Durably accept typed input for immediate activation or the session FIFO.
///
/// `submission_request_id` is generated by the authenticated caller and is
/// used only for idempotent retry. Worker allocates the durable
/// `submission_id` returned by [`Event::SubmissionAccepted`].
Submit {
submission_request_id: String,
input: Vec<Segment>, input: Vec<Segment>,
}, },
/// Runtime-internal Run carrying an opaque correlation id that is committed /// Runtime-internal Submit with the same request identity contract. This
/// with the resulting UserInput entry. This variant is not serializable on /// variant is not serializable on the public Client → Worker protocol.
/// the public Client → Worker protocol.
#[serde(skip)] #[serde(skip)]
#[cfg_attr(feature = "typescript", ts(skip))] #[cfg_attr(feature = "typescript", ts(skip))]
RunTracked { SubmitTracked {
submission_request_id: String,
input: Vec<Segment>, input: Vec<Segment>,
submission_id: String,
}, },
/// Human-readable text injected into the target Worker's LLM context /// Human-readable text injected into the target Worker's LLM context
/// as a non-blocking system message. `auto_run` controls whether an /// as a non-blocking system message. `auto_run` controls whether an
@@ -50,12 +60,25 @@ pub enum Method {
/// No side effects beyond LLM context; use `WorkerEvent` for typed /// No side effects beyond LLM context; use `WorkerEvent` for typed
/// lifecycle reports. /// lifecycle reports.
Notify { Notify {
notification_request_id: String,
message: String, message: String,
#[serde(default = "default_true", skip_serializing_if = "is_true")] #[serde(default = "default_true", skip_serializing_if = "is_true")]
auto_run: bool, auto_run: bool,
}, },
/// Typed lifecycle report from a child Worker to its direct parent. /// Typed lifecycle report from a child Worker to its direct parent.
WorkerEvent(WorkerEvent), WorkerEvent(WorkerEvent),
/// Return the authoritative FIFO summary without exposing queued payloads.
ListPendingSubmissions,
/// Remove one queued submission. Running or already activated submissions
/// are immutable and therefore cannot be cancelled here.
CancelPendingSubmission {
submission_id: String,
},
/// Remove every queued submission while preserving the active run.
ClearPendingSubmissions,
/// Activate the next queued submission while the Worker is idle. This is an
/// explicit recovery operation and never resumes a paused run implicitly.
ContinuePending,
Resume, Resume,
Cancel, Cancel,
/// Stop the in-flight turn and transition to `Paused`. /// Stop the in-flight turn and transition to `Paused`.
@@ -68,7 +91,7 @@ pub enum Method {
/// Request an explicit compaction while the Worker is otherwise idle. /// Request an explicit compaction while the Worker is otherwise idle.
/// ///
/// This is a typed control method: clients must not send `compact` as a /// This is a typed control method: clients must not send `compact` as a
/// `Method::Run` user message. /// `Method::Submit` user message.
Compact, Compact,
/// Ask the Worker to list valid rewind targets from its authoritative session log. /// Ask the Worker to list valid rewind targets from its authoritative session log.
ListRewindTargets, ListRewindTargets,
@@ -181,7 +204,7 @@ impl WorkerEvent {
/// One typed piece of a user submission. /// One typed piece of a user submission.
/// ///
/// `Method::Run` and `Event::UserMessage` carry `Vec<Segment>`. Dumb /// `Method::Submit` and `Event::UserMessage` carry `Vec<Segment>`. Dumb
/// clients (CLI piping, scripts) only need to produce a single /// clients (CLI piping, scripts) only need to produce a single
/// `Segment::Text`; richer clients (TUI / GUI) construct typed atoms /// `Segment::Text`; richer clients (TUI / GUI) construct typed atoms
/// (paste chips, file refs) and /// (paste chips, file refs) and
@@ -404,12 +427,13 @@ impl Segment {
} }
impl Method { impl Method {
/// Convenience: a `Run` carrying a single `Segment::Text`. /// Convenience: a `Submit` carrying a single `Segment::Text`.
/// Used by dumb clients, inter-Worker tools, and tests that only have /// Used by dumb clients, inter-Worker tools, and tests that only have
/// a string to forward. /// a string to forward.
pub fn run_text(s: impl Into<String>) -> Self { pub fn submit_text(submission_request_id: impl Into<String>, text: impl Into<String>) -> Self {
Self::Run { Self::Submit {
input: vec![Segment::text(s)], submission_request_id: submission_request_id.into(),
input: vec![Segment::text(text)],
} }
} }
} }
@@ -503,6 +527,37 @@ pub enum ToolResultDisposition {
OutcomeUnknown, OutcomeUnknown,
} }
/// Durable acceptance result for one idempotent Submit request.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum SubmissionDisposition {
Started,
Queued,
}
/// Bounded public projection of one pending submission. Payload segments and
/// provenance remain in the session log and are intentionally not exposed.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
pub struct PendingSubmissionSummary {
pub submission_id: String,
pub accepted_at_ms: u64,
pub segment_count: u32,
pub byte_len: u64,
}
/// Revisioned session-owned FIFO projection used by snapshots and live events.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
pub struct PendingSubmissionsSnapshot {
pub revision: u64,
#[serde(default)]
pub notification_count: u32,
#[serde(default)]
pub submissions: Vec<PendingSubmissionSummary>,
}
/// Canonical, storage-independent projection of committed session history. /// Canonical, storage-independent projection of committed session history.
/// ///
/// Worker protocols expose this DTO instead of append-log records. New /// Worker protocols expose this DTO instead of append-log records. New
@@ -511,6 +566,8 @@ pub enum ToolResultDisposition {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
pub struct SessionSnapshot { pub struct SessionSnapshot {
#[serde(default)]
pub pending_submissions: PendingSubmissionsSnapshot,
pub entries: Vec<SessionSnapshotEntry>, pub entries: Vec<SessionSnapshotEntry>,
} }
@@ -609,16 +666,27 @@ pub struct SessionToolAttachment {
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(tag = "event", content = "data", rename_all = "snake_case")] #[serde(tag = "event", content = "data", rename_all = "snake_case")]
pub enum Event { pub enum Event {
/// A user input message was accepted, persisted as /// Durable Submit acceptance. A `Started` receipt follows the atomic
/// `LogEntry::AnnotatedUserInput`, and is about to start a new turn. /// UserInput commit; a `Queued` receipt follows the durable FIFO checkpoint.
/// Broadcast to every subscribed client so TUI / GUI instances show /// Repeating the same request id and exact payload returns the same receipt
/// the same user line that reconnect snapshots would replay from /// without appending or activating twice.
/// history; clients must not synthesize a separate pending/fake SubmissionAccepted {
/// message for accepted runs. submission_request_id: String,
/// submission_id: String,
/// Fires exactly once per committed user input, after disposition: SubmissionDisposition,
/// `InvokeStart { kind: UserSend }` and before the first },
/// `TurnStart`. Rejected runs (e.g. `AlreadyRunning`) do not emit. /// Correlated rejection before durable acceptance.
SubmissionRejected {
submission_request_id: String,
message: String,
},
/// Revisioned FIFO replacement following enqueue, activation, cancel, or clear.
PendingSubmissionsChanged {
pending: PendingSubmissionsSnapshot,
},
/// A user input message persisted as `LogEntry::AnnotatedUserInput` and
/// activated for a turn. Broadcast to every subscribed client so TUI / GUI
/// instances show the same user line that reconnect snapshots replay.
UserMessage { UserMessage {
segments: Vec<Segment>, segments: Vec<Segment>,
}, },
@@ -641,7 +709,7 @@ pub enum Event {
/// ///
/// Marker event for the start of an Invoke range; the range extends /// Marker event for the start of an Invoke range; the range extends
/// implicitly until the next `InvokeStart`. Fires for every accepted /// implicitly until the next `InvokeStart`. Fires for every accepted
/// `Method::Run` (kind=`UserSend`), `Method::Notify` (kind=`Notify`), /// `Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`),
/// `Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other /// `Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other
/// IDLE-breaking trigger. Mid-run interrupts (e.g. hook output, /// IDLE-breaking trigger. Mid-run interrupts (e.g. hook output,
/// typed system reminder insertion that doesn't break IDLE) do not /// typed system reminder insertion that doesn't break IDLE) do not
@@ -1193,7 +1261,7 @@ pub enum TurnResult {
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub enum InvokeKind { pub enum InvokeKind {
/// `Method::Run` — a user submission. /// `Method::Submit` — a user submission.
UserSend, UserSend,
/// `Method::Notify` — free-text notification injected into history. /// `Method::Notify` — free-text notification injected into history.
Notify, Notify,
@@ -1216,7 +1284,7 @@ pub enum RunResult {
Finished, Finished,
Paused, Paused,
LimitReached, LimitReached,
/// The accepted Method::Run produced no assistant/tool output before /// The accepted Method::Submit produced no assistant/tool output before
/// user interruption, so the Worker rolled the submit-time turn state back /// user interruption, so the Worker rolled the submit-time turn state back
/// to its pre-submit snapshot. Clients should treat the Worker as Idle and /// to its pre-submit snapshot. Clients should treat the Worker as Idle and
/// restore the just-submitted input into the editable composer if desired. /// restore the just-submitted input into the editable composer if desired.
@@ -1285,26 +1353,30 @@ mod tests {
use super::*; use super::*;
#[test] #[test]
fn method_run_json_roundtrip() { fn method_submit_json_roundtrip_and_run_is_rejected() {
let json = r#"{"method":"run","params":{"input":[{"kind":"text","content":"Hello"}]}}"#; let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#;
let method: Method = serde_json::from_str(json).unwrap(); let method: Method = serde_json::from_str(json).unwrap();
match &method { match &method {
Method::Run { input } => { Method::Submit { input, .. } => {
assert_eq!(input.len(), 1); assert_eq!(input.len(), 1);
match &input[0] { match &input[0] {
Segment::Text { content } => assert_eq!(content, "Hello"), Segment::Text { content } => assert_eq!(content, "Hello"),
other => panic!("expected Text, got {other:?}"), other => panic!("expected Text, got {other:?}"),
} }
} }
other => panic!("expected Run, got {other:?}"), other => panic!("expected Submit, got {other:?}"),
} }
let serialized = serde_json::to_string(&method).unwrap(); let serialized = serde_json::to_string(&method).unwrap();
assert_eq!(serialized, json); assert_eq!(serialized, json);
assert!(
serde_json::from_str::<Method>(r#"{"method":"run","params":{"input":[]}}"#).is_err()
);
} }
#[test] #[test]
fn method_run_paste_segment_roundtrip() { fn method_submit_paste_segment_roundtrip() {
let method = Method::Run { let method = Method::Submit {
submission_request_id: "request-1".to_string(),
input: vec![ input: vec![
Segment::text("see "), Segment::text("see "),
Segment::Paste { Segment::Paste {
@@ -1318,7 +1390,7 @@ mod tests {
let json = serde_json::to_string(&method).unwrap(); let json = serde_json::to_string(&method).unwrap();
let decoded: Method = serde_json::from_str(&json).unwrap(); let decoded: Method = serde_json::from_str(&json).unwrap();
match decoded { match decoded {
Method::Run { input } => { Method::Submit { input, .. } => {
assert_eq!(input.len(), 2); assert_eq!(input.len(), 2);
match &input[1] { match &input[1] {
Segment::Paste { Segment::Paste {
@@ -1335,7 +1407,7 @@ mod tests {
other => panic!("expected Paste, got {other:?}"), other => panic!("expected Paste, got {other:?}"),
} }
} }
other => panic!("expected Run, got {other:?}"), other => panic!("expected Submit, got {other:?}"),
} }
} }
@@ -1389,8 +1461,9 @@ mod tests {
} }
#[test] #[test]
fn method_run_flow_segment_roundtrip() { fn method_submit_flow_segment_roundtrip() {
let method = Method::Run { let method = Method::Submit {
submission_request_id: "request-1".to_string(),
input: vec![ input: vec![
Segment::Flow { Segment::Flow {
selector: "builtin:coder-review".to_string(), selector: "builtin:coder-review".to_string(),
@@ -1404,7 +1477,7 @@ mod tests {
let decoded = serde_json::from_str::<Method>(&json).unwrap(); let decoded = serde_json::from_str::<Method>(&json).unwrap();
assert!(matches!( assert!(matches!(
decoded, decoded,
Method::Run { input } Method::Submit { input, .. }
if matches!( if matches!(
input.as_slice(), input.as_slice(),
[ [
@@ -1416,15 +1489,15 @@ mod tests {
} }
#[test] #[test]
fn runtime_tracked_run_is_not_public_protocol_json() { fn runtime_tracked_submit_is_not_public_protocol_json() {
let method = Method::RunTracked { let method = Method::SubmitTracked {
input: vec![Segment::text("private")], input: vec![Segment::text("private")],
submission_id: "submission-1".to_string(), submission_request_id: "request-1".to_string(),
}; };
assert!(serde_json::to_string(&method).is_err()); assert!(serde_json::to_string(&method).is_err());
assert!( assert!(
serde_json::from_str::<Method>( serde_json::from_str::<Method>(
r#"{"method":"run_tracked","input":[],"submission_id":"forged"}"#, r#"{"method":"submit_tracked","input":[],"submission_id":"forged"}"#,
) )
.is_err() .is_err()
); );
@@ -1442,16 +1515,16 @@ mod tests {
} }
#[test] #[test]
fn method_run_with_unknown_segment_decodes() { fn method_submit_with_unknown_segment_decodes() {
let json = r#"{"method":"run","params":{"input":[{"kind":"text","content":"hi"},{"kind":"future_thing","x":1}]}}"#; let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"hi"},{"kind":"future_thing","x":1}]}}"#;
let method: Method = serde_json::from_str(json).unwrap(); let method: Method = serde_json::from_str(json).unwrap();
match method { match method {
Method::Run { input } => { Method::Submit { input, .. } => {
assert_eq!(input.len(), 2); assert_eq!(input.len(), 2);
assert!(matches!(input[0], Segment::Text { .. })); assert!(matches!(input[0], Segment::Text { .. }));
assert!(matches!(input[1], Segment::Unknown)); assert!(matches!(input[1], Segment::Unknown));
} }
other => panic!("expected Run, got {other:?}"), other => panic!("expected Submit, got {other:?}"),
} }
} }
@@ -1648,11 +1721,11 @@ mod tests {
#[test] #[test]
fn method_notify_json_roundtrip_defaults_to_auto_run() { fn method_notify_json_roundtrip_defaults_to_auto_run() {
let json = r#"{"method":"notify","params":{"message":"turn done"}}"#; let json = r#"{"method":"notify","params":{"notification_request_id":"notification-1","message":"turn done"}}"#;
let method: Method = serde_json::from_str(json).unwrap(); let method: Method = serde_json::from_str(json).unwrap();
assert!(matches!( assert!(matches!(
method, method,
Method::Notify { ref message, auto_run: true } if message == "turn done" Method::Notify { ref message, auto_run: true, .. } if message == "turn done"
)); ));
let serialized = serde_json::to_string(&method).unwrap(); let serialized = serde_json::to_string(&method).unwrap();
assert_eq!(serialized, json); assert_eq!(serialized, json);
@@ -1660,11 +1733,11 @@ mod tests {
#[test] #[test]
fn method_notify_weak_json_roundtrip_serializes_auto_run_false() { fn method_notify_weak_json_roundtrip_serializes_auto_run_false() {
let json = r#"{"method":"notify","params":{"message":"progress","auto_run":false}}"#; let json = r#"{"method":"notify","params":{"notification_request_id":"notification-1","message":"progress","auto_run":false}}"#;
let method: Method = serde_json::from_str(json).unwrap(); let method: Method = serde_json::from_str(json).unwrap();
assert!(matches!( assert!(matches!(
method, method,
Method::Notify { ref message, auto_run: false } if message == "progress" Method::Notify { ref message, auto_run: false, .. } if message == "progress"
)); ));
assert_eq!(serde_json::to_string(&method).unwrap(), json); assert_eq!(serde_json::to_string(&method).unwrap(), json);
} }
@@ -1725,6 +1798,7 @@ mod tests {
fn event_snapshot_format() { fn event_snapshot_format() {
let event = Event::Snapshot { let event = Event::Snapshot {
session: SessionSnapshot { session: SessionSnapshot {
pending_submissions: PendingSubmissionsSnapshot::default(),
entries: vec![SessionSnapshotEntry { entries: vec![SessionSnapshotEntry {
entry_id: "entry-1".into(), entry_id: "entry-1".into(),
timestamp: 1, timestamp: 1,
@@ -1776,6 +1850,7 @@ mod tests {
let event = Event::Snapshot { let event = Event::Snapshot {
session: SessionSnapshot { session: SessionSnapshot {
pending_submissions: PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: Greeting { greeting: Greeting {
@@ -1844,6 +1919,7 @@ mod tests {
fn event_segment_rotated_roundtrip() { fn event_segment_rotated_roundtrip() {
let event = Event::SegmentRotated { let event = Event::SegmentRotated {
session: SessionSnapshot { session: SessionSnapshot {
pending_submissions: PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
}; };
+8 -5
View File
@@ -8,11 +8,11 @@ use crate::{
CompletionKind, ErrorCode, Event, Greeting, InFlightBlock, InFlightSnapshot, CompletionKind, ErrorCode, Event, Greeting, InFlightBlock, InFlightSnapshot,
InFlightToolCallState, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnapshot,
InvokeKind, MemoryWorkerEvent, Method, PasteArtifactAvailability, PasteArtifactMediaType, InvokeKind, MemoryWorkerEvent, Method, PasteArtifactAvailability, PasteArtifactMediaType,
PasteArtifactRef, Permission, RewindSummary, RewindTarget, RewindTargetId, RunResult, PasteArtifactRef, PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission,
ScopeRule, Segment, SessionContentPart, SessionEntryProvenance, SessionMessageRole, RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart,
SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment, SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry,
ToolResultDisposition, TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition,
WorkerStatus, TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, WorkerStatus,
subscription::{ subscription::{
EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame, EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame,
SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest, SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest,
@@ -75,6 +75,9 @@ pub fn generated_protocol_types() -> String {
push_decl::<SessionToolAttachment>(&cfg, &mut output); push_decl::<SessionToolAttachment>(&cfg, &mut output);
push_decl::<SessionSnapshotEntryData>(&cfg, &mut output); push_decl::<SessionSnapshotEntryData>(&cfg, &mut output);
push_decl::<SessionSnapshotEntry>(&cfg, &mut output); push_decl::<SessionSnapshotEntry>(&cfg, &mut output);
push_decl::<PendingSubmissionSummary>(&cfg, &mut output);
push_decl::<PendingSubmissionsSnapshot>(&cfg, &mut output);
push_decl::<SubmissionDisposition>(&cfg, &mut output);
push_decl::<SessionSnapshot>(&cfg, &mut output); push_decl::<SessionSnapshot>(&cfg, &mut output);
push_decl::<InternalWorkerKind>(&cfg, &mut output); push_decl::<InternalWorkerKind>(&cfg, &mut output);
push_decl::<InternalWorkerRef>(&cfg, &mut output); push_decl::<InternalWorkerRef>(&cfg, &mut output);
@@ -183,6 +183,7 @@ fn canonicalize_history_entry(
item, item,
metadata: legacy_metadata(segment_id, line_index, 0), metadata: legacy_metadata(segment_id, line_index, 0),
}, },
extensions: Vec::new(),
}, },
} }
} }
+5 -2
View File
@@ -71,7 +71,7 @@ pub fn project_session_snapshot(session_id: SessionId, log: &[LogEntry]) -> Sess
entries.push(history_entry(entry, *ts, data)); entries.push(history_entry(entry, *ts, data));
} }
} }
LogEntry::AnnotatedSystemItem { ts, entry } => entries.push(system_entry( LogEntry::AnnotatedSystemItem { ts, entry, .. } => entries.push(system_entry(
&entry.item, &entry.item,
entry.metadata.entry_id.0.clone(), entry.metadata.entry_id.0.clone(),
*ts, *ts,
@@ -100,7 +100,10 @@ pub fn project_session_snapshot(session_id: SessionId, log: &[LogEntry]) -> Sess
} }
} }
SessionSnapshot { entries } SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries,
}
} }
fn extend_history( fn extend_history(
+1
View File
@@ -287,6 +287,7 @@ pub fn append_system_item(
LogEntry::AnnotatedSystemItem { LogEntry::AnnotatedSystemItem {
ts: segment_log::now_millis(), ts: segment_log::now_millis(),
entry, entry,
extensions: Vec::new(),
}, },
) )
} }
+10 -1
View File
@@ -112,6 +112,8 @@ pub enum LogEntry {
AnnotatedSystemItem { AnnotatedSystemItem {
ts: u64, ts: u64,
entry: LoggedSystemHistoryEntry, entry: LoggedSystemHistoryEntry,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
extensions: Vec<SessionExtension>,
}, },
/// Turn boundary. Records the turn count after increment. /// Turn boundary. Records the turn count after increment.
@@ -312,12 +314,19 @@ pub fn collect_state(entries: &[LogEntry]) -> RestoredState {
state.annotated_history.push(entry.clone()); state.annotated_history.push(entry.clone());
state.history.push(Item::from(entry.item.clone())); state.history.push(Item::from(entry.item.clone()));
} }
LogEntry::AnnotatedSystemItem { entry, .. } => { LogEntry::AnnotatedSystemItem {
entry, extensions, ..
} => {
state.annotated_history.push(LoggedHistoryEntry { state.annotated_history.push(LoggedHistoryEntry {
item: LoggedItem::from(entry.item.to_history_item()), item: LoggedItem::from(entry.item.to_history_item()),
metadata: entry.metadata.clone(), metadata: entry.metadata.clone(),
}); });
state.history.push(entry.item.to_history_item()); state.history.push(entry.item.to_history_item());
state.extensions.extend(
extensions
.iter()
.map(|extension| (extension.domain.clone(), extension.payload.clone())),
);
} }
LogEntry::TurnEnd { turn_count, .. } => { LogEntry::TurnEnd { turn_count, .. } => {
if let Some(active_turn_count) = &mut state.active_run_turn_count { if let Some(active_turn_count) = &mut state.active_run_turn_count {
+13 -3
View File
@@ -99,7 +99,10 @@ async fn in_process_host_runs_text_and_read_tool_then_shuts_down() {
let mut protocol_client = host.connect(); let mut protocol_client = host.connect();
protocol_client protocol_client
.send(&Method::run_text("read the probe")) .send(&Method::submit_text(
protocol::new_submission_request_id(),
"read the probe",
))
.await .await
.expect("submit input"); .expect("submit input");
@@ -336,11 +339,15 @@ async fn standalone_restore_preserves_history_tasks_notifications_and_cwd_scope(
let worker_id = host.worker_id(); let worker_id = host.worker_id();
let mut protocol_client = host.connect(); let mut protocol_client = host.connect();
protocol_client protocol_client
.send(&Method::run_text("first request")) .send(&Method::submit_text(
protocol::new_submission_request_id(),
"first request",
))
.await?; .await?;
wait_for_run_end(&mut protocol_client).await?; wait_for_run_end(&mut protocol_client).await?;
protocol_client protocol_client
.send(&Method::Notify { .send(&Method::Notify {
notification_request_id: protocol::new_submission_request_id(),
message: "persisted notification".to_string(), message: "persisted notification".to_string(),
auto_run: true, auto_run: true,
}) })
@@ -394,7 +401,10 @@ async fn standalone_restore_preserves_history_tasks_notifications_and_cwd_scope(
assert!(snapshot.contains("persisted notification"), "{snapshot}"); assert!(snapshot.contains("persisted notification"), "{snapshot}");
protocol_client protocol_client
.send(&Method::run_text("continue after restore")) .send(&Method::submit_text(
protocol::new_submission_request_id(),
"continue after restore",
))
.await?; .await?;
wait_for_run_end(&mut protocol_client).await?; wait_for_run_end(&mut protocol_client).await?;
let request = second_inspection let request = second_inspection
+81 -153
View File
@@ -102,23 +102,6 @@ struct RollbackSubmitState {
turn_before: usize, turn_before: usize,
} }
#[derive(Clone)]
pub struct QueuedInput {
segments: Vec<Segment>,
preview: String,
}
impl QueuedInput {
fn new(segments: Vec<Segment>) -> Self {
let preview = Segment::flatten_to_text(&segments);
Self { segments, preview }
}
pub fn preview(&self) -> &str {
&self.preview
}
}
struct ComposerInputHistory { struct ComposerInputHistory {
entries: VecDeque<Vec<Segment>>, entries: VecDeque<Vec<Segment>>,
browse: Option<ComposerInputHistoryBrowse>, browse: Option<ComposerInputHistoryBrowse>,
@@ -272,7 +255,7 @@ pub struct App {
/// Current transient actionbar notice. Notices are local UI state only: /// Current transient actionbar notice. Notices are local UI state only:
/// they are never appended to transcript/session history or LLM context. /// they are never appended to transcript/session history or LLM context.
actionbar_notice: Option<ActionbarNotice>, actionbar_notice: Option<ActionbarNotice>,
/// Normal composer input that is submitted as `Method::Run`. /// Normal composer input that is submitted as `Method::Submit`.
pub input: InputBuffer, pub input: InputBuffer,
/// Separate command-line input. It is never submitted as a user message. /// Separate command-line input. It is never submitted as a user message.
pub command_input: InputBuffer, pub command_input: InputBuffer,
@@ -333,9 +316,8 @@ pub struct App {
/// Top entry index of the task pane's visible window. Clamped on /// Top entry index of the task pane's visible window. Clamped on
/// render so it never points past the end of the list. /// render so it never points past the end of the list.
pub task_pane_scroll: usize, pub task_pane_scroll: usize,
/// TUI-local FIFO of user inputs submitted while the Worker is already running. /// Authoritative WorkerSession FIFO summary received from snapshot/live events.
/// Entries have not been sent to the Worker yet, so they remain editable/cancellable locally. pending_submissions: protocol::PendingSubmissionsSnapshot,
queued_inputs: VecDeque<QueuedInput>,
/// TUI-local readline-style composer input history. This is intentionally /// TUI-local readline-style composer input history. This is intentionally
/// client-side only: recalled entries are plain drafts until submitted again. /// client-side only: recalled entries are plain drafts until submitted again.
input_history: ComposerInputHistory, input_history: ComposerInputHistory,
@@ -395,7 +377,7 @@ impl App {
text_selection: TextSelectionState::default(), text_selection: TextSelectionState::default(),
task_pane_open: false, task_pane_open: false,
task_pane_scroll: 0, task_pane_scroll: 0,
queued_inputs: VecDeque::new(), pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
input_history: ComposerInputHistory::new(), input_history: ComposerInputHistory::new(),
input_history_store: None, input_history_store: None,
pending_submit_rollback: None, pending_submit_rollback: None,
@@ -768,18 +750,12 @@ impl App {
return None; return None;
} }
self.record_input_history(segments.clone()); self.record_input_history(segments.clone());
if self.running {
self.queued_inputs.push_back(QueuedInput::new(segments));
self.input.clear();
self.completion = None;
return None;
}
self.input.clear(); self.input.clear();
Some(self.method_for_run(segments)) Some(self.method_for_run(segments))
} }
pub fn restore_unsent_run(&mut self, method: &Method) { pub fn restore_unsent_run(&mut self, method: &Method) {
let Method::Run { input } = method else { let Method::Submit { input, .. } = method else {
return; return;
}; };
self.pending_submit_rollback = None; self.pending_submit_rollback = None;
@@ -787,8 +763,9 @@ impl App {
self.input.replace_with_segments(input); self.input.replace_with_segments(input);
self.completion = None; self.completion = None;
} else { } else {
self.queued_inputs self.push_error(
.push_front(QueuedInput::new(input.clone())); "Submit transport failed; current Composer was preserved and the unsent input was not queued.",
);
} }
} }
@@ -804,7 +781,10 @@ impl App {
block_start: self.blocks.len(), block_start: self.blocks.len(),
turn_before: self.turn_index, turn_before: self.turn_index,
}); });
Method::Run { input: segments } Method::Submit {
submission_request_id: protocol::new_submission_request_id(),
input: segments,
}
} }
fn record_input_history(&mut self, segments: Vec<Segment>) { fn record_input_history(&mut self, segments: Vec<Segment>) {
@@ -825,7 +805,7 @@ impl App {
} }
pub fn queued_input_count(&self) -> usize { pub fn queued_input_count(&self) -> usize {
self.queued_inputs.len() self.pending_submissions.submissions.len()
} }
#[cfg(test)] #[cfg(test)]
@@ -911,35 +891,10 @@ impl App {
} }
pub fn next_queued_input_preview(&self) -> Option<&str> { pub fn next_queued_input_preview(&self) -> Option<&str> {
self.queued_inputs.front().map(QueuedInput::preview) self.pending_submissions
} .submissions
.first()
pub fn clear_queued_inputs(&mut self) -> usize { .map(|submission| submission.submission_id.as_str())
let cleared = self.queued_inputs.len();
self.queued_inputs.clear();
cleared
}
pub fn restore_next_queued_input_to_composer(&mut self) -> bool {
if self.queued_inputs.is_empty() {
return false;
}
if !self.input.is_empty() {
self.push_error("Composer is not empty; clear it before editing queued input.");
return false;
}
let Some(queued) = self.queued_inputs.pop_front() else {
return false;
};
self.input_history.cancel_browse();
self.input.replace_with_segments(&queued.segments);
self.completion = None;
true
}
fn pop_next_queued_run(&mut self) -> Option<Method> {
let queued = self.queued_inputs.pop_front()?;
Some(self.method_for_run(queued.segments))
} }
pub fn clear_actionbar_notice(&mut self) { pub fn clear_actionbar_notice(&mut self) {
@@ -1123,6 +1078,11 @@ impl App {
} }
match event { match event {
Event::SubmissionAccepted { .. } => {}
Event::SubmissionRejected { message, .. } => self.push_error(message),
Event::PendingSubmissionsChanged { pending } => {
self.pending_submissions = pending;
}
Event::UserMessage { segments } => { Event::UserMessage { segments } => {
self.turn_index += 1; self.turn_index += 1;
self.blocks.push(Block::TurnHeader { self.blocks.push(Block::TurnHeader {
@@ -1372,9 +1332,6 @@ impl App {
WorkerStatus::Idle WorkerStatus::Idle
} }
}); });
if matches!(result, RunResult::Finished | RunResult::LimitReached) {
return self.pop_next_queued_run();
}
} }
} }
Event::CompactStart { .. } => { Event::CompactStart { .. } => {
@@ -1449,6 +1406,7 @@ impl App {
internal_workers, internal_workers,
} => { } => {
self.rewind_refresh_fence = false; self.rewind_refresh_fence = false;
self.pending_submissions = session.pending_submissions.clone();
self.restore_snapshot(&session, greeting, in_flight); self.restore_snapshot(&session, greeting, in_flight);
self.replace_internal_worker_snapshots(internal_workers); self.replace_internal_worker_snapshots(internal_workers);
self.set_worker_status(status); self.set_worker_status(status);
@@ -2681,7 +2639,10 @@ mod rewind_refresh_tests {
}); });
app.handle_worker_event(Event::RewindApplied { app.handle_worker_event(Event::RewindApplied {
session: protocol::SessionSnapshot { entries: vec![] }, session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![],
},
input: vec![Segment::text("selected rewind input")], input: vec![Segment::text("selected rewind input")],
summary: summary(3), summary: summary(3),
}); });
@@ -2700,7 +2661,10 @@ mod rewind_refresh_tests {
}); });
app.handle_worker_event(Event::RewindApplied { app.handle_worker_event(Event::RewindApplied {
session: protocol::SessionSnapshot { entries: vec![] }, session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![],
},
input: vec![Segment::text("rewound input")], input: vec![Segment::text("rewound input")],
summary: summary(1), summary: summary(1),
}); });
@@ -2743,7 +2707,10 @@ mod rewind_refresh_tests {
}); });
app.handle_worker_event(Event::RewindApplied { app.handle_worker_event(Event::RewindApplied {
session: protocol::SessionSnapshot { entries: vec![] }, session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![],
},
input: vec![Segment::text("rewound input")], input: vec![Segment::text("rewound input")],
summary: summary(2), summary: summary(2),
}); });
@@ -2877,7 +2844,7 @@ mod composer_history_persistence_tests {
path: "src/lib.rs".into(), path: "src/lib.rs".into(),
}, },
]); ]);
assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
let mut reloaded = App::new_with_input_history_store("test".into(), store); let mut reloaded = App::new_with_input_history_store("test".into(), store);
assert!(reloaded.browse_input_history_older()); assert!(reloaded.browse_input_history_older());
@@ -2958,7 +2925,7 @@ mod composer_history_persistence_tests {
app.insert_char(c); app.insert_char(c);
} }
match app.submit_input() { match app.submit_input() {
Some(Method::Run { input }) => input, Some(Method::Submit { input, .. }) => input,
other => panic!("expected Run, got {other:?}"), other => panic!("expected Run, got {other:?}"),
} }
} }
@@ -3424,72 +3391,43 @@ mod completion_flow_tests {
} }
#[test] #[test]
fn running_submit_is_queued_locally_and_clears_composer() { fn running_submit_is_sent_to_the_worker_and_not_queued_locally() {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
app.set_worker_status(WorkerStatus::Running); app.set_worker_status(WorkerStatus::Running);
insert_text(&mut app, "queued turn"); insert_text(&mut app, "queued turn");
assert!(app.submit_input().is_none()); let method = app.submit_input();
assert_eq!(app.queued_input_count(), 1); assert!(matches!(method, Some(Method::Submit { .. })));
assert_eq!(app.next_queued_input_preview(), Some("queued turn")); assert_eq!(app.queued_input_count(), 0);
assert_eq!(input_text(&app), ""); assert_eq!(input_text(&app), "");
} }
#[test] #[test]
fn finished_run_auto_sends_next_queued_input() { fn pending_submission_projection_is_worker_authoritative() {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
app.set_worker_status(WorkerStatus::Running); app.handle_worker_event(Event::PendingSubmissionsChanged {
insert_text(&mut app, "next turn"); pending: protocol::PendingSubmissionsSnapshot {
assert!(app.submit_input().is_none()); revision: 3,
notification_count: 0,
let method = app.handle_worker_event(Event::RunEnd { submissions: vec![protocol::PendingSubmissionSummary {
result: RunResult::Finished, submission_id: "submission-1".into(),
accepted_at_ms: 7,
segment_count: 2,
byte_len: 42,
}],
},
}); });
match method {
Some(Method::Run { input }) => {
assert_eq!(Segment::flatten_to_text(&input), "next turn");
}
other => panic!("expected queued Run, got {other:?}"),
}
assert_eq!(app.queued_input_count(), 0);
}
#[test]
fn limit_reached_run_auto_sends_next_queued_input() {
let mut app = App::new("test".into());
app.set_worker_status(WorkerStatus::Running);
insert_text(&mut app, "next after limit");
assert!(app.submit_input().is_none());
let method = app.handle_worker_event(Event::RunEnd {
result: RunResult::LimitReached,
});
match method {
Some(Method::Run { input }) => {
assert_eq!(Segment::flatten_to_text(&input), "next after limit");
}
other => panic!("expected queued Run, got {other:?}"),
}
assert_eq!(app.queued_input_count(), 0);
}
#[test]
fn paused_and_rolled_back_run_do_not_auto_send_queue() {
for result in [RunResult::Paused, RunResult::RolledBack] {
let mut app = App::new("test".into());
app.set_worker_status(WorkerStatus::Running);
insert_text(&mut app, "held turn");
assert!(app.submit_input().is_none());
let method = app.handle_worker_event(Event::RunEnd { result });
assert!(method.is_none());
assert_eq!(app.queued_input_count(), 1); assert_eq!(app.queued_input_count(), 1);
assert_eq!(app.next_queued_input_preview(), Some("held turn")); assert_eq!(app.next_queued_input_preview(), Some("submission-1"));
} assert!(
app.handle_worker_event(Event::RunEnd {
result: RunResult::Finished,
})
.is_none()
);
assert_eq!(app.queued_input_count(), 1);
} }
#[test] #[test]
@@ -3501,24 +3439,6 @@ mod completion_flow_tests {
assert_eq!(app.queued_input_count(), 0); assert_eq!(app.queued_input_count(), 0);
} }
#[test]
fn queued_input_can_be_restored_to_composer_or_cleared() {
let mut app = App::new("test".into());
app.set_worker_status(WorkerStatus::Running);
insert_text(&mut app, "edit me");
assert!(app.submit_input().is_none());
assert!(app.restore_next_queued_input_to_composer());
assert_eq!(app.queued_input_count(), 0);
assert_eq!(input_text(&app), "edit me");
app.input.clear();
insert_text(&mut app, "clear me");
assert!(app.submit_input().is_none());
assert_eq!(app.clear_queued_inputs(), 1);
assert_eq!(app.queued_input_count(), 0);
}
fn insert_text(app: &mut App, text: &str) { fn insert_text(app: &mut App, text: &str) {
for c in text.chars() { for c in text.chars() {
app.insert_char(c); app.insert_char(c);
@@ -3530,7 +3450,7 @@ mod completion_flow_tests {
app.insert_char(c); app.insert_char(c);
} }
match app.submit_input() { match app.submit_input() {
Some(Method::Run { input }) => input, Some(Method::Submit { input, .. }) => input,
other => panic!("expected Run, got {other:?}"), other => panic!("expected Run, got {other:?}"),
} }
} }
@@ -3675,6 +3595,7 @@ mod completion_flow_tests {
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
greeting: test_greeting(), greeting: test_greeting(),
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
status: WorkerStatus::Running, status: WorkerStatus::Running,
@@ -3783,6 +3704,7 @@ mod completion_flow_tests {
revision, revision,
status: WorkerStatus::Idle, status: WorkerStatus::Idle,
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
in_flight: protocol::InFlightSnapshot::default(), in_flight: protocol::InFlightSnapshot::default(),
@@ -4000,6 +3922,7 @@ mod completion_flow_tests {
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
greeting: test_greeting(), greeting: test_greeting(),
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
status: WorkerStatus::Idle, status: WorkerStatus::Idle,
@@ -4051,6 +3974,7 @@ mod completion_flow_tests {
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
greeting: test_greeting(), greeting: test_greeting(),
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
status: WorkerStatus::Idle, status: WorkerStatus::Idle,
@@ -4064,6 +3988,7 @@ mod completion_flow_tests {
}, },
revision: 4, revision: 4,
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
status: WorkerStatus::Running, status: WorkerStatus::Running,
@@ -4222,6 +4147,7 @@ mod completion_flow_tests {
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting, greeting,
@@ -4437,23 +4363,23 @@ mod completion_flow_tests {
} }
#[test] #[test]
fn input_history_records_queued_inputs_and_suppresses_consecutive_duplicates() { fn input_history_records_running_submits_and_suppresses_consecutive_duplicates() {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
app.running = true; app.running = true;
for c in "repeat".chars() { for c in "repeat".chars() {
app.insert_char(c); app.insert_char(c);
} }
assert!(app.submit_input().is_none()); assert!(app.submit_input().is_some());
assert_eq!(app.input_history_len(), 1); assert_eq!(app.input_history_len(), 1);
assert_eq!(app.queued_input_count(), 1); assert_eq!(app.queued_input_count(), 0);
for c in "repeat".chars() { for c in "repeat".chars() {
app.insert_char(c); app.insert_char(c);
} }
assert!(app.submit_input().is_none()); assert!(app.submit_input().is_some());
assert_eq!(app.input_history_len(), 1); assert_eq!(app.input_history_len(), 1);
assert_eq!(app.queued_input_count(), 2); assert_eq!(app.queued_input_count(), 0);
app.insert_char(' '); app.insert_char(' ');
assert!(app.submit_input().is_none()); assert!(app.submit_input().is_none());
@@ -4481,7 +4407,7 @@ mod completion_flow_tests {
}, },
]; ];
app.input.replace_with_segments(&original); app.input.replace_with_segments(&original);
assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
assert!(app.browse_input_history_older()); assert!(app.browse_input_history_older());
assert_eq!(app.input.submit_segments(), original); assert_eq!(app.input.submit_segments(), original);
@@ -4493,7 +4419,7 @@ mod completion_flow_tests {
for c in "sent".chars() { for c in "sent".chars() {
app.insert_char(c); app.insert_char(c);
} }
assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
for c in "draft".chars() { for c in "draft".chars() {
app.insert_char(c); app.insert_char(c);
@@ -4511,7 +4437,7 @@ mod completion_flow_tests {
for c in "sent".chars() { for c in "sent".chars() {
app.insert_char(c); app.insert_char(c);
} }
assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
assert!(app.browse_input_history_older()); assert!(app.browse_input_history_older());
assert!(app.input_history_is_browsing()); assert!(app.input_history_is_browsing());
@@ -4528,17 +4454,19 @@ mod completion_flow_tests {
for c in "first".chars() { for c in "first".chars() {
app.insert_char(c); app.insert_char(c);
} }
assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
for c in "second".chars() { for c in "second".chars() {
app.insert_char(c); app.insert_char(c);
} }
assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
assert!(app.browse_input_history_older()); assert!(app.browse_input_history_older());
assert!(app.browse_input_history_older()); assert!(app.browse_input_history_older());
let method = app.submit_input(); let method = app.submit_input();
match method { match method {
Some(Method::Run { input }) => assert_eq!(Segment::flatten_to_text(&input), "first"), Some(Method::Submit { input, .. }) => {
assert_eq!(Segment::flatten_to_text(&input), "first")
}
other => panic!("expected recalled run, got {other:?}"), other => panic!("expected recalled run, got {other:?}"),
} }
assert_eq!(app.input_history_len(), 3); assert_eq!(app.input_history_len(), 3);
+91 -107
View File
@@ -270,8 +270,8 @@ impl<T: Socket> ConsoleConnection<T> {
async fn send(&mut self, method: &Method) -> Result<(), Box<dyn std::error::Error>> { async fn send(&mut self, method: &Method) -> Result<(), Box<dyn std::error::Error>> {
let mut prepared = method.clone(); let mut prepared = method.clone();
let carries_attachments = let carries_attachments =
matches!(prepared, Method::Run { .. }) && !self.pending_attachments.is_empty(); matches!(prepared, Method::Submit { .. }) && !self.pending_attachments.is_empty();
if let Method::Run { input } = &mut prepared { if let Method::Submit { input, .. } = &mut prepared {
input.extend( input.extend(
self.pending_attachments self.pending_attachments
.iter() .iter()
@@ -569,6 +569,7 @@ async fn run_e2e_rewind_fixture(
app.connected = true; app.connected = true;
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
status: WorkerStatus::Idle, status: WorkerStatus::Idle,
@@ -697,6 +698,7 @@ async fn run_e2e_rewind_fixture(
if submitted_at.elapsed() >= apply_delay { if submitted_at.elapsed() >= apply_delay {
app.handle_worker_event(Event::RewindApplied { app.handle_worker_event(Event::RewindApplied {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
input: vec![Segment::text("rewind-live-refresh")], input: vec![Segment::text("rewind-live-refresh")],
@@ -916,7 +918,7 @@ async fn run_loop<T: Socket>(
} }
fn attachment_command_path(method: &Method) -> Option<PathBuf> { fn attachment_command_path(method: &Method) -> Option<PathBuf> {
let Method::Run { input } = method else { let Method::Submit { input, .. } = method else {
return None; return None;
}; };
let [Segment::Text { content }] = input.as_slice() else { let [Segment::Text { content }] = input.as_slice() else {
@@ -927,7 +929,7 @@ fn attachment_command_path(method: &Method) -> Option<PathBuf> {
} }
fn is_clear_attachments_command(method: &Method) -> bool { fn is_clear_attachments_command(method: &Method) -> bool {
let Method::Run { input } = method else { let Method::Submit { input, .. } = method else {
return false; return false;
}; };
matches!( matches!(
@@ -941,7 +943,7 @@ async fn send_console_method<T: Socket>(
client: &mut ConsoleConnection<T>, client: &mut ConsoleConnection<T>,
method: &Method, method: &Method,
) -> Result<(), Box<dyn std::error::Error>> { ) -> Result<(), Box<dyn std::error::Error>> {
if matches!(method, Method::Run { .. }) && client.has_active_uploads() { if matches!(method, Method::Submit { .. }) && client.has_active_uploads() {
app.restore_unsent_run(method); app.restore_unsent_run(method);
app.flash_actionbar_notice( app.flash_actionbar_notice(
"Attachment upload is still in progress; wait or use /clear-attachments.", "Attachment upload is still in progress; wait or use /clear-attachments.",
@@ -953,7 +955,7 @@ async fn send_console_method<T: Socket>(
} }
let sends_attachments = let sends_attachments =
matches!(method, Method::Run { .. }) && !client.pending_attachments.is_empty(); matches!(method, Method::Submit { .. }) && !client.pending_attachments.is_empty();
if let Err(error) = client.send(method).await { if let Err(error) = client.send(method).await {
if sends_attachments { if sends_attachments {
app.restore_unsent_run(method); app.restore_unsent_run(method);
@@ -1151,15 +1153,10 @@ fn handle_key(app: &mut App, key: KeyEvent) -> Option<Method> {
KeyCode::Char(c) KeyCode::Char(c)
if c.eq_ignore_ascii_case(&'q') && alt && !ctrl && !app.is_command_mode() => if c.eq_ignore_ascii_case(&'q') && alt && !ctrl && !app.is_command_mode() =>
{ {
if app.restore_next_queued_input_to_composer() { Some(Some(Method::ContinuePending))
Some(app.refresh_completion())
} else {
Some(None)
}
} }
KeyCode::Char(c) if c.eq_ignore_ascii_case(&'c') && alt && !ctrl => { KeyCode::Char(c) if c.eq_ignore_ascii_case(&'c') && alt && !ctrl => {
app.clear_queued_inputs(); Some(Some(Method::ClearPendingSubmissions))
Some(None)
} }
KeyCode::Char('c') if ctrl => Some(handle_pause_or_quit(app)), KeyCode::Char('c') if ctrl => Some(handle_pause_or_quit(app)),
KeyCode::Char('x') if ctrl => Some(handle_cancel_or_shutdown(app)), KeyCode::Char('x') if ctrl => Some(handle_cancel_or_shutdown(app)),
@@ -1427,7 +1424,6 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
WorkerStatus::Running | WorkerStatus::Paused WorkerStatus::Running | WorkerStatus::Paused
) { ) {
app.shutdown_confirm = None; app.shutdown_confirm = None;
app.clear_queued_inputs();
return Some(Method::Cancel); return Some(Method::Cancel);
} }
if let Some(pressed_at) = app.shutdown_confirm if let Some(pressed_at) = app.shutdown_confirm
@@ -1450,7 +1446,6 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
/// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running). /// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running).
fn handle_pause_or_quit(app: &mut App) -> Option<Method> { fn handle_pause_or_quit(app: &mut App) -> Option<Method> {
if app.worker_status == WorkerStatus::Running { if app.worker_status == WorkerStatus::Running {
app.clear_queued_inputs();
return Some(Method::Pause); return Some(Method::Pause);
} }
if let Some(t) = app.quit_confirm if let Some(t) = app.quit_confirm
@@ -1476,8 +1471,8 @@ mod tests {
use crate::text_selection::{HistoryViewport, SelectionRow}; use crate::text_selection::{HistoryViewport, SelectionRow};
use async_trait::async_trait; use async_trait::async_trait;
use protocol::{ use protocol::{
Event, RewindTarget, RewindTargetId, RunResult, Segment, UploadedFileAvailability, Event, RewindTarget, RewindTargetId, Segment, UploadedFileAvailability, UploadedFileRef,
UploadedFileRef, WorkerStatus, WorkerStatus,
}; };
#[test] #[test]
@@ -1490,7 +1485,8 @@ mod tests {
#[test] #[test]
fn client_local_attachment_commands_are_typed_and_do_not_send_the_path() { fn client_local_attachment_commands_are_typed_and_do_not_send_the_path() {
let attach = Method::Run { let attach = Method::Submit {
submission_request_id: protocol::new_submission_request_id(),
input: vec![Segment::text("/attach /tmp/report.md")], input: vec![Segment::text("/attach /tmp/report.md")],
}; };
assert_eq!( assert_eq!(
@@ -1499,7 +1495,8 @@ mod tests {
); );
assert!(!is_clear_attachments_command(&attach)); assert!(!is_clear_attachments_command(&attach));
let clear = Method::Run { let clear = Method::Submit {
submission_request_id: protocol::new_submission_request_id(),
input: vec![Segment::text("/clear-attachments")], input: vec![Segment::text("/clear-attachments")],
}; };
assert!(is_clear_attachments_command(&clear)); assert!(is_clear_attachments_command(&clear));
@@ -1605,7 +1602,7 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn queued_attachment_send_failure_restores_draft_without_exiting_console() { async fn running_attachment_submit_failure_restores_draft_without_exiting_console() {
let file = UploadedFileRef { let file = UploadedFileRef {
artifact_id: "artifact-queued".into(), artifact_id: "artifact-queued".into(),
file_name: "queued.txt".into(), file_name: "queued.txt".into(),
@@ -1631,13 +1628,10 @@ mod tests {
let mut app = App::new("worker".into()); let mut app = App::new("worker".into());
app.set_worker_status(WorkerStatus::Running); app.set_worker_status(WorkerStatus::Running);
app.input.insert_str("queued inspect"); app.input.insert_str("queued inspect");
assert!(app.submit_input().is_none());
let method = app let method = app
.handle_worker_event(Event::RunEnd { .submit_input()
result: RunResult::Finished, .expect("running Submit is sent immediately");
})
.expect("queued run must be released");
send_console_method(&mut app, &mut connection, &method) send_console_method(&mut app, &mut connection, &method)
.await .await
.unwrap(); .unwrap();
@@ -1960,7 +1954,7 @@ mod tests {
} }
#[test] #[test]
fn running_enter_queues_instead_of_sending_run() { fn running_enter_sends_submit_to_worker() {
let mut app = App::new("agent".to_string()); let mut app = App::new("agent".to_string());
app.set_worker_status(WorkerStatus::Running); app.set_worker_status(WorkerStatus::Running);
for c in "queued".chars() { for c in "queued".chars() {
@@ -1973,102 +1967,80 @@ mod tests {
); );
} }
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); assert!(matches!(
handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)),
Some(Method::Submit { .. })
));
assert_eq!(app.queued_input_count(), 1); assert_eq!(app.queued_input_count(), 0);
assert_eq!(app.next_queued_input_preview(), Some("queued"));
assert_eq!(input_text(&app), ""); assert_eq!(input_text(&app), "");
} }
#[test] #[test]
fn queued_input_keybindings_restore_and_clear() { fn pending_queue_shortcuts_send_worker_operations() {
let mut app = App::new("agent".to_string()); let mut app = App::new("test".into());
app.set_worker_status(WorkerStatus::Running); app.handle_worker_event(Event::PendingSubmissionsChanged {
for c in "edit queued".chars() { pending: protocol::PendingSubmissionsSnapshot {
assert!( revision: 2,
handle_key( notification_count: 0,
&mut app, submissions: vec![protocol::PendingSubmissionSummary {
KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE) submission_id: "submission-1".into(),
) accepted_at_ms: 1,
.is_none() segment_count: 1,
); byte_len: 6,
} }],
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); },
});
assert!( let continue_next = handle_key(
handle_key(
&mut app, &mut app,
KeyEvent::new(KeyCode::Char('q'), KeyModifiers::ALT) KeyEvent::new(KeyCode::Char('q'), KeyModifiers::ALT),
)
.is_none()
); );
assert_eq!(app.queued_input_count(), 0); assert!(matches!(continue_next, Some(Method::ContinuePending)));
assert_eq!(input_text(&app), "edit queued");
app.input.clear();
for c in "clear queued".chars() {
assert!(
handle_key(
&mut app,
KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)
)
.is_none()
);
}
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none());
assert_eq!(app.queued_input_count(), 1); assert_eq!(app.queued_input_count(), 1);
assert!( let clear = handle_key(
handle_key(
&mut app, &mut app,
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::ALT) KeyEvent::new(KeyCode::Char('c'), KeyModifiers::ALT),
)
.is_none()
); );
assert_eq!(app.queued_input_count(), 0); assert!(matches!(clear, Some(Method::ClearPendingSubmissions)));
assert_eq!(app.queued_input_count(), 1);
} }
#[test] #[test]
fn pause_and_cancel_clear_queued_input() { fn pause_and_cancel_preserve_authoritative_pending_queue() {
let mut app = App::new("agent".to_string()); let mut app = App::new("test".into());
app.handle_worker_event(Event::PendingSubmissionsChanged {
pending: protocol::PendingSubmissionsSnapshot {
revision: 2,
notification_count: 0,
submissions: vec![protocol::PendingSubmissionSummary {
submission_id: "submission-1".into(),
accepted_at_ms: 1,
segment_count: 1,
byte_len: 6,
}],
},
});
app.set_worker_status(WorkerStatus::Running); app.set_worker_status(WorkerStatus::Running);
for c in "queued".chars() { assert!(matches!(
assert!(
handle_key( handle_key(
&mut app,
KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)
)
.is_none()
);
}
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none());
assert_eq!(app.queued_input_count(), 1);
let pause = handle_key(
&mut app, &mut app,
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL), KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL),
); ),
assert!(matches!(pause, Some(Method::Pause))); Some(Method::Pause)
assert_eq!(app.queued_input_count(), 0); ));
for c in "queued again".chars() {
assert!(
handle_key(
&mut app,
KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)
)
.is_none()
);
}
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none());
assert_eq!(app.queued_input_count(), 1); assert_eq!(app.queued_input_count(), 1);
let cancel = handle_key( app.set_worker_status(WorkerStatus::Running);
assert!(matches!(
handle_key(
&mut app, &mut app,
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
); ),
assert!(matches!(cancel, Some(Method::Cancel))); Some(Method::Cancel)
assert_eq!(app.queued_input_count(), 0); ));
assert_eq!(app.queued_input_count(), 1);
} }
#[test] #[test]
@@ -2535,13 +2507,19 @@ mod tests {
let mut app = App::new("agent".to_string()); let mut app = App::new("agent".to_string());
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
greeting: test_greeting(), greeting: test_greeting(),
session: protocol::SessionSnapshot { entries: vec![] }, session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![],
},
status: WorkerStatus::Idle, status: WorkerStatus::Idle,
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}); });
app.handle_worker_event(Event::RewindApplied { app.handle_worker_event(Event::RewindApplied {
session: protocol::SessionSnapshot { entries: vec![] }, session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![],
},
input: vec![Segment::Text { input: vec![Segment::Text {
content: "retry this".into(), content: "retry this".into(),
}], }],
@@ -2562,7 +2540,10 @@ mod tests {
let mut app = App::new("agent".to_string()); let mut app = App::new("agent".to_string());
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
greeting: test_greeting(), greeting: test_greeting(),
session: protocol::SessionSnapshot { entries: vec![] }, session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![],
},
status: WorkerStatus::Idle, status: WorkerStatus::Idle,
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
@@ -2570,7 +2551,10 @@ mod tests {
type_keys(&mut app, "draft"); type_keys(&mut app, "draft");
app.handle_worker_event(Event::RewindApplied { app.handle_worker_event(Event::RewindApplied {
session: protocol::SessionSnapshot { entries: vec![] }, session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![],
},
input: vec![Segment::Text { input: vec![Segment::Text {
content: "retry this".into(), content: "retry this".into(),
}], }],
@@ -2918,12 +2902,12 @@ mod tests {
type_keys(&mut app, "first"); type_keys(&mut app, "first");
assert!(matches!( assert!(matches!(
handle_key(&mut app, key(KeyCode::Enter)), handle_key(&mut app, key(KeyCode::Enter)),
Some(Method::Run { .. }) Some(Method::Submit { .. })
)); ));
type_keys(&mut app, "second"); type_keys(&mut app, "second");
assert!(matches!( assert!(matches!(
handle_key(&mut app, key(KeyCode::Enter)), handle_key(&mut app, key(KeyCode::Enter)),
Some(Method::Run { .. }) Some(Method::Submit { .. })
)); ));
assert_eq!(input_text(&app), ""); assert_eq!(input_text(&app), "");
@@ -2954,7 +2938,7 @@ mod tests {
type_keys(&mut app, "sent"); type_keys(&mut app, "sent");
assert!(matches!( assert!(matches!(
handle_key(&mut app, key(KeyCode::Enter)), handle_key(&mut app, key(KeyCode::Enter)),
Some(Method::Run { .. }) Some(Method::Submit { .. })
)); ));
type_keys(&mut app, "draft\nbody"); type_keys(&mut app, "draft\nbody");
app.move_cursor_start(); app.move_cursor_start();
+21 -14
View File
@@ -1880,7 +1880,7 @@ fn actionbar_left_item(app: &App, now: Instant) -> Option<(String, Style)> {
} }
if app.queued_input_count() > 0 { if app.queued_input_count() > 0 {
return Some(( return Some((
"Alt-q edit queued Alt-c clear queued".to_string(), "Alt-q continue queued Alt-c clear queued".to_string(),
Style::default().fg(Color::DarkGray), Style::default().fg(Color::DarkGray),
)); ));
} }
@@ -2136,9 +2136,24 @@ mod tests {
use super::*; use super::*;
use crate::app::{ActionbarNoticeLevel, ActionbarNoticeSource, App}; use crate::app::{ActionbarNoticeLevel, ActionbarNoticeSource, App};
use crate::block::{ToolCallBlock, ToolCallState}; use crate::block::{ToolCallBlock, ToolCallState};
use protocol::WorkerStatus; use protocol::Event;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
fn set_pending_submission(app: &mut App, id: &str) {
app.handle_worker_event(Event::PendingSubmissionsChanged {
pending: protocol::PendingSubmissionsSnapshot {
revision: 1,
notification_count: 0,
submissions: vec![protocol::PendingSubmissionSummary {
submission_id: id.into(),
accepted_at_ms: 1,
segment_count: 1,
byte_len: 1,
}],
},
});
}
#[test] #[test]
fn run_status_line_matches_console_metrics_and_spinner_frame() { fn run_status_line_matches_console_metrics_and_spinner_frame() {
let now = Instant::now(); let now = Instant::now();
@@ -2251,15 +2266,11 @@ mod tests {
#[test] #[test]
fn queue_status_text_includes_count_and_preview() { fn queue_status_text_includes_count_and_preview() {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
app.set_worker_status(WorkerStatus::Running); set_pending_submission(&mut app, "submission-1");
for c in "queued preview".chars() {
app.insert_char(c);
}
assert!(app.submit_input().is_none());
assert_eq!( assert_eq!(
queue_status_text(&app), queue_status_text(&app),
Some("queued: 1 — queued preview".to_string()) Some("queued: 1 — submission-1".to_string())
); );
} }
@@ -2289,14 +2300,10 @@ mod tests {
Some("Worker keeps running. Press Ctrl-C again to exit TUI.".into()) Some("Worker keeps running. Press Ctrl-C again to exit TUI.".into())
); );
app.set_worker_status(WorkerStatus::Running); set_pending_submission(&mut app, "submission-1");
for c in "queued turn".chars() {
app.insert_char(c);
}
assert!(app.submit_input().is_none());
assert_eq!( assert_eq!(
actionbar_left_item(&app, now).map(|(text, _)| text), actionbar_left_item(&app, now).map(|(text, _)| text),
Some("Alt-q edit queued Alt-c clear queued".into()) Some("Alt-q continue queued Alt-c clear queued".into())
); );
app.enter_command_mode(); app.enter_command_mode();
+21 -16
View File
@@ -41,14 +41,12 @@ pub enum WorkerExecutionOperation {
Cancel, Cancel,
} }
/// Evidence that a user input reached the durable Worker session boundary. /// Evidence that a Submit request reached the durable Worker session boundary.
///
/// This is intentionally distinct from accepting a method on the Worker's
/// in-memory channel. For Flow submissions, the committed UserInput entry also
/// carries the initial Flow runtime-state extension.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerInputCommitAck { pub struct WorkerSubmissionAck {
pub submission_request_id: String,
pub submission_id: String, pub submission_id: String,
pub disposition: protocol::SubmissionDisposition,
} }
/// Typed execution result class. Results are transient operation outcomes and /// Typed execution result class. Results are transient operation outcomes and
@@ -61,7 +59,7 @@ pub struct WorkerExecutionResult {
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub message: Option<String>, pub message: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub input_commit: Option<WorkerInputCommitAck>, pub submission: Option<WorkerSubmissionAck>,
} }
/// Backend result class for a Worker execution operation. /// Backend result class for a Worker execution operation.
@@ -85,22 +83,26 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Accepted, outcome: WorkerExecutionOutcome::Accepted,
run_state, run_state,
message: None, message: None,
input_commit: None, submission: None,
} }
} }
pub fn accepted_input_committed( pub fn accepted_submission(
operation: WorkerExecutionOperation, operation: WorkerExecutionOperation,
run_state: WorkerExecutionRunState, run_state: WorkerExecutionRunState,
submission_request_id: impl Into<String>,
submission_id: impl Into<String>, submission_id: impl Into<String>,
disposition: protocol::SubmissionDisposition,
) -> Self { ) -> Self {
Self { Self {
operation, operation,
outcome: WorkerExecutionOutcome::Accepted, outcome: WorkerExecutionOutcome::Accepted,
run_state, run_state,
message: None, message: None,
input_commit: Some(WorkerInputCommitAck { submission: Some(WorkerSubmissionAck {
submission_request_id: submission_request_id.into(),
submission_id: submission_id.into(), submission_id: submission_id.into(),
disposition,
}), }),
} }
} }
@@ -111,7 +113,7 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Busy, outcome: WorkerExecutionOutcome::Busy,
run_state: WorkerExecutionRunState::Busy, run_state: WorkerExecutionRunState::Busy,
message: Some(message.into()), message: Some(message.into()),
input_commit: None, submission: None,
} }
} }
@@ -121,7 +123,7 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Rejected, outcome: WorkerExecutionOutcome::Rejected,
run_state: WorkerExecutionRunState::Stopped, run_state: WorkerExecutionRunState::Stopped,
message: Some(message.into()), message: Some(message.into()),
input_commit: None, submission: None,
} }
} }
@@ -131,7 +133,7 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Errored, outcome: WorkerExecutionOutcome::Errored,
run_state: WorkerExecutionRunState::Errored, run_state: WorkerExecutionRunState::Errored,
message: Some(message.into()), message: Some(message.into()),
input_commit: None, submission: None,
} }
} }
@@ -141,7 +143,7 @@ impl WorkerExecutionResult {
outcome: WorkerExecutionOutcome::Unsupported, outcome: WorkerExecutionOutcome::Unsupported,
run_state: WorkerExecutionRunState::Stopped, run_state: WorkerExecutionRunState::Stopped,
message: Some(message.into()), message: Some(message.into()),
input_commit: None, submission: None,
} }
} }
@@ -618,14 +620,17 @@ mod tests {
use super::*; use super::*;
#[test] #[test]
fn input_commit_ack_survives_json_round_trip() { fn submission_ack_survives_json_round_trip() {
let result = WorkerExecutionResult::accepted_input_committed( let result = WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy, WorkerExecutionRunState::Busy,
"request-1",
"submission-1", "submission-1",
protocol::SubmissionDisposition::Started,
); );
let json = serde_json::to_string(&result).unwrap(); let json = serde_json::to_string(&result).unwrap();
assert!(json.contains("\"submission_request_id\":\"request-1\""));
assert!(json.contains("\"submission_id\":\"submission-1\"")); assert!(json.contains("\"submission_id\":\"submission-1\""));
assert_eq!( assert_eq!(
serde_json::from_str::<WorkerExecutionResult>(&json).unwrap(), serde_json::from_str::<WorkerExecutionResult>(&json).unwrap(),
+8 -4
View File
@@ -2735,11 +2735,13 @@ mod tests {
_handle: &WorkerExecutionHandle, _handle: &WorkerExecutionHandle,
input: WorkerInput, input: WorkerInput,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
if let Some(submission_id) = input.submission_id { if let Some(submission_id) = input.submission_request_id {
WorkerExecutionResult::accepted_input_committed( WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle, WorkerExecutionRunState::Idle,
submission_id.clone(),
submission_id, submission_id,
protocol::SubmissionDisposition::Started,
) )
} else { } else {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(
@@ -3059,11 +3061,13 @@ mod ws_tests {
_handle: &WorkerExecutionHandle, _handle: &WorkerExecutionHandle,
input: WorkerInput, input: WorkerInput,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
if let Some(submission_id) = input.submission_id { if let Some(submission_id) = input.submission_request_id {
WorkerExecutionResult::accepted_input_committed( WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle, WorkerExecutionRunState::Idle,
submission_id.clone(),
submission_id, submission_id,
protocol::SubmissionDisposition::Started,
) )
} else { } else {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(
+24 -6
View File
@@ -25,10 +25,10 @@ impl WorkerInputKind {
pub struct WorkerInput { pub struct WorkerInput {
pub kind: WorkerInputKind, pub kind: WorkerInputKind,
pub content: String, pub content: String,
/// Runtime-generated correlation id. This is never accepted from public /// Authenticated client-generated idempotency key. Runtime generates one
/// JSON input and is consumed only by the execution backend. /// only for trusted internal callers that omit it.
#[serde(skip)] #[serde(default, skip_serializing_if = "Option::is_none")]
pub submission_id: Option<String>, pub submission_request_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub segments: Option<Vec<Segment>>, pub segments: Option<Vec<Segment>>,
} }
@@ -38,7 +38,7 @@ impl WorkerInput {
Self { Self {
kind: WorkerInputKind::User, kind: WorkerInputKind::User,
content: content.into(), content: content.into(),
submission_id: None, submission_request_id: None,
segments: None, segments: None,
} }
} }
@@ -47,7 +47,7 @@ impl WorkerInput {
Self { Self {
kind: WorkerInputKind::Notify, kind: WorkerInputKind::Notify,
content: content.into(), content: content.into(),
submission_id: None, submission_request_id: None,
segments: None, segments: None,
} }
} }
@@ -57,6 +57,21 @@ impl WorkerInput {
mod tests { mod tests {
use super::WorkerInput; use super::WorkerInput;
#[test]
fn submission_request_id_round_trips_for_authenticated_client_retry() {
let input: WorkerInput = serde_json::from_value(serde_json::json!({
"kind": "user",
"content": "message",
"submission_request_id": "request-1"
}))
.unwrap();
assert_eq!(input.submission_request_id.as_deref(), Some("request-1"));
assert_eq!(
serde_json::to_value(input).unwrap()["submission_request_id"],
"request-1"
);
}
#[test] #[test]
fn notify_is_an_operation_and_legacy_system_kind_is_rejected() { fn notify_is_an_operation_and_legacy_system_kind_is_rejected() {
assert_eq!( assert_eq!(
@@ -78,4 +93,7 @@ mod tests {
pub struct WorkerInteractionAck { pub struct WorkerInteractionAck {
pub worker_ref: WorkerRef, pub worker_ref: WorkerRef,
pub status: WorkerStatus, pub status: WorkerStatus,
/// Present for User Submit and absent for non-Submit interactions.
#[serde(skip_serializing_if = "Option::is_none")]
pub submission: Option<crate::execution::WorkerSubmissionAck>,
} }
+58 -46
View File
@@ -748,8 +748,12 @@ impl Runtime {
let state = self.lock()?; let state = self.lock()?;
state.worker(&worker_ref)?.request.initial_input.clone() state.worker(&worker_ref)?.request.initial_input.clone()
} { } {
let expected_submission_id = Uuid::now_v7().to_string(); let expected_submission_id = initial_input
initial_input.submission_id = Some(expected_submission_id.clone()); .submission_request_id
.clone()
.filter(|request_id| !request_id.trim().is_empty())
.unwrap_or_else(|| Uuid::now_v7().to_string());
initial_input.submission_request_id = Some(expected_submission_id.clone());
let dispatch_result = backend.dispatch_input(&handle, initial_input.clone()); let dispatch_result = backend.dispatch_input(&handle, initial_input.clone());
if !dispatch_result.is_accepted() { if !dispatch_result.is_accepted() {
let _ = backend.stop_worker(&handle); let _ = backend.stop_worker(&handle);
@@ -763,9 +767,9 @@ impl Runtime {
}); });
} }
let has_commit_ack = dispatch_result let has_commit_ack = dispatch_result
.input_commit .submission
.as_ref() .as_ref()
.is_some_and(|ack| ack.submission_id == expected_submission_id); .is_some_and(|ack| ack.submission_request_id == expected_submission_id);
if !has_commit_ack { if !has_commit_ack {
let _ = backend.stop_worker(&handle); let _ = backend.stop_worker(&handle);
self.rollback_failed_create(&worker_ref)?; self.rollback_failed_create(&worker_ref)?;
@@ -1146,9 +1150,14 @@ impl Runtime {
mut input: WorkerInput, mut input: WorkerInput,
) -> Result<WorkerInteractionAck, RuntimeError> { ) -> Result<WorkerInteractionAck, RuntimeError> {
validate_worker_input(&input)?; validate_worker_input(&input)?;
let expected_submission_id = if input.kind == WorkerInputKind::User { let expected_submission_id =
let submission_id = Uuid::now_v7().to_string(); if matches!(input.kind, WorkerInputKind::User | WorkerInputKind::Notify) {
input.submission_id = Some(submission_id.clone()); let submission_id = input
.submission_request_id
.clone()
.filter(|request_id| !request_id.trim().is_empty())
.unwrap_or_else(|| Uuid::now_v7().to_string());
input.submission_request_id = Some(submission_id.clone());
Some(submission_id) Some(submission_id)
} else { } else {
None None
@@ -1191,13 +1200,13 @@ impl Runtime {
} }
if let Some(expected_submission_id) = expected_submission_id if let Some(expected_submission_id) = expected_submission_id
&& dispatch_result && dispatch_result
.input_commit .submission
.as_ref() .as_ref()
.is_none_or(|ack| ack.submission_id != expected_submission_id) .is_none_or(|ack| ack.submission_request_id != expected_submission_id)
{ {
let result = WorkerExecutionResult::rejected( let result = WorkerExecutionResult::rejected(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
"execution backend did not acknowledge the committed Runtime submission id", "execution backend did not acknowledge the committed Runtime submission request id",
); );
self.record_execution_result(worker_ref, result.clone())?; self.record_execution_result(worker_ref, result.clone())?;
return Err(RuntimeError::WorkerExecutionRejected { return Err(RuntimeError::WorkerExecutionRejected {
@@ -1209,6 +1218,7 @@ impl Runtime {
}); });
} }
let submission = dispatch_result.submission.clone();
let mut state = self.lock()?; let mut state = self.lock()?;
state.ensure_running()?; state.ensure_running()?;
let worker = state.worker_mut(worker_ref)?; let worker = state.worker_mut(worker_ref)?;
@@ -1225,6 +1235,7 @@ impl Runtime {
Ok(WorkerInteractionAck { Ok(WorkerInteractionAck {
worker_ref: worker_ref.clone(), worker_ref: worker_ref.clone(),
status, status,
submission,
}) })
} }
@@ -1706,6 +1717,7 @@ impl Runtime {
} }
Ok(protocol::Event::Snapshot { Ok(protocol::Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: protocol::Greeting { greeting: protocol::Greeting {
@@ -3250,17 +3262,9 @@ fn validate_worker_input(input: &WorkerInput) -> Result<(), RuntimeError> {
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
fn input_protocol_event(input: &WorkerInput) -> Option<protocol::Event> { fn input_protocol_event(input: &WorkerInput) -> Option<protocol::Event> {
match input.kind { match input.kind {
WorkerInputKind::User => Some(protocol::Event::UserMessage { // Submit is projected only after the Worker commits UserInput. Queued
segments: input.segments.clone().unwrap_or_else(|| { // payloads must never become model- or client-visible history early.
vec![protocol::Segment::Text { WorkerInputKind::User | WorkerInputKind::Notify => None,
content: input.content.clone(),
}]
}),
}),
// The committed `SystemItem::Notification` is the sole agent-visible
// and Console-visible authority for Notify. A synthetic observation
// here would display the same notification twice.
WorkerInputKind::Notify => None,
WorkerInputKind::Compact WorkerInputKind::Compact
| WorkerInputKind::ListRewindTargets | WorkerInputKind::ListRewindTargets
| WorkerInputKind::RegisterPeer => Some(protocol::Event::SystemItem { | WorkerInputKind::RegisterPeer => Some(protocol::Event::SystemItem {
@@ -3435,6 +3439,7 @@ mod tests {
); );
let snapshot = protocol::Event::Snapshot { let snapshot = protocol::Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: protocol::Greeting { greeting: protocol::Greeting {
@@ -3475,7 +3480,7 @@ mod tests {
let input = WorkerInput { let input = WorkerInput {
kind: WorkerInputKind::User, kind: WorkerInputKind::User,
content: String::new(), content: String::new(),
submission_id: None, submission_request_id: None,
segments: Some(vec![protocol::Segment::Flow { segments: Some(vec![protocol::Segment::Flow {
selector: "builtin:coder-review".to_string(), selector: "builtin:coder-review".to_string(),
}]), }]),
@@ -3488,7 +3493,7 @@ mod tests {
let input = WorkerInput { let input = WorkerInput {
kind: WorkerInputKind::User, kind: WorkerInputKind::User,
content: String::new(), content: String::new(),
submission_id: None, submission_request_id: None,
segments: Some(Vec::new()), segments: Some(Vec::new()),
}; };
assert!(matches!( assert!(matches!(
@@ -3503,7 +3508,7 @@ mod tests {
request.initial_input = Some(WorkerInput { request.initial_input = Some(WorkerInput {
kind: WorkerInputKind::User, kind: WorkerInputKind::User,
content: String::new(), content: String::new(),
submission_id: None, submission_request_id: None,
segments: Some(vec![protocol::Segment::Flow { segments: Some(vec![protocol::Segment::Flow {
selector: "builtin:coder-review".to_string(), selector: "builtin:coder-review".to_string(),
}]), }]),
@@ -4005,7 +4010,7 @@ mod tests {
_handle: &WorkerExecutionHandle, _handle: &WorkerExecutionHandle,
input: WorkerInput, input: WorkerInput,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
let submission_id = input.submission_id.clone(); let submission_id = input.submission_request_id.clone();
self.dispatched_inputs.lock().unwrap().push(input); self.dispatched_inputs.lock().unwrap().push(input);
let mut result = self let mut result = self
.dispatch_result .dispatch_result
@@ -4013,19 +4018,21 @@ mod tests {
.unwrap() .unwrap()
.clone() .clone()
.unwrap_or_else(|| { .unwrap_or_else(|| {
WorkerExecutionResult::accepted_input_committed( WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle, WorkerExecutionRunState::Idle,
"request-test",
"test-submission", "test-submission",
protocol::SubmissionDisposition::Started,
) )
}); });
if !self if !self
.preserve_commit_ack_submission_id .preserve_commit_ack_submission_id
.load(Ordering::SeqCst) .load(Ordering::SeqCst)
&& let (Some(ack), Some(submission_id)) = && let (Some(ack), Some(submission_id)) =
(result.input_commit.as_mut(), submission_id) (result.submission.as_mut(), submission_id)
{ {
ack.submission_id = submission_id; ack.submission_request_id = submission_id;
} }
result result
} }
@@ -4717,10 +4724,12 @@ mod tests {
#[test] #[test]
fn create_worker_uses_committed_input_ack_run_state() { fn create_worker_uses_committed_input_ack_run_state() {
let (runtime, backend) = runtime_and_backend(); let (runtime, backend) = runtime_and_backend();
backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed( backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle, WorkerExecutionRunState::Idle,
"request-test",
"test-submission", "test-submission",
protocol::SubmissionDisposition::Started,
)); ));
let mut request = task_request("committed initial input is already idle"); let mut request = task_request("committed initial input is already idle");
request.initial_input = Some(WorkerInput::user("start the ticket")); request.initial_input = Some(WorkerInput::user("start the ticket"));
@@ -4731,13 +4740,15 @@ mod tests {
} }
#[test] #[test]
fn create_worker_rejects_mismatched_input_commit_acknowledgement() { fn create_worker_rejects_mismatched_submission_acknowledgement() {
let (runtime, backend) = runtime_and_backend(); let (runtime, backend) = runtime_and_backend();
backend.preserve_commit_ack_submission_id(); backend.preserve_commit_ack_submission_id();
backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed( backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy, WorkerExecutionRunState::Busy,
"request-test",
"forged-submission", "forged-submission",
protocol::SubmissionDisposition::Started,
)); ));
let mut request = task_request("mismatched initial input commit ack"); let mut request = task_request("mismatched initial input commit ack");
request.initial_input = Some(WorkerInput::user("start the ticket")); request.initial_input = Some(WorkerInput::user("start the ticket"));
@@ -4866,6 +4877,7 @@ mod tests {
&detail.worker_ref, &detail.worker_ref,
protocol::Event::Snapshot { protocol::Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![protocol::SessionSnapshotEntry { entries: vec![protocol::SessionSnapshotEntry {
entry_id: "restored-log-entry".to_owned(), entry_id: "restored-log-entry".to_owned(),
timestamp: 1, timestamp: 1,
@@ -4937,10 +4949,14 @@ mod tests {
_handle: &WorkerExecutionHandle, _handle: &WorkerExecutionHandle,
input: WorkerInput, input: WorkerInput,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
WorkerExecutionResult::accepted_input_committed( WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle, WorkerExecutionRunState::Idle,
input.submission_id.expect("Runtime submission id"), "request-test",
input
.submission_request_id
.expect("Runtime submission request id"),
protocol::SubmissionDisposition::Started,
) )
} }
} }
@@ -5030,7 +5046,7 @@ mod tests {
request.initial_input = Some(WorkerInput { request.initial_input = Some(WorkerInput {
kind: WorkerInputKind::User, kind: WorkerInputKind::User,
content: String::new(), content: String::new(),
submission_id: None, submission_request_id: None,
segments: Some(vec![ segments: Some(vec![
protocol::Segment::Flow { protocol::Segment::Flow {
selector: "builtin:coder-review".to_string(), selector: "builtin:coder-review".to_string(),
@@ -5068,7 +5084,7 @@ mod tests {
let input = WorkerInput { let input = WorkerInput {
kind: WorkerInputKind::User, kind: WorkerInputKind::User,
content: String::new(), content: String::new(),
submission_id: None, submission_request_id: None,
segments: Some(vec![protocol::Segment::Flow { segments: Some(vec![protocol::Segment::Flow {
selector: "builtin:coder-review".to_string(), selector: "builtin:coder-review".to_string(),
}]), }]),
@@ -5083,11 +5099,11 @@ mod tests {
assert_eq!(dispatched[0].kind, input.kind); assert_eq!(dispatched[0].kind, input.kind);
assert_eq!(dispatched[0].content, input.content); assert_eq!(dispatched[0].content, input.content);
assert_eq!(dispatched[0].segments, input.segments); assert_eq!(dispatched[0].segments, input.segments);
let submission_id = dispatched[0] let submission_request_id = dispatched[0]
.submission_id .submission_request_id
.as_deref() .as_deref()
.expect("Runtime submission id"); .expect("Runtime submission request id");
Uuid::parse_str(submission_id).expect("submission id UUID"); Uuid::parse_str(submission_request_id).expect("submission request id UUID");
} }
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
@@ -5113,11 +5129,7 @@ mod tests {
let observations = runtime let observations = runtime
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero()) .read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
.unwrap(); .unwrap();
assert_eq!(observations.len(), 1); assert!(observations.is_empty());
assert!(matches!(
observations[0].payload,
protocol::Event::UserMessage { .. }
));
runtime runtime
.observe_worker_event( .observe_worker_event(
@@ -5135,8 +5147,8 @@ mod tests {
let observations = runtime let observations = runtime
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero()) .read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
.unwrap(); .unwrap();
assert_eq!(observations.len(), 2); assert_eq!(observations.len(), 1);
let protocol::Event::SystemItem { item } = &observations[1].payload else { let protocol::Event::SystemItem { item } = &observations[0].payload else {
panic!("committed notification observation must be a system item"); panic!("committed notification observation must be a system item");
}; };
assert_eq!(item["kind"], "notification"); assert_eq!(item["kind"], "notification");
+119 -119
View File
@@ -39,7 +39,7 @@ use crate::working_directory::{
}; };
use async_trait::async_trait; use async_trait::async_trait;
use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus}; use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus};
use session_store::{CombinedStore, LogEntry, WorkerAggregateStore, WorkerSessionStore}; use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore};
#[cfg(test)] #[cfg(test)]
use session_store::{FsStore, FsWorkerStore}; use session_store::{FsStore, FsWorkerStore};
use tokio::runtime::Runtime; use tokio::runtime::Runtime;
@@ -57,11 +57,10 @@ use worker::feature::builtin::{
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session}; use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session};
use worker::{ use worker::{
PreparedWorker, PromptCatalogSource, SegmentLogSink, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, PreparedWorker, PromptCatalogSource, SegmentLogSink, Worker, WorkerBootstrap,
Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, WorkerBootstrapError, WorkerBootstrapLayout, WorkerControllerTransport, WorkerError,
WorkerControllerTransport, WorkerError, WorkerFilesystemAuthority, WorkerHandle, WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, WorkerWorkspaceContext,
WorkerSharedState, WorkerWorkspaceContext, WorkspaceClient, WorkspaceId, WorkspaceClient, WorkspaceId, bash_output_dir_for_worker_id,
bash_output_dir_for_worker_id,
}; };
const DEFAULT_BACKEND_ID: &str = "worker-crate"; const DEFAULT_BACKEND_ID: &str = "worker-crate";
@@ -70,17 +69,6 @@ const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
// returns a typed execution error instead of leaving the outer waiter to time out. // returns a typed execution error instead of leaving the outer waiter to time out.
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9); const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9);
fn user_input_has_submission(entry: &LogEntry, submission_id: &str) -> bool {
let extensions = match entry {
LogEntry::AnnotatedUserInput { extensions, .. } => extensions,
_ => return false,
};
extensions.iter().any(|extension| {
extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN
&& extension.payload["submission_id"].as_str() == Some(submission_id)
})
}
pub struct RuntimeWorkerController { pub struct RuntimeWorkerController {
pub handle: WorkerHandle, pub handle: WorkerHandle,
pub shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>, pub shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>,
@@ -1342,126 +1330,75 @@ where
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message)) .unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
} }
fn send_user_input_and_wait_for_commit( fn send_submit_and_wait_for_acceptance(
&self, &self,
operation: WorkerExecutionOperation, operation: WorkerExecutionOperation,
worker: WorkerHandle, worker: WorkerHandle,
method: Method, method: Method,
submission_id: String, submission_request_id: String,
accepted_run_state: WorkerExecutionRunState, accepted_run_state: WorkerExecutionRunState,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
let acknowledged_submission_id = submission_id.clone(); let request_id = submission_request_id.clone();
self.run_on_adapter_runtime(async move { self.run_on_adapter_runtime(async move {
// Subscribe before enqueueing the input so the acknowledgement cannot // Subscribe before enqueueing so a fast durable acceptance cannot
// race with a fast Worker commit. The opaque submission id is stored in // race the Runtime acknowledgement.
// the same UserInput entry as the transformed Flow input and its state.
let (_, mut committed_entries) = worker.sink.subscribe_with_snapshot();
let committed_probe = worker.clone();
let mut events = worker.subscribe(); let mut events = worker.subscribe();
worker worker
.send(method) .send(method)
.await .await
.map_err(|err| format!("failed to send Worker method: {err}"))?; .map_err(|err| format!("failed to send Worker method: {err}"))?;
let timeout_probe = committed_probe.clone(); tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move {
let timeout_submission_id = submission_id.clone();
let acknowledgement = tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move {
let input_was_committed = || {
committed_probe
.committed_entries()
.iter()
.any(|entry| user_input_has_submission(entry, &submission_id))
};
loop { loop {
tokio::select! { match events.recv().await {
entry = committed_entries.recv() => { Ok(Event::SubmissionAccepted {
match entry { submission_request_id,
Ok(entry) if user_input_has_submission(&entry, &submission_id) => { submission_id,
return Ok(()); disposition,
}) if submission_request_id == request_id => {
return Ok((submission_id, disposition));
} }
Ok(_) => {} Ok(Event::SubmissionRejected {
Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { submission_request_id,
if input_was_committed() { message,
return Ok(()); }) if submission_request_id == request_id => {
return Err(format!("worker rejected Submit: {message}"));
} }
return Err(format!(
"worker input commit acknowledgement lagged by {skipped} entry event(s)"
));
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
if input_was_committed() {
return Ok(());
}
return Err(
"worker entry stream closed before user input was committed"
.to_string(),
);
}
}
}
event = events.recv() => {
match event {
Ok(Event::Error { message, .. }) => { Ok(Event::Error { message, .. }) => {
if input_was_committed() {
return Ok(());
}
return Err(format!( return Err(format!(
"worker rejected user input before session commit: {message}" "worker rejected Submit before durable acceptance: {message}"
)); ));
} }
Ok(Event::Shutdown) => { Ok(Event::Shutdown) => {
if input_was_committed() {
return Ok(());
}
return Err( return Err(
"worker shut down before user input was committed".to_string() "worker shut down before Submit was durably accepted".to_string()
); );
} }
Ok(_) => {} Ok(_) => {}
Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
if input_was_committed() {
return Ok(());
}
return Err(format!( return Err(format!(
"worker input commit acknowledgement lagged by {skipped} protocol event(s)" "worker Submit acknowledgement lagged by {skipped} protocol event(s)"
)); ));
} }
Err(tokio::sync::broadcast::error::RecvError::Closed) => { Err(tokio::sync::broadcast::error::RecvError::Closed) => {
if input_was_committed() {
return Ok(());
}
return Err( return Err(
"worker event stream closed before user input was committed" "worker event stream closed before Submit was durably accepted"
.to_string(), .to_string(),
); );
} }
} }
} }
}
}
}) })
.await; .await
.map_err(|_| "timed out waiting for durable Worker Submit acceptance".to_string())?
match acknowledgement {
Ok(result) => result,
Err(_) => {
if timeout_probe
.committed_entries()
.iter()
.any(|entry| user_input_has_submission(entry, &timeout_submission_id))
{
Ok(())
} else {
Err("timed out waiting for worker user input commit".to_string())
}
}
}
}) })
.map(|_| { .map(|(submission_id, disposition)| {
WorkerExecutionResult::accepted_input_committed( WorkerExecutionResult::accepted_submission(
operation, operation,
accepted_run_state, accepted_run_state,
acknowledged_submission_id, submission_request_id,
submission_id,
disposition,
) )
}) })
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message)) .unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
@@ -1582,8 +1519,8 @@ impl<F> Drop for WorkerRuntimeExecutionBackend<F> {
fn method_starts_turn(method: &Method) -> bool { fn method_starts_turn(method: &Method) -> bool {
matches!( matches!(
method, method,
Method::Run { .. } Method::Submit { .. }
| Method::RunTracked { .. } | Method::SubmitTracked { .. }
| Method::Notify { auto_run: true, .. } | Method::Notify { auto_run: true, .. }
| Method::Resume | Method::Resume
| Method::Compact | Method::Compact
@@ -1609,8 +1546,8 @@ fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExec
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState { fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
match method { match method {
Method::Run { .. } Method::Submit { .. }
| Method::RunTracked { .. } | Method::SubmitTracked { .. }
| Method::Notify { auto_run: true, .. } | Method::Notify { auto_run: true, .. }
| Method::Resume | Method::Resume
| Method::Compact => WorkerExecutionRunState::Busy, | Method::Compact => WorkerExecutionRunState::Busy,
@@ -1963,6 +1900,9 @@ where
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
worker, worker,
Method::Notify { Method::Notify {
notification_request_id: input
.submission_request_id
.unwrap_or_else(protocol::new_submission_request_id),
message: input.content, message: input.content,
auto_run: true, auto_run: true,
}, },
@@ -1975,21 +1915,23 @@ where
return result; return result;
} }
if worker.shared_state.get_status() != WorkerStatus::Idle let is_user_submit = input.kind == WorkerInputKind::User;
|| busy let status = worker.shared_state.get_status();
let claimed_here = status == WorkerStatus::Idle
&& busy
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_err() .is_ok();
{ if !is_user_submit && !claimed_here {
return WorkerExecutionResult::busy( return WorkerExecutionResult::busy(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
"Worker is already running; runtime adapter v0 does not queue input", "Worker is already running",
); );
} }
let (method, submission_id) = match input.kind { let (method, submission_request_id) = match input.kind {
WorkerInputKind::User => { WorkerInputKind::User => {
let Some(submission_id) = input let Some(submission_id) = input
.submission_id .submission_request_id
.filter(|submission_id| !submission_id.trim().is_empty()) .filter(|submission_id| !submission_id.trim().is_empty())
else { else {
busy.store(false, Ordering::SeqCst); busy.store(false, Ordering::SeqCst);
@@ -1999,11 +1941,11 @@ where
); );
}; };
( (
Method::RunTracked { Method::SubmitTracked {
submission_request_id: submission_id.clone(),
input: input.segments.unwrap_or_else(|| { input: input.segments.unwrap_or_else(|| {
vec![Segment::text(input.content.trim().to_string())] vec![Segment::text(input.content.trim().to_string())]
}), }),
submission_id: submission_id.clone(),
}, },
Some(submission_id), Some(submission_id),
) )
@@ -2021,21 +1963,21 @@ where
), ),
}; };
let accepted_run_state = match method { let accepted_run_state = match method {
Method::Run { .. } Method::Submit { .. }
| Method::RunTracked { .. } | Method::SubmitTracked { .. }
| Method::Notify { .. } | Method::Notify { .. }
| Method::Compact => WorkerExecutionRunState::Busy, | Method::Compact => WorkerExecutionRunState::Busy,
_ => WorkerExecutionRunState::Idle, _ => WorkerExecutionRunState::Idle,
}; };
let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle; let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle;
let waits_for_user_input_commit = submission_id.is_some(); let waits_for_submission_acceptance = submission_request_id.is_some();
let result = if waits_for_user_input_commit { let result = if waits_for_submission_acceptance {
self.send_user_input_and_wait_for_commit( self.send_submit_and_wait_for_acceptance(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
worker, worker,
method, method,
submission_id.expect("tracked Run has submission id"), submission_request_id.expect("Submit must have a submission request id"),
accepted_run_state, accepted_run_state,
) )
} else { } else {
@@ -2046,7 +1988,9 @@ where
accepted_run_state, accepted_run_state,
) )
}; };
if accepted_is_idle || result.outcome != crate::execution::WorkerExecutionOutcome::Accepted if accepted_is_idle
|| (claimed_here
&& result.outcome != crate::execution::WorkerExecutionOutcome::Accepted)
{ {
busy.store(false, Ordering::SeqCst); busy.store(false, Ordering::SeqCst);
} }
@@ -3400,6 +3344,60 @@ mod tests {
); );
} }
#[test]
fn running_worker_accepts_a_second_submit_as_queued() {
let client = MockClient::sequential(vec![MockResponse::Hang(vec![])]);
let runtime_base = tempfile::tempdir().unwrap();
let cwd = tempfile::tempdir().unwrap();
let store = tempfile::tempdir().unwrap();
let factory = MockFactory {
client,
runtime_base: runtime_base.path().to_path_buf(),
cwd: cwd.path().to_path_buf(),
store_dir: store.path().join("sessions"),
worker_metadata_dir: store.path().join("workers"),
observed_cwds: Arc::new(Mutex::new(Vec::new())),
observed_workspace_clients: Arc::new(Mutex::new(Vec::new())),
};
let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
let runtime =
EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend).unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let detail = runtime
.create_worker(create_request("queued-submit"))
.unwrap();
let mut first_input = WorkerInput::user("first");
first_input.submission_request_id = Some("request-first".into());
let first = runtime
.send_input(&detail.worker_ref, first_input.clone())
.unwrap();
assert_eq!(
first.submission.as_ref().map(|ack| ack.disposition),
Some(protocol::SubmissionDisposition::Started)
);
let retry = runtime.send_input(&detail.worker_ref, first_input).unwrap();
assert_eq!(retry.submission, first.submission);
let mut conflicting_retry = WorkerInput::user("different");
conflicting_retry.submission_request_id = Some("request-first".into());
assert!(
runtime
.send_input(&detail.worker_ref, conflicting_retry)
.is_err(),
"same request id with a different payload must fail"
);
let mut second_input = WorkerInput::user("second");
second_input.submission_request_id = Some("request-second".into());
let second = runtime
.send_input(&detail.worker_ref, second_input)
.unwrap();
assert_eq!(
second.submission.as_ref().map(|ack| ack.disposition),
Some(protocol::SubmissionDisposition::Queued)
);
}
#[test] #[test]
fn create_with_initial_input_returns_after_session_commit() { fn create_with_initial_input_returns_after_session_commit() {
let client = MockClient::new(simple_text_events()); let client = MockClient::new(simple_text_events());
@@ -3449,8 +3447,10 @@ mod tests {
}; };
extensions extensions
.iter() .iter()
.find(|extension| extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN) .find(|extension| extension.domain == "worker.pending_activations.v1")
.and_then(|extension| extension.payload["submission_id"].as_str()) .and_then(|extension| {
extension.payload["receipts"][0]["submission_id"].as_str()
})
}) })
.expect("committed input submission id"); .expect("committed input submission id");
uuid::Uuid::parse_str(submission_id).expect("opaque submission id is a UUID"); uuid::Uuid::parse_str(submission_id).expect("opaque submission id is a UUID");
+1
View File
@@ -20,6 +20,7 @@ protocol = { workspace = true, features = ["json-schema"] }
client = { workspace = true } client = { workspace = true }
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true } serde_json = { workspace = true }
sha2 = { workspace = true }
reqwest = { version = "0.13", default-features = false, features = ["blocking", "native-tls"] } reqwest = { version = "0.13", default-features = false, features = ["blocking", "native-tls"] }
thiserror = { workspace = true } thiserror = { workspace = true }
tokio = { workspace = true, features = ["fs", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } tokio = { workspace = true, features = ["fs", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] }
+4 -1
View File
@@ -101,7 +101,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Send a run method // Send a run method
handle handle
.send(Method::run_text("What is the capital of France?")) .send(Method::submit_text(
protocol::new_submission_request_id(),
"What is the capital of France?",
))
.await?; .await?;
// Wait for completion // Wait for completion
+361 -149
View File
@@ -5,7 +5,7 @@ use std::sync::atomic::Ordering;
use agen::EngineError; use agen::EngineError;
use agen::llm_client::client::LlmClient; use agen::llm_client::client::LlmClient;
use session_store::WorkerMetadataStore; use session_store::WorkerMetadataStore;
use session_store::{LogEntry, SessionExtension, Store}; use session_store::{LogEntry, Store};
use tokio::sync::{broadcast, mpsc, oneshot}; use tokio::sync::{broadcast, mpsc, oneshot};
use crate::discovery::WorkerDiscovery; use crate::discovery::WorkerDiscovery;
@@ -23,16 +23,12 @@ use crate::shutdown_after_idle::{
}; };
use crate::spawn::registry::SpawnedWorkerRegistry; use crate::spawn::registry::SpawnedWorkerRegistry;
use crate::spawn::tool::sub_worker_spawn_tool; use crate::spawn::tool::sub_worker_spawn_tool;
use crate::worker::{ use crate::worker::{SystemItemCommitter, Worker, WorkerError, WorkerRunResult};
SystemItemCommitter, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, WorkerError,
WorkerRunResult,
};
use protocol::{ use protocol::{
AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent, AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent,
CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus, CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus,
CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice, CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice,
ErrorCode, Event, Method, RewindTargetId, RunResult, Segment, TurnResult, UploadedFileRef, ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef, WorkerStatus,
WorkerStatus,
}; };
use workdir::{ use workdir::{
CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot, CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot,
@@ -58,6 +54,7 @@ pub struct WorkerHandle {
spawned_registry: Arc<SpawnedWorkerRegistry>, spawned_registry: Arc<SpawnedWorkerRegistry>,
artifact_store: Arc<dyn Store>, artifact_store: Arc<dyn Store>,
session_id: session_store::SessionId, session_id: session_store::SessionId,
pending_activations: Arc<std::sync::Mutex<crate::worker::PendingActivationState>>,
} }
impl WorkerHandle { impl WorkerHandle {
@@ -131,8 +128,15 @@ impl WorkerHandle {
let in_flight = snapshot_from_guard(&in_flight_guard); let in_flight = snapshot_from_guard(&in_flight_guard);
(entries, entry_rx, in_flight) (entries, entry_rx, in_flight)
}; };
let mut session =
session_store::public_snapshot::project_current_session_snapshot(&entries);
session.pending_submissions = self
.pending_activations
.lock()
.expect("pending activation state poisoned")
.snapshot();
let event = Event::Snapshot { let event = Event::Snapshot {
session: session_store::public_snapshot::project_current_session_snapshot(&entries), session,
greeting: self.shared_state.greeting.clone(), greeting: self.shared_state.greeting.clone(),
status: self.shared_state.get_status(), status: self.shared_state.get_status(),
in_flight, in_flight,
@@ -213,21 +217,41 @@ async fn finish_controller_run<C, St>(
/// `Worker::*` entry point — `RunForNotification` carries none because /// `Worker::*` entry point — `RunForNotification` carries none because
/// `worker.run_for_notification()` drains the NotifyBuffer on its own. /// `worker.run_for_notification()` drains the NotifyBuffer on its own.
enum PendingRun { enum PendingRun {
Run(Vec<Segment>), Submit(crate::worker::PendingSubmission),
RunTracked {
input: Vec<Segment>,
extension: SessionExtension,
},
/// Self-initiated turn kicked from the notify buffer. The carried /// Self-initiated turn kicked from the notify buffer. The carried
/// `InvokeKind` is the trigger that flipped the Worker from IDLE /// `InvokeKind` is the trigger that flipped the Worker from IDLE
/// (Notify or WorkerEvent) and is recorded by the Invoke marker /// (Notify or WorkerEvent) and is recorded by the Invoke marker
/// committed at the start of `worker.run_for_notification`. /// committed at the start of `worker.run_for_notification`.
RunForNotification(protocol::InvokeKind), RunForNotification {
invoke_kind: protocol::InvokeKind,
notification_request_id: Option<String>,
},
Resume, Resume,
} }
fn prepare_pending_run<St: Store + Clone>(
pending_submissions: &crate::worker::PendingSubmissionHandle<St>,
notify_buffer: &NotifyBuffer,
) -> Result<Option<PendingRun>, crate::worker::PendingSubmissionError> {
Ok(match pending_submissions.prepare_next_activation()? {
Some(crate::worker::PendingActivation::Submission(submission)) => {
Some(PendingRun::Submit(submission))
}
Some(crate::worker::PendingActivation::Notification(notification)) => {
let extension = pending_submissions.notification_activation_extension();
let notification_request_id = notification.notification_request_id.clone();
notify_buffer.push_durable_notify(notification.message, extension);
Some(PendingRun::RunForNotification {
invoke_kind: protocol::InvokeKind::Notify,
notification_request_id: Some(notification_request_id),
})
}
None => None,
})
}
impl PendingRun { impl PendingRun {
/// Whether this turn was kicked off by the parent (via `Method::Run` /// Whether this turn was kicked off by the parent (via `Method::Submit`
/// or `Method::Resume`). Used by [`drive_turn`] to gate upward /// or `Method::Resume`). Used by [`drive_turn`] to gate upward
/// `WorkerEvent::TurnEnded` / `WorkerEvent::Errored` reports so the parent /// `WorkerEvent::TurnEnded` / `WorkerEvent::Errored` reports so the parent
/// only sees completion signals for work it actually delegated. /// only sees completion signals for work it actually delegated.
@@ -235,16 +259,12 @@ impl PendingRun {
/// notify buffer (Notify / inbound WorkerEvent) and stays silent. /// notify buffer (Notify / inbound WorkerEvent) and stays silent.
fn is_parent_originated(&self) -> bool { fn is_parent_originated(&self) -> bool {
match self { match self {
PendingRun::Run(_) | PendingRun::RunTracked { .. } | PendingRun::Resume => true, PendingRun::Submit(_) | PendingRun::Resume => true,
PendingRun::RunForNotification(_) => false, PendingRun::RunForNotification { .. } => false,
} }
} }
} }
fn should_auto_run_notification(status: WorkerStatus, auto_run: bool) -> bool {
auto_run && status == WorkerStatus::Idle
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// WorkerController — actor that owns a Worker // WorkerController — actor that owns a Worker
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -552,6 +572,7 @@ impl WorkerController {
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone()); let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
let session_id = worker.session_id(); let session_id = worker.session_id();
let pending_activations = worker.pending_activation_state();
let handle = WorkerHandle { let handle = WorkerHandle {
method_tx, method_tx,
working_event_tx: working_event_tx.clone(), working_event_tx: working_event_tx.clone(),
@@ -563,6 +584,7 @@ impl WorkerController {
spawned_registry: spawned_registry.clone(), spawned_registry: spawned_registry.clone(),
artifact_store, artifact_store,
session_id, session_id,
pending_activations,
}; };
let socket_server = match transport { let socket_server = match transport {
@@ -1291,6 +1313,7 @@ async fn controller_loop<C, St>(
spawned_registry.clone(), spawned_registry.clone(),
); );
let mut pending: Option<PendingRun> = None; let mut pending: Option<PendingRun> = None;
let pending_submissions = worker.pending_submission_handle();
loop { loop {
// Top-of-iteration: if an event handler staged a run, fire it // Top-of-iteration: if an event handler staged a run, fire it
@@ -1307,8 +1330,8 @@ async fn controller_loop<C, St>(
// interrupted/error turn from being carried into the next snapshot. // interrupted/error turn from being carried into the next snapshot.
worker.clear_in_flight_events(); worker.clear_in_flight_events();
let parent_originated = run.is_parent_originated(); let parent_originated = run.is_parent_originated();
let user_input_run = matches!(&run, PendingRun::Run(_) | PendingRun::RunTracked { .. }); let user_input_submit = matches!(&run, PendingRun::Submit(_));
if !user_input_run { if !user_input_submit {
set_controller_status( set_controller_status(
&shared_state, &shared_state,
&runtime_dir, &runtime_dir,
@@ -1317,37 +1340,21 @@ async fn controller_loop<C, St>(
) )
.await; .await;
} }
let (mut new_status, shutdown) = match run { let notification_request_id = match &run {
PendingRun::Run(input) => { PendingRun::RunForNotification {
notification_request_id,
..
} => notification_request_id.clone(),
_ => None,
};
let (mut new_status, shutdown, may_drain_pending) = match run {
PendingRun::Submit(submission) => {
let (input_commit_tx, input_commit_rx) = oneshot::channel(); let (input_commit_tx, input_commit_rx) = oneshot::channel();
let committed_submission = submission.clone();
let extension = pending_submissions.activation_extension();
drive_turn( drive_turn(
worker.run_with_input_extensions_and_commit_hook( worker.run_with_input_extensions_and_commit_hook(
input, submission.input,
Vec::new(),
move || {
let _ = input_commit_tx.send(());
},
),
&mut method_rx,
&working_event_tx,
&cancel_tx,
&pause_tx,
&shared_state,
&runtime_dir,
Some(input_commit_rx),
&notify_buffer,
self_parent_socket.as_ref(),
&spawner_name,
&spawned_registry,
parent_originated,
)
.await
}
PendingRun::RunTracked { input, extension } => {
let (input_commit_tx, input_commit_rx) = oneshot::channel();
drive_turn(
worker.run_with_input_extensions_and_commit_hook(
input,
vec![extension], vec![extension],
move || { move || {
let _ = input_commit_tx.send(()); let _ = input_commit_tx.send(());
@@ -1359,8 +1366,9 @@ async fn controller_loop<C, St>(
&pause_tx, &pause_tx,
&shared_state, &shared_state,
&runtime_dir, &runtime_dir,
Some(input_commit_rx), Some((input_commit_rx, committed_submission)),
&notify_buffer, &notify_buffer,
&pending_submissions,
self_parent_socket.as_ref(), self_parent_socket.as_ref(),
&spawner_name, &spawner_name,
&spawned_registry, &spawned_registry,
@@ -1368,9 +1376,9 @@ async fn controller_loop<C, St>(
) )
.await .await
} }
PendingRun::RunForNotification(kind) => { PendingRun::RunForNotification { invoke_kind, .. } => {
drive_turn( drive_turn(
worker.run_for_notification(kind), worker.run_for_notification(invoke_kind),
&mut method_rx, &mut method_rx,
&working_event_tx, &working_event_tx,
&cancel_tx, &cancel_tx,
@@ -1379,6 +1387,7 @@ async fn controller_loop<C, St>(
&runtime_dir, &runtime_dir,
None, None,
&notify_buffer, &notify_buffer,
&pending_submissions,
self_parent_socket.as_ref(), self_parent_socket.as_ref(),
&spawner_name, &spawner_name,
&spawned_registry, &spawned_registry,
@@ -1397,6 +1406,7 @@ async fn controller_loop<C, St>(
&runtime_dir, &runtime_dir,
None, None,
&notify_buffer, &notify_buffer,
&pending_submissions,
self_parent_socket.as_ref(), self_parent_socket.as_ref(),
&spawner_name, &spawner_name,
&spawned_registry, &spawned_registry,
@@ -1405,11 +1415,33 @@ async fn controller_loop<C, St>(
.await .await
} }
}; };
if !shutdown && new_status == WorkerStatus::Idle && notify_buffer.has_auto_run_pending() if let Some(notification_request_id) = notification_request_id {
{ pending_submissions.finish_notification_activation(&notification_request_id);
pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify)); }
if !shutdown && may_drain_pending && new_status == WorkerStatus::Idle {
match prepare_pending_run(&pending_submissions, &notify_buffer) {
Ok(Some(next)) => {
pending = Some(next);
new_status = WorkerStatus::Running; new_status = WorkerStatus::Running;
} }
Ok(None) => {
if notify_buffer.has_auto_run_pending() {
pending = Some(PendingRun::RunForNotification {
invoke_kind: protocol::InvokeKind::Notify,
notification_request_id: None,
});
new_status = WorkerStatus::Running;
}
}
Err(error) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::Internal,
message: error.to_string(),
});
}
}
}
finish_controller_run( finish_controller_run(
&mut worker, &mut worker,
&shared_state, &shared_state,
@@ -1435,61 +1467,118 @@ async fn controller_loop<C, St>(
}; };
match method { match method {
Method::Run { input } => { Method::Submit {
if shared_state.get_status() == WorkerStatus::Running { submission_request_id,
// Defensive: the inner select! inside drive_turn input,
// already rejects `Run` while a turn is live, so
// this branch is only reachable across a race window
// around status flips.
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::AlreadyRunning,
message: "Worker is already executing a turn".into(),
});
continue;
} }
// Stage the run without a speculative user-message echo. | Method::SubmitTracked {
// `Worker::run` validates the input, commits submission_request_id,
// `LogEntry::AnnotatedUserInput`, and the session-log sink turns that
// committed entry into the live `Event::UserMessage`. That
// keeps every client ordered against `SegmentStart` replay and
// makes persisted history the single source of visible user
// input. Paused→Run cleanup (orphan tool_result closure +
// interrupt system note) is applied inside `Worker::run` itself
// when the worker's `last_run_interrupted` flag is set.
pending = Some(PendingRun::Run(input));
}
Method::RunTracked {
input, input,
submission_id,
} => { } => {
// Runtime-correlated submissions retain their opaque id in the let request_id = submission_request_id.clone();
// same durable UserInput record used for Flow state. match pending_submissions.accept(submission_request_id, input, true) {
let extension = SessionExtension::new( Ok(acceptance) => {
WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, if let Some(activation) = acceptance.activation {
serde_json::json!({ "submission_id": submission_id }), pending = Some(PendingRun::Submit(activation));
); } else {
pending = Some(PendingRun::RunTracked { input, extension }); let _ = working_event_tx.send(Event::SubmissionAccepted {
submission_request_id: acceptance.submission_request_id,
submission_id: acceptance.submission_id,
disposition: acceptance.disposition,
});
}
}
Err(error) => {
let _ = working_event_tx.send(Event::SubmissionRejected {
submission_request_id: request_id,
message: error.to_string(),
});
} }
Method::Notify { message, auto_run } => {
// Client-side live echo is delivered as `Event::SystemItem`
// once the interceptor commits the corresponding
// `LogEntry::AnnotatedSystemItem` entry — drained out of the
// notify buffer + broadcast through the sink. No
// separate echo here.
worker.push_notify(message, auto_run);
// RUNNING: the in-flight turn drains the buffer at its next
// pending_history_appends; if an auto-run notification remains
// at turn end, the Controller stages a follow-up notification
// turn. Paused notifications remain queued until Resume/Run.
// IDLE: `auto_run` notifications stage RunForNotification;
// weak progress notices stay queued until an explicit run.
if should_auto_run_notification(shared_state.get_status(), auto_run) {
pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify));
} }
} }
Method::Notify {
notification_request_id,
message,
auto_run,
} => {
if auto_run {
match pending_submissions.accept_notification(notification_request_id, message)
{
Ok(true) => match prepare_pending_run(&pending_submissions, &notify_buffer)
{
Ok(Some(next)) => pending = Some(next),
Ok(None) => {}
Err(error) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::Internal,
message: error.to_string(),
});
}
},
Ok(false) => {}
Err(error) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::InvalidRequest,
message: error.to_string(),
});
}
}
} else {
worker.push_notify(message, false);
}
}
Method::ListPendingSubmissions => {
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
pending: pending_submissions.snapshot(),
});
}
Method::CancelPendingSubmission { submission_id } => {
match pending_submissions.cancel(&submission_id) {
Ok(pending_snapshot) => {
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
pending: pending_snapshot,
});
}
Err(error) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::InvalidRequest,
message: error.to_string(),
});
}
}
}
Method::ClearPendingSubmissions => match pending_submissions.clear() {
Ok(pending_snapshot) => {
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
pending: pending_snapshot,
});
}
Err(error) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::Internal,
message: error.to_string(),
});
}
},
Method::ContinuePending => {
match prepare_pending_run(&pending_submissions, &notify_buffer) {
Ok(Some(next)) => pending = Some(next),
Ok(None) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::InvalidRequest,
message: "pending activation queue is empty".into(),
});
}
Err(error) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::Internal,
message: error.to_string(),
});
}
}
}
Method::Resume => { Method::Resume => {
if shared_state.get_status() != WorkerStatus::Paused { if shared_state.get_status() != WorkerStatus::Paused {
let _ = working_event_tx.send(Event::Error { let _ = working_event_tx.send(Event::Error {
@@ -1703,9 +1792,10 @@ async fn controller_loop<C, St>(
// notification is not stranded. Matches the // notification is not stranded. Matches the
// `Method::Notify` idle path. // `Method::Notify` idle path.
if shared_state.get_status() == WorkerStatus::Idle { if shared_state.get_status() == WorkerStatus::Idle {
pending = Some(PendingRun::RunForNotification( pending = Some(PendingRun::RunForNotification {
protocol::InvokeKind::WorkerEvent, invoke_kind: protocol::InvokeKind::WorkerEvent,
)); notification_request_id: None,
});
} }
} }
} }
@@ -1788,12 +1878,12 @@ async fn handle_inbound_worker_event(
/// as `Errored` — only the worker-execution `Err` branch below fires. /// as `Errored` — only the worker-execution `Err` branch below fires.
/// ///
/// `parent_originated` further restricts both upward reports to turns /// `parent_originated` further restricts both upward reports to turns
/// the parent actually delegated (`Method::Run` / `Method::Resume`). /// the parent actually delegated (`Method::Submit` / `Method::Resume`).
/// `Method::Notify` / inbound `WorkerEvent` auto-kicks complete silently /// `Method::Notify` / inbound `WorkerEvent` auto-kicks complete silently
/// so the parent's history does not get flooded with child-internal /// so the parent's history does not get flooded with child-internal
/// turn boundaries. /// turn boundaries.
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
async fn drive_turn<F>( async fn drive_turn<F, St>(
worker_future: F, worker_future: F,
method_rx: &mut mpsc::Receiver<Method>, method_rx: &mut mpsc::Receiver<Method>,
working_event_tx: &broadcast::Sender<Event>, working_event_tx: &broadcast::Sender<Event>,
@@ -1801,15 +1891,17 @@ async fn drive_turn<F>(
pause_tx: &mpsc::Sender<()>, pause_tx: &mpsc::Sender<()>,
shared_state: &Arc<WorkerSharedState>, shared_state: &Arc<WorkerSharedState>,
runtime_dir: &RuntimeDir, runtime_dir: &RuntimeDir,
mut input_commit_rx: Option<oneshot::Receiver<()>>, mut input_commit: Option<(oneshot::Receiver<()>, crate::worker::PendingSubmission)>,
notify_buffer: &NotifyBuffer, notify_buffer: &NotifyBuffer,
pending_submissions: &crate::worker::PendingSubmissionHandle<St>,
parent_socket: Option<&PathBuf>, parent_socket: Option<&PathBuf>,
self_name: &str, self_name: &str,
spawned_registry: &Arc<SpawnedWorkerRegistry>, spawned_registry: &Arc<SpawnedWorkerRegistry>,
parent_originated: bool, parent_originated: bool,
) -> (WorkerStatus, bool) ) -> (WorkerStatus, bool, bool)
where where
F: std::future::Future<Output = Result<WorkerRunResult, WorkerError>>, F: std::future::Future<Output = Result<WorkerRunResult, WorkerError>>,
St: Store + Clone,
{ {
tokio::pin!(worker_future); tokio::pin!(worker_future);
let mut shutdown_requested = false; let mut shutdown_requested = false;
@@ -1822,13 +1914,25 @@ where
// Running snapshot contract deterministic even for immediate clients. // Running snapshot contract deterministic even for immediate clients.
biased; biased;
committed = async { committed = async {
input_commit_rx input_commit
.as_mut() .as_mut()
.map(|(receiver, _)| receiver)
.expect("input commit receiver guarded by select condition") .expect("input commit receiver guarded by select condition")
.await .await
}, if input_commit_rx.is_some() => { }, if input_commit.is_some() => {
input_commit_rx = None; let submission = input_commit.take().map(|(_, submission)| submission);
if committed.is_ok() { if committed.is_ok() {
if let Some(submission) = submission {
pending_submissions.finish_activation(&submission.submission_id);
let _ = working_event_tx.send(Event::SubmissionAccepted {
submission_request_id: submission.submission_request_id,
submission_id: submission.submission_id,
disposition: protocol::SubmissionDisposition::Started,
});
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
pending: pending_submissions.snapshot(),
});
}
set_controller_status( set_controller_status(
shared_state, shared_state,
runtime_dir, runtime_dir,
@@ -1836,11 +1940,33 @@ where
WorkerStatus::Running, WorkerStatus::Running,
) )
.await; .await;
} else if let Some(submission) = submission {
pending_submissions.abort_activation(submission);
} }
} }
result = &mut worker_future => { result = &mut worker_future => {
if let Some((mut receiver, submission)) = input_commit.take() {
match receiver.try_recv() {
Ok(()) => {
pending_submissions.finish_activation(&submission.submission_id);
let _ = working_event_tx.send(Event::SubmissionAccepted {
submission_request_id: submission.submission_request_id,
submission_id: submission.submission_id,
disposition: protocol::SubmissionDisposition::Started,
});
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
pending: pending_submissions.snapshot(),
});
}
Err(_) => pending_submissions.abort_activation(submission),
}
}
return match result { return match result {
Ok(r) => { Ok(r) => {
let may_drain_pending = matches!(
&r,
WorkerRunResult::Finished | WorkerRunResult::LimitReached
);
let (status, run_result) = match r { let (status, run_result) = match r {
WorkerRunResult::Finished if pause_requested => { WorkerRunResult::Finished if pause_requested => {
(WorkerStatus::Paused, RunResult::Paused) (WorkerStatus::Paused, RunResult::Paused)
@@ -1851,7 +1977,7 @@ where
WorkerRunResult::RolledBack => (WorkerStatus::Idle, RunResult::RolledBack), WorkerRunResult::RolledBack => (WorkerStatus::Idle, RunResult::RolledBack),
WorkerRunResult::Interrupted { .. } if pause_requested => { WorkerRunResult::Interrupted { .. } if pause_requested => {
let _ = working_event_tx.send(Event::RunEnd { result: RunResult::Paused }); let _ = working_event_tx.send(Event::RunEnd { result: RunResult::Paused });
return (WorkerStatus::Paused, shutdown_requested); return (WorkerStatus::Paused, shutdown_requested, false);
} }
WorkerRunResult::Interrupted { code, message } => { WorkerRunResult::Interrupted { code, message } => {
let _ = working_event_tx.send(Event::Error { let _ = working_event_tx.send(Event::Error {
@@ -1867,7 +1993,7 @@ where
}, },
); );
} }
return (WorkerStatus::Idle, shutdown_requested); return (WorkerStatus::Idle, shutdown_requested, false);
} }
}; };
let _ = working_event_tx.send(Event::RunEnd { result: run_result }); let _ = working_event_tx.send(Event::RunEnd { result: run_result });
@@ -1879,7 +2005,7 @@ where
}, },
); );
} }
(status, shutdown_requested) (status, shutdown_requested, may_drain_pending)
} }
Err(WorkerError::Engine(EngineError::Cancelled)) if pause_requested => { Err(WorkerError::Engine(EngineError::Cancelled)) if pause_requested => {
// User-initiated Pause. Report the transition to // User-initiated Pause. Report the transition to
@@ -1888,7 +2014,7 @@ where
// that channel is reserved for worker runtime // that channel is reserved for worker runtime
// failures, not deliberate interruptions. // failures, not deliberate interruptions.
let _ = working_event_tx.send(Event::RunEnd { result: RunResult::Paused }); let _ = working_event_tx.send(Event::RunEnd { result: RunResult::Paused });
(WorkerStatus::Paused, shutdown_requested) (WorkerStatus::Paused, shutdown_requested, false)
} }
Err(e) => { Err(e) => {
let code = worker_error_code(&e); let code = worker_error_code(&e);
@@ -1906,11 +2032,11 @@ where
}, },
); );
} }
(WorkerStatus::Idle, shutdown_requested) (WorkerStatus::Idle, shutdown_requested, false)
} }
}; };
} }
method = method_rx.recv() => { method = method_rx.recv(), if input_commit.is_none() => {
match method { match method {
Some(Method::Cancel) => { Some(Method::Cancel) => {
let _ = cancel_tx.try_send(()); let _ = cancel_tx.try_send(());
@@ -1923,12 +2049,71 @@ where
shutdown_requested = true; shutdown_requested = true;
let _ = cancel_tx.try_send(()); let _ = cancel_tx.try_send(());
} }
Some(Method::Run { .. } | Method::RunTracked { .. } | Method::Resume) => { Some(Method::Submit {
submission_request_id,
input,
}
| Method::SubmitTracked {
submission_request_id,
input,
}) => {
let request_id = submission_request_id.clone();
match pending_submissions.accept(submission_request_id, input, false) {
Ok(acceptance) => {
let _ = working_event_tx.send(Event::SubmissionAccepted {
submission_request_id: acceptance.submission_request_id,
submission_id: acceptance.submission_id,
disposition: acceptance.disposition,
});
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
pending: pending_submissions.snapshot(),
});
}
Err(error) => {
let _ = working_event_tx.send(Event::SubmissionRejected {
submission_request_id: request_id,
message: error.to_string(),
});
}
}
}
Some(Method::Resume | Method::ContinuePending) => {
let _ = working_event_tx.send(Event::Error { let _ = working_event_tx.send(Event::Error {
code: ErrorCode::AlreadyRunning, code: ErrorCode::AlreadyRunning,
message: "Worker is already executing a turn".into(), message: "Worker is already executing a turn".into(),
}); });
} }
Some(Method::ListPendingSubmissions) => {
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
pending: pending_submissions.snapshot(),
});
}
Some(Method::CancelPendingSubmission { submission_id }) => {
match pending_submissions.cancel(&submission_id) {
Ok(pending) => {
let _ = working_event_tx.send(Event::PendingSubmissionsChanged { pending });
}
Err(error) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::InvalidRequest,
message: error.to_string(),
});
}
}
}
Some(Method::ClearPendingSubmissions) => {
match pending_submissions.clear() {
Ok(pending) => {
let _ = working_event_tx.send(Event::PendingSubmissionsChanged { pending });
}
Err(error) => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::Internal,
message: error.to_string(),
});
}
}
}
Some(Method::Compact | Method::ListRewindTargets | Method::RewindTo { .. }) => { Some(Method::Compact | Method::ListRewindTargets | Method::RewindTo { .. }) => {
let _ = working_event_tx.send(Event::Error { let _ = working_event_tx.send(Event::Error {
code: ErrorCode::AlreadyRunning, code: ErrorCode::AlreadyRunning,
@@ -1936,11 +2121,28 @@ where
.into(), .into(),
}); });
} }
Some(Method::Notify { message, auto_run }) => { Some(Method::Notify {
// Live echo arrives via `Event::SystemItem` once notification_request_id,
// the in-flight turn's next `pending_history_appends` message,
// drains this entry through the interceptor. auto_run,
notify_buffer.push_notify(message, auto_run); }) => {
if auto_run {
if let Err(error) = pending_submissions.accept_notification(
notification_request_id,
message,
) {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::InvalidRequest,
message: error.to_string(),
});
} else {
let _ = working_event_tx.send(Event::PendingSubmissionsChanged {
pending: pending_submissions.snapshot(),
});
}
} else {
notify_buffer.push_notify(message, false);
}
} }
Some(Method::ListCompletions { .. }) => {} Some(Method::ListCompletions { .. }) => {}
Some(Method::ListWorkers | Method::RestoreWorker { .. } | Method::RegisterPeer { .. }) => { Some(Method::ListWorkers | Method::RestoreWorker { .. } | Method::RegisterPeer { .. }) => {
@@ -1969,7 +2171,7 @@ where
None => { None => {
let _ = cancel_tx.try_send(()); let _ = cancel_tx.try_send(());
shared_state.set_status(WorkerStatus::Idle); shared_state.set_status(WorkerStatus::Idle);
return (WorkerStatus::Idle, false); return (WorkerStatus::Idle, false, false);
} }
} }
} }
@@ -2134,19 +2336,14 @@ mod tests {
#[test] #[test]
fn pending_run_parent_origin_table() { fn pending_run_parent_origin_table() {
assert!(PendingRun::Run(Vec::new()).is_parent_originated());
assert!(PendingRun::Resume.is_parent_originated()); assert!(PendingRun::Resume.is_parent_originated());
assert!( assert!(
!PendingRun::RunForNotification(protocol::InvokeKind::Notify).is_parent_originated() !PendingRun::RunForNotification {
); invoke_kind: protocol::InvokeKind::Notify,
notification_request_id: None,
} }
.is_parent_originated()
#[test] );
fn notification_auto_run_gate_only_allows_idle_auto_run() {
assert!(should_auto_run_notification(WorkerStatus::Idle, true));
assert!(!should_auto_run_notification(WorkerStatus::Idle, false));
assert!(!should_auto_run_notification(WorkerStatus::Running, true));
assert!(!should_auto_run_notification(WorkerStatus::Paused, true));
} }
struct DriveTurnEnv { struct DriveTurnEnv {
@@ -2161,6 +2358,7 @@ mod tests {
_pause_rx: mpsc::Receiver<()>, _pause_rx: mpsc::Receiver<()>,
shared_state: Arc<WorkerSharedState>, shared_state: Arc<WorkerSharedState>,
notify_buffer: NotifyBuffer, notify_buffer: NotifyBuffer,
pending_submissions: crate::worker::PendingSubmissionHandle<session_store::FsStore>,
spawned_registry: Arc<SpawnedWorkerRegistry>, spawned_registry: Arc<SpawnedWorkerRegistry>,
parent_socket_path: PathBuf, parent_socket_path: PathBuf,
runtime_dir: Arc<RuntimeDir>, runtime_dir: Arc<RuntimeDir>,
@@ -2194,6 +2392,8 @@ mod tests {
}, },
)); ));
let notify_buffer = NotifyBuffer::new(); let notify_buffer = NotifyBuffer::new();
let pending_submissions =
crate::worker::PendingSubmissionHandle::for_test(&temp.path().join("pending-sessions"));
let spawned_registry = SpawnedWorkerRegistry::new(runtime_dir.clone()); let spawned_registry = SpawnedWorkerRegistry::new(runtime_dir.clone());
let parent_socket_path = temp.path().join("parent.sock"); let parent_socket_path = temp.path().join("parent.sock");
@@ -2207,6 +2407,7 @@ mod tests {
_pause_rx: pause_rx, _pause_rx: pause_rx,
shared_state, shared_state,
notify_buffer, notify_buffer,
pending_submissions,
spawned_registry, spawned_registry,
parent_socket_path, parent_socket_path,
runtime_dir, runtime_dir,
@@ -2225,6 +2426,7 @@ mod tests {
writer writer
.write(&Event::Snapshot { .write(&Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: protocol::Greeting { greeting: protocol::Greeting {
@@ -2259,7 +2461,7 @@ mod tests {
let recv = tokio::spawn(recv_worker_event(listener, Duration::from_secs(2))); let recv = tokio::spawn(recv_worker_event(listener, Duration::from_secs(2)));
let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) };
let (status, shutdown) = drive_turn( let (status, shutdown, _) = drive_turn(
worker_future, worker_future,
&mut env.method_rx, &mut env.method_rx,
&env.working_event_tx, &env.working_event_tx,
@@ -2269,6 +2471,7 @@ mod tests {
&env.runtime_dir, &env.runtime_dir,
None, None,
&env.notify_buffer, &env.notify_buffer,
&env.pending_submissions,
Some(&env.parent_socket_path), Some(&env.parent_socket_path),
"child-worker", "child-worker",
&env.spawned_registry, &env.spawned_registry,
@@ -2302,7 +2505,7 @@ mod tests {
Ok::<_, WorkerError>(WorkerRunResult::Finished) Ok::<_, WorkerError>(WorkerRunResult::Finished)
}; };
let started_at = std::time::Instant::now(); let started_at = std::time::Instant::now();
let (status, shutdown) = drive_turn( let (status, shutdown, _) = drive_turn(
worker_future, worker_future,
&mut env.method_rx, &mut env.method_rx,
&env.working_event_tx, &env.working_event_tx,
@@ -2312,6 +2515,7 @@ mod tests {
&env.runtime_dir, &env.runtime_dir,
None, None,
&env.notify_buffer, &env.notify_buffer,
&env.pending_submissions,
None, None,
"child-worker", "child-worker",
&env.spawned_registry, &env.spawned_registry,
@@ -2332,7 +2536,7 @@ mod tests {
let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener"); let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener");
let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) };
let (status, _) = drive_turn( let (status, _, _) = drive_turn(
worker_future, worker_future,
&mut env.method_rx, &mut env.method_rx,
&env.working_event_tx, &env.working_event_tx,
@@ -2342,6 +2546,7 @@ mod tests {
&env.runtime_dir, &env.runtime_dir,
None, None,
&env.notify_buffer, &env.notify_buffer,
&env.pending_submissions,
Some(&env.parent_socket_path), Some(&env.parent_socket_path),
"child-worker", "child-worker",
&env.spawned_registry, &env.spawned_registry,
@@ -2370,7 +2575,7 @@ mod tests {
"boom from test".into(), "boom from test".into(),
))) )))
}; };
let (status, _) = drive_turn( let (status, _, _) = drive_turn(
worker_future, worker_future,
&mut env.method_rx, &mut env.method_rx,
&env.working_event_tx, &env.working_event_tx,
@@ -2380,6 +2585,7 @@ mod tests {
&env.runtime_dir, &env.runtime_dir,
None, None,
&env.notify_buffer, &env.notify_buffer,
&env.pending_submissions,
Some(&env.parent_socket_path), Some(&env.parent_socket_path),
"child-worker", "child-worker",
&env.spawned_registry, &env.spawned_registry,
@@ -2414,7 +2620,7 @@ mod tests {
"boom from notify".into(), "boom from notify".into(),
))) )))
}; };
let (status, _) = drive_turn( let (status, _, _) = drive_turn(
worker_future, worker_future,
&mut env.method_rx, &mut env.method_rx,
&env.working_event_tx, &env.working_event_tx,
@@ -2424,6 +2630,7 @@ mod tests {
&env.runtime_dir, &env.runtime_dir,
None, None,
&env.notify_buffer, &env.notify_buffer,
&env.pending_submissions,
Some(&env.parent_socket_path), Some(&env.parent_socket_path),
"child-worker", "child-worker",
&env.spawned_registry, &env.spawned_registry,
@@ -2456,7 +2663,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
Ok::<_, WorkerError>(WorkerRunResult::Finished) Ok::<_, WorkerError>(WorkerRunResult::Finished)
}; };
let (status, shutdown) = drive_turn( let (status, shutdown, _) = drive_turn(
worker_future, worker_future,
&mut env.method_rx, &mut env.method_rx,
&env.working_event_tx, &env.working_event_tx,
@@ -2466,6 +2673,7 @@ mod tests {
&env.runtime_dir, &env.runtime_dir,
None, None,
&env.notify_buffer, &env.notify_buffer,
&env.pending_submissions,
Some(&env.parent_socket_path), Some(&env.parent_socket_path),
"parent", "parent",
&env.spawned_registry, &env.spawned_registry,
@@ -2495,7 +2703,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
Ok::<_, WorkerError>(WorkerRunResult::Finished) Ok::<_, WorkerError>(WorkerRunResult::Finished)
}; };
let (status, shutdown) = drive_turn( let (status, shutdown, _) = drive_turn(
worker_future, worker_future,
&mut env.method_rx, &mut env.method_rx,
&env.working_event_tx, &env.working_event_tx,
@@ -2505,6 +2713,7 @@ mod tests {
&env.runtime_dir, &env.runtime_dir,
None, None,
&env.notify_buffer, &env.notify_buffer,
&env.pending_submissions,
Some(&env.parent_socket_path), Some(&env.parent_socket_path),
"parent", "parent",
&env.spawned_registry, &env.spawned_registry,
@@ -2522,6 +2731,7 @@ mod tests {
let mut env = make_env().await; let mut env = make_env().await;
env._method_tx env._method_tx
.send(Method::Notify { .send(Method::Notify {
notification_request_id: protocol::new_submission_request_id(),
message: "continue".into(), message: "continue".into(),
auto_run: true, auto_run: true,
}) })
@@ -2532,7 +2742,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
Ok::<_, WorkerError>(WorkerRunResult::Finished) Ok::<_, WorkerError>(WorkerRunResult::Finished)
}; };
let (status, shutdown) = drive_turn( let (status, shutdown, _) = drive_turn(
worker_future, worker_future,
&mut env.method_rx, &mut env.method_rx,
&env.working_event_tx, &env.working_event_tx,
@@ -2542,6 +2752,7 @@ mod tests {
&env.runtime_dir, &env.runtime_dir,
None, None,
&env.notify_buffer, &env.notify_buffer,
&env.pending_submissions,
Some(&env.parent_socket_path), Some(&env.parent_socket_path),
"parent", "parent",
&env.spawned_registry, &env.spawned_registry,
@@ -2551,8 +2762,8 @@ mod tests {
assert_eq!(status, WorkerStatus::Idle); assert_eq!(status, WorkerStatus::Idle);
assert!(!shutdown); assert!(!shutdown);
assert_eq!(env.notify_buffer.len(), 1); assert_eq!(env.notify_buffer.len(), 0);
assert!(env.notify_buffer.has_auto_run_pending()); assert_eq!(env.pending_submissions.snapshot().notification_count, 1);
} }
#[tokio::test] #[tokio::test]
@@ -2568,7 +2779,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
Ok::<_, WorkerError>(WorkerRunResult::Finished) Ok::<_, WorkerError>(WorkerRunResult::Finished)
}; };
let (status, shutdown) = drive_turn( let (status, shutdown, _) = drive_turn(
worker_future, worker_future,
&mut env.method_rx, &mut env.method_rx,
&env.working_event_tx, &env.working_event_tx,
@@ -2578,6 +2789,7 @@ mod tests {
&env.runtime_dir, &env.runtime_dir,
None, None,
&env.notify_buffer, &env.notify_buffer,
&env.pending_submissions,
Some(&env.parent_socket_path), Some(&env.parent_socket_path),
"child-worker", "child-worker",
&env.spawned_registry, &env.spawned_registry,
+24 -3
View File
@@ -1012,7 +1012,15 @@ async fn send_peer_notify(socket_path: &Path, message: String) -> io::Result<()>
} }
async fn send_notify(socket_path: &Path, message: String, auto_run: bool) -> io::Result<()> { async fn send_notify(socket_path: &Path, message: String, auto_run: bool) -> io::Result<()> {
connect_and_send(socket_path, &Method::Notify { message, auto_run }).await connect_and_send(
socket_path,
&Method::Notify {
notification_request_id: protocol::new_submission_request_id(),
message,
auto_run,
},
)
.await
} }
fn json_content<T: Serialize>(value: &T) -> Result<String, ToolError> { fn json_content<T: Serialize>(value: &T) -> Result<String, ToolError> {
@@ -1482,6 +1490,7 @@ mod tests {
writer writer
.write(&Event::Snapshot { .write(&Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: protocol::Greeting { greeting: protocol::Greeting {
@@ -1517,6 +1526,7 @@ mod tests {
writer writer
.write(&Event::Snapshot { .write(&Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: protocol::Greeting { greeting: protocol::Greeting {
@@ -1536,7 +1546,10 @@ mod tests {
.await .await
.unwrap(); .unwrap();
let method = reader.next::<Method>().await.unwrap().unwrap(); let method = reader.next::<Method>().await.unwrap().unwrap();
if let Method::Notify { message, auto_run } = method { if let Method::Notify {
message, auto_run, ..
} = method
{
assert!(auto_run); assert!(auto_run);
tx.send(message).await.unwrap(); tx.send(message).await.unwrap();
} else { } else {
@@ -1608,6 +1621,7 @@ mod tests {
writer writer
.write(&Event::Snapshot { .write(&Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: protocol::Greeting { greeting: protocol::Greeting {
@@ -1634,6 +1648,7 @@ mod tests {
writer writer
.write(&Event::Snapshot { .write(&Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: protocol::Greeting { greeting: protocol::Greeting {
@@ -1653,7 +1668,10 @@ mod tests {
.await .await
.unwrap(); .unwrap();
let method = reader.next::<Method>().await.unwrap().unwrap(); let method = reader.next::<Method>().await.unwrap().unwrap();
if let Method::Notify { message, auto_run } = method { if let Method::Notify {
message, auto_run, ..
} = method
{
assert!(!auto_run); assert!(!auto_run);
tx.send(message).await.unwrap(); tx.send(message).await.unwrap();
} else { } else {
@@ -1738,6 +1756,7 @@ mod tests {
writer writer
.write(&Event::Snapshot { .write(&Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: protocol::Greeting { greeting: protocol::Greeting {
@@ -1790,6 +1809,8 @@ mod tests {
let _ = writer let _ = writer
.write(&Event::Snapshot { .write(&Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(
),
entries: Vec::new(), entries: Vec::new(),
}, },
greeting: protocol::Greeting { greeting: protocol::Greeting {
@@ -803,7 +803,10 @@ mod tests {
.collect(); .collect();
Ok(WorkerSessionCapture { Ok(WorkerSessionCapture {
segment_id: "segment".to_string(), segment_id: "segment".to_string(),
session: protocol::SessionSnapshot { entries }, session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries,
},
}) })
} }
} }
+21 -6
View File
@@ -176,12 +176,16 @@ impl WorkerInterceptor {
/// `Item::system_message`s reach the worker via /// `Item::system_message`s reach the worker via
/// `ContinueWith` / `pending_history_appends`, so on-disk order /// `ContinueWith` / `pending_history_appends`, so on-disk order
/// matches worker-history order. /// matches worker-history order.
fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> { fn commit_system_items_with_extensions(
&self,
items: &[(SystemItem, Vec<session_store::SessionExtension>)],
) -> Result<(), session_store::StoreError> {
let Some(writer) = self.log_writer.as_ref() else { let Some(writer) = self.log_writer.as_ref() else {
return Ok(()); return Ok(());
}; };
for item in items { for (item, extensions) in items {
let entry = writer.commit_system_item(item.clone())?; let entry =
writer.commit_system_item_with_extensions(item.clone(), extensions.clone())?;
self.pending_committed_history self.pending_committed_history
.lock() .lock()
.expect("pending committed history poisoned") .expect("pending committed history poisoned")
@@ -190,6 +194,16 @@ impl WorkerInterceptor {
Ok(()) Ok(())
} }
fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> {
self.commit_system_items_with_extensions(
&items
.iter()
.cloned()
.map(|item| (item, Vec::new()))
.collect::<Vec<_>>(),
)
}
fn current_turn_index(&self) -> usize { fn current_turn_index(&self) -> usize {
self.next_turn_index self.next_turn_index
.load(Ordering::Relaxed) .load(Ordering::Relaxed)
@@ -327,7 +341,8 @@ impl Interceptor<SessionHistoryMetadata> for WorkerInterceptor {
projection_digest: projection.catalog_digest.clone(), projection_digest: projection.catalog_digest.clone(),
logical_name: "internal.notify_wrapper".to_string(), logical_name: "internal.notify_wrapper".to_string(),
}; };
let mut system_items: Vec<SystemItem> = Vec::with_capacity(drained.len()); let mut system_items: Vec<(SystemItem, Vec<session_store::SessionExtension>)> =
Vec::with_capacity(drained.len());
let mut items: Vec<Item> = Vec::with_capacity(drained.len()); let mut items: Vec<Item> = Vec::with_capacity(drained.len());
for entry in &drained { for entry in &drained {
let system_item = match build_system_item_with_provenance( let system_item = match build_system_item_with_provenance(
@@ -345,9 +360,9 @@ impl Interceptor<SessionHistoryMetadata> for WorkerInterceptor {
} }
}; };
items.push(system_item.to_history_item()); items.push(system_item.to_history_item());
system_items.push(system_item); system_items.push((system_item, entry.extensions()));
} }
if let Err(error) = self.commit_system_items(&system_items) { if let Err(error) = self.commit_system_items_with_extensions(&system_items) {
self.pending_notifies.requeue_front(drained); self.pending_notifies.requeue_front(drained);
return Err(InterceptorError::new( return Err(InterceptorError::new(
InterceptorErrorCategory::Dependency, InterceptorErrorCategory::Dependency,
+32 -4
View File
@@ -25,7 +25,7 @@ use std::collections::VecDeque;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use protocol::WorkerEvent; use protocol::WorkerEvent;
use session_store::SystemItem; use session_store::{SessionExtension, SystemItem};
use tracing::warn; use tracing::warn;
use crate::prompt::catalog::{CatalogError, PromptCatalog}; use crate::prompt::catalog::{CatalogError, PromptCatalog};
@@ -41,8 +41,23 @@ const CAPACITY: usize = 128;
/// is available. /// is available.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum PendingNotify { pub enum PendingNotify {
Notify { message: String, auto_run: bool }, Notify {
WorkerEvent { event: WorkerEvent }, message: String,
auto_run: bool,
extensions: Vec<SessionExtension>,
},
WorkerEvent {
event: WorkerEvent,
},
}
impl PendingNotify {
pub(crate) fn extensions(&self) -> Vec<SessionExtension> {
match self {
PendingNotify::Notify { extensions, .. } => extensions.clone(),
PendingNotify::WorkerEvent { .. } => Vec::new(),
}
}
} }
/// Shared, mutex-guarded buffer of pending entries. /// Shared, mutex-guarded buffer of pending entries.
@@ -62,7 +77,19 @@ impl NotifyBuffer {
/// oldest entry is dropped and a `tracing::warn` is emitted — the /// oldest entry is dropped and a `tracing::warn` is emitted — the
/// caller should never hit this in normal operation. /// caller should never hit this in normal operation.
pub fn push_notify(&self, message: String, auto_run: bool) { pub fn push_notify(&self, message: String, auto_run: bool) {
self.push_entry(PendingNotify::Notify { message, auto_run }); self.push_entry(PendingNotify::Notify {
message,
auto_run,
extensions: Vec::new(),
});
}
pub fn push_durable_notify(&self, message: String, extension: SessionExtension) {
self.push_entry(PendingNotify::Notify {
message,
auto_run: true,
extensions: vec![extension],
});
} }
/// Push a typed worker-event entry onto the queue. /// Push a typed worker-event entry onto the queue.
@@ -202,6 +229,7 @@ mod tests {
let entry = PendingNotify::Notify { let entry = PendingNotify::Notify {
message: "hello".into(), message: "hello".into(),
auto_run: false, auto_run: false,
extensions: Vec::new(),
}; };
let catalog = PromptCatalog::builtins_only().unwrap(); let catalog = PromptCatalog::builtins_only().unwrap();
let item = build_system_item(&entry, &catalog).unwrap(); let item = build_system_item(&entry, &catalog).unwrap();
+5 -5
View File
@@ -57,9 +57,9 @@ pub use session_history::{
}; };
pub use shared_state::WorkerSharedState; pub use shared_state::WorkerSharedState;
pub use worker::{ pub use worker::{
LocalWorkingDirectory, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, WorkerError, LocalWorkingDirectory, Worker, WorkerError, WorkerFilesystemAuthority, WorkerRunResult,
WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceClient, WorkerWorkspaceContext, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspacePromptCatalogResolution, WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, WorkspaceWorkerDiscoveryRequest, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, marker_workspace_client,
apply_worker_manifest, marker_workspace_client, unavailable_workspace_client, unavailable_workspace_client,
}; };
+1
View File
@@ -291,6 +291,7 @@ mod tests {
prompt_provenance: None, prompt_provenance: None,
}, },
), ),
extensions: Vec::new(),
} }
} }
+1
View File
@@ -72,6 +72,7 @@ mod tests {
fn snapshot(entries: Vec<serde_json::Value>) -> Event { fn snapshot(entries: Vec<serde_json::Value>) -> Event {
Event::Snapshot { Event::Snapshot {
session: protocol::SessionSnapshot { session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: entries entries: entries
.into_iter() .into_iter()
.enumerate() .enumerate()
+7 -2
View File
@@ -58,7 +58,7 @@ struct SubWorkerSpawnInput {
/// a host path and grants no authority. When omitted, the Workdir root is used. /// a host path and grants no authority. When omitted, the Workdir root is used.
#[serde(default)] #[serde(default)]
cwd: Option<String>, cwd: Option<String>,
/// First message sent to the spawned SubWorker via `Method::Run`. /// First message sent to the spawned SubWorker via `Method::Submit`.
task: String, task: String,
/// Allow rules delegated to the spawned SubWorker. Must be a subset of the /// Allow rules delegated to the spawned SubWorker. Must be a subset of the
/// spawner's explicit delegation authority; direct tool scope alone is not /// spawner's explicit delegation authority; direct tool scope alone is not
@@ -235,7 +235,11 @@ impl ParentNotificationTarget {
}; };
tokio::spawn(async move { tokio::spawn(async move {
if let Err(error) = parent_method_tx if let Err(error) = parent_method_tx
.send(Method::Notify { message, auto_run }) .send(Method::Notify {
notification_request_id: protocol::new_submission_request_id(),
message,
auto_run,
})
.await .await
{ {
tracing::warn!( tracing::warn!(
@@ -1267,6 +1271,7 @@ enabled = false
Method::Notify { Method::Notify {
message, message,
auto_run: true, auto_run: true,
..
} if message.contains("SubWorker `reviewer-child` turn ended with status Idle") } if message.contains("SubWorker `reviewer-child` turn ended with status Idle")
)); ));
assert!(!runtime.path().join("reviewer-child/sock").exists()); assert!(!runtime.path().join("reviewer-child/sock").exists());
+851 -6
View File
@@ -1,3 +1,4 @@
use std::collections::VecDeque;
#[cfg(test)] #[cfg(test)]
use std::path::Path; use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
@@ -68,6 +69,130 @@ const LARGE_PASTE_INLINE_MAX_BYTES: usize = 32 * 1024;
const WORKER_ORCHESTRATION_INSTRUCTION_ID: &str = "worker.orchestration"; const WORKER_ORCHESTRATION_INSTRUCTION_ID: &str = "worker.orchestration";
const WORKER_ORCHESTRATION_PROMPT_REF: &str = "common.worker_orchestration"; const WORKER_ORCHESTRATION_PROMPT_REF: &str = "common.worker_orchestration";
const FEATURE_HOOK_CHAIN_TIMEOUT: Duration = Duration::from_secs(30); const FEATURE_HOOK_CHAIN_TIMEOUT: Duration = Duration::from_secs(30);
const SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN: &str = "worker.pending_activations.v1";
const MAX_PENDING_SUBMISSIONS: usize = 32;
const MAX_PENDING_SUBMISSION_BYTES: u64 = 1024 * 1024;
const MAX_PENDING_ARTIFACT_REFS: usize = 64;
const MAX_ACTIVATION_REQUEST_ID_BYTES: usize = 128;
const MAX_SUBMISSION_RECEIPTS: usize = 128;
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct PendingSubmission {
pub(crate) submission_request_id: String,
pub(crate) submission_id: String,
payload_digest: String,
accepted_at_ms: u64,
activation_sequence: u64,
provenance: WorkerHistoryProvenance,
#[serde(default)]
was_queued: bool,
pub(crate) input: Vec<Segment>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct SubmissionReceipt {
submission_request_id: String,
submission_id: String,
payload_digest: String,
disposition: protocol::SubmissionDisposition,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub(crate) struct PendingNotification {
pub(crate) notification_request_id: String,
pub(crate) message: String,
payload_digest: String,
accepted_at_ms: u64,
activation_sequence: u64,
provenance: WorkerHistoryProvenance,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct NotificationReceipt {
notification_request_id: String,
payload_digest: String,
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
pub(crate) struct PendingActivationState {
revision: u64,
next_activation_sequence: u64,
/// A prepared activation remains in checkpoints until the same atomic
/// UserInput record commits the clearing checkpoint. Restore puts it back
/// at the FIFO head.
activating: Option<PendingSubmission>,
activating_notification: Option<PendingNotification>,
pending: VecDeque<PendingSubmission>,
pending_notifications: VecDeque<PendingNotification>,
receipts: VecDeque<SubmissionReceipt>,
notification_receipts: VecDeque<NotificationReceipt>,
}
impl PendingActivationState {
pub(crate) fn snapshot(&self) -> protocol::PendingSubmissionsSnapshot {
protocol::PendingSubmissionsSnapshot {
revision: self.revision,
notification_count: u32::try_from(self.pending_notifications.len()).unwrap_or(u32::MAX),
submissions: self
.pending
.iter()
.map(|pending| protocol::PendingSubmissionSummary {
submission_id: pending.submission_id.clone(),
accepted_at_ms: pending.accepted_at_ms,
segment_count: u32::try_from(pending.input.len()).unwrap_or(u32::MAX),
byte_len: submission_payload_len(&pending.input),
})
.collect(),
}
}
fn remember_notification_receipt(&mut self, receipt: NotificationReceipt) {
self.notification_receipts.push_back(receipt);
while self.notification_receipts.len() > MAX_SUBMISSION_RECEIPTS {
self.notification_receipts.pop_front();
}
}
fn remember_receipt(&mut self, receipt: SubmissionReceipt) {
self.receipts.push_back(receipt);
while self.receipts.len() > MAX_SUBMISSION_RECEIPTS {
self.receipts.pop_front();
}
}
}
fn submission_payload_len(input: &[Segment]) -> u64 {
serde_json::to_vec(input)
.map(|bytes| u64::try_from(bytes.len()).unwrap_or(u64::MAX))
.unwrap_or(u64::MAX)
}
fn submission_payload_digest(input: &[Segment]) -> String {
use sha2::Digest as _;
sha2::Sha256::digest(serde_json::to_vec(input).unwrap_or_default())
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
fn submission_artifact_ref_count(input: &[Segment]) -> usize {
input
.iter()
.filter(|segment| {
matches!(
segment,
Segment::PasteArtifact { .. } | Segment::UploadedFile { .. }
)
})
.count()
}
fn pending_activation_extension(state: &PendingActivationState) -> SessionExtension {
SessionExtension {
domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(),
payload: serde_json::to_value(state).expect("pending activation state must serialize"),
}
}
fn hook_run_exit(exit: &EngineRunExit) -> RunCommittedExit { fn hook_run_exit(exit: &EngineRunExit) -> RunCommittedExit {
match exit { match exit {
@@ -970,15 +1095,489 @@ where
} }
} }
/// Type-erased commit handle for the interceptor. Lets the #[derive(Debug, Clone)]
/// interceptor commit `SystemItem`s without being generic over the pub(crate) enum PendingActivation {
Submission(PendingSubmission),
Notification(PendingNotification),
}
#[derive(Debug, Clone)]
pub(crate) struct SubmissionAcceptance {
pub(crate) submission_request_id: String,
pub(crate) submission_id: String,
pub(crate) disposition: protocol::SubmissionDisposition,
pub(crate) activation: Option<PendingSubmission>,
}
#[derive(Debug, thiserror::Error)]
pub(crate) enum PendingSubmissionError {
#[error("submission_request_id must not be empty")]
EmptyRequestId,
#[error("activation request id exceeds {MAX_ACTIVATION_REQUEST_ID_BYTES} bytes")]
RequestIdLimit,
#[error("submission input must contain at least one typed segment")]
EmptyInput,
#[error("submission request id was already used with a different payload")]
IdempotencyConflict,
#[error("pending submission queue is full (maximum {MAX_PENDING_SUBMISSIONS})")]
CountLimit,
#[error("pending submission bytes exceed {MAX_PENDING_SUBMISSION_BYTES}")]
ByteLimit,
#[error("pending submission artifact references exceed {MAX_PENDING_ARTIFACT_REFS}")]
ArtifactLimit,
#[error("pending submission not found: {0}")]
NotFound(String),
#[error("pending submission state persistence failed: {0}")]
Store(#[from] StoreError),
}
#[derive(Clone)]
pub(crate) struct PendingSubmissionHandle<St: Clone> {
state: Arc<Mutex<PendingActivationState>>,
writer: LogWriterHandle<St>,
}
impl<St> PendingSubmissionHandle<St>
where
St: Store + Clone,
{
fn persist_locked(&self, state: &PendingActivationState) -> Result<(), PendingSubmissionError> {
self.writer.append_entry_locked(LogEntry::Extension {
ts: segment_log::now_millis(),
domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(),
payload: serde_json::to_value(state).expect("pending activation state must serialize"),
})?;
Ok(())
}
pub(crate) fn accept(
&self,
submission_request_id: String,
input: Vec<Segment>,
activate_now: bool,
) -> Result<SubmissionAcceptance, PendingSubmissionError> {
if submission_request_id.trim().is_empty() {
return Err(PendingSubmissionError::EmptyRequestId);
}
if submission_request_id.len() > MAX_ACTIVATION_REQUEST_ID_BYTES {
return Err(PendingSubmissionError::RequestIdLimit);
}
if input.is_empty() {
return Err(PendingSubmissionError::EmptyInput);
}
let payload_digest = submission_payload_digest(&input);
let _append_guard = self
.writer
.state
.append_lock
.lock()
.expect("segment append lock poisoned");
let mut current = self
.state
.lock()
.expect("pending activation state poisoned");
let original = current.clone();
if let Some(receipt) = current
.receipts
.iter()
.find(|receipt| receipt.submission_request_id == submission_request_id)
{
if receipt.payload_digest != payload_digest {
return Err(PendingSubmissionError::IdempotencyConflict);
}
return Ok(SubmissionAcceptance {
submission_request_id,
submission_id: receipt.submission_id.clone(),
disposition: receipt.disposition,
activation: None,
});
}
let submission_id = uuid::Uuid::now_v7().to_string();
let pending = PendingSubmission {
submission_request_id: submission_request_id.clone(),
submission_id: submission_id.clone(),
payload_digest: payload_digest.clone(),
accepted_at_ms: segment_log::now_millis(),
activation_sequence: current.next_activation_sequence,
provenance: WorkerHistoryProvenance::LegacyUnknown,
was_queued: !activate_now,
input,
};
current.next_activation_sequence = current.next_activation_sequence.saturating_add(1);
let disposition = if activate_now {
protocol::SubmissionDisposition::Started
} else {
protocol::SubmissionDisposition::Queued
};
current.remember_receipt(SubmissionReceipt {
submission_request_id: submission_request_id.clone(),
submission_id: submission_id.clone(),
payload_digest,
disposition,
});
current.revision = current.revision.saturating_add(1);
if activate_now {
current.activating = Some(pending.clone());
} else {
let count = current
.pending
.len()
.saturating_add(current.pending_notifications.len())
.saturating_add(1);
if count > MAX_PENDING_SUBMISSIONS {
*current = original;
return Err(PendingSubmissionError::CountLimit);
}
let bytes = current
.pending
.iter()
.map(|pending| submission_payload_len(&pending.input))
.sum::<u64>()
.saturating_add(
current
.pending_notifications
.iter()
.map(|pending| u64::try_from(pending.message.len()).unwrap_or(u64::MAX))
.sum::<u64>(),
)
.saturating_add(submission_payload_len(&pending.input));
if bytes > MAX_PENDING_SUBMISSION_BYTES {
*current = original;
return Err(PendingSubmissionError::ByteLimit);
}
let artifact_refs = current
.pending
.iter()
.map(|pending| submission_artifact_ref_count(&pending.input))
.sum::<usize>()
.saturating_add(submission_artifact_ref_count(&pending.input));
if artifact_refs > MAX_PENDING_ARTIFACT_REFS {
*current = original;
return Err(PendingSubmissionError::ArtifactLimit);
}
current.pending.push_back(pending.clone());
if let Err(error) = self.persist_locked(&current) {
*current = original;
return Err(error);
}
}
Ok(SubmissionAcceptance {
submission_request_id,
submission_id,
disposition,
activation: activate_now.then_some(pending),
})
}
pub(crate) fn accept_notification(
&self,
notification_request_id: String,
message: String,
) -> Result<bool, PendingSubmissionError> {
if notification_request_id.trim().is_empty() {
return Err(PendingSubmissionError::EmptyRequestId);
}
if notification_request_id.len() > MAX_ACTIVATION_REQUEST_ID_BYTES {
return Err(PendingSubmissionError::RequestIdLimit);
}
let payload_digest = submission_payload_digest(&[Segment::text(message.clone())]);
let _append_guard = self
.writer
.state
.append_lock
.lock()
.expect("segment append lock poisoned");
let mut state = self
.state
.lock()
.expect("pending activation state poisoned");
if let Some(receipt) = state
.notification_receipts
.iter()
.find(|receipt| receipt.notification_request_id == notification_request_id)
{
if receipt.payload_digest != payload_digest {
return Err(PendingSubmissionError::IdempotencyConflict);
}
return Ok(false);
}
if state
.pending
.len()
.saturating_add(state.pending_notifications.len())
>= MAX_PENDING_SUBMISSIONS
{
return Err(PendingSubmissionError::CountLimit);
}
let queued_bytes = state
.pending
.iter()
.map(|pending| submission_payload_len(&pending.input))
.sum::<u64>()
.saturating_add(
state
.pending_notifications
.iter()
.map(|pending| u64::try_from(pending.message.len()).unwrap_or(u64::MAX))
.sum::<u64>(),
)
.saturating_add(u64::try_from(message.len()).unwrap_or(u64::MAX));
if queued_bytes > MAX_PENDING_SUBMISSION_BYTES {
return Err(PendingSubmissionError::ByteLimit);
}
let original = state.clone();
let activation_sequence = state.next_activation_sequence;
state.next_activation_sequence = state.next_activation_sequence.saturating_add(1);
state.pending_notifications.push_back(PendingNotification {
notification_request_id: notification_request_id.clone(),
message,
payload_digest: payload_digest.clone(),
accepted_at_ms: segment_log::now_millis(),
activation_sequence,
provenance: WorkerHistoryProvenance::BackendInstruction {
operation_id: Some(notification_request_id.clone()),
},
});
state.remember_notification_receipt(NotificationReceipt {
notification_request_id,
payload_digest,
});
state.revision = state.revision.saturating_add(1);
if let Err(error) = self.persist_locked(&state) {
*state = original;
return Err(error);
}
Ok(true)
}
pub(crate) fn prepare_next_activation(
&self,
) -> Result<Option<PendingActivation>, PendingSubmissionError> {
let _append_guard = self
.writer
.state
.append_lock
.lock()
.expect("segment append lock poisoned");
let mut state = self
.state
.lock()
.expect("pending activation state poisoned");
if state.activating.is_some() || state.activating_notification.is_some() {
return Ok(None);
}
let submission_sequence = state.pending.front().map(|item| item.activation_sequence);
let notification_sequence = state
.pending_notifications
.front()
.map(|item| item.activation_sequence);
if notification_sequence.is_some()
&& (submission_sequence.is_none() || notification_sequence < submission_sequence)
{
let notification = state
.pending_notifications
.pop_front()
.expect("notification sequence came from queue head");
state.activating_notification = Some(notification.clone());
state.revision = state.revision.saturating_add(1);
return Ok(Some(PendingActivation::Notification(notification)));
}
if submission_sequence.is_some() {
let pending = state
.pending
.pop_front()
.expect("submission sequence came from queue head");
state.activating = Some(pending.clone());
state.revision = state.revision.saturating_add(1);
return Ok(Some(PendingActivation::Submission(pending)));
}
Ok(None)
}
pub(crate) fn abort_activation(&self, pending: PendingSubmission) {
let mut state = self
.state
.lock()
.expect("pending activation state poisoned");
if state.activating.as_ref().map(|item| &item.submission_id) != Some(&pending.submission_id)
{
return;
}
state.activating = None;
if pending.was_queued {
state.pending.push_front(pending);
} else {
state
.receipts
.retain(|receipt| receipt.submission_id != pending.submission_id);
}
state.revision = state.revision.saturating_add(1);
}
pub(crate) fn activation_extension(&self) -> SessionExtension {
let state = self
.state
.lock()
.expect("pending activation state poisoned");
let mut committed = state.clone();
if let Some(activating) = &committed.activating
&& let Some(receipt) = committed
.receipts
.iter_mut()
.find(|receipt| receipt.submission_id == activating.submission_id)
{
receipt.disposition = protocol::SubmissionDisposition::Started;
}
committed.activating = None;
committed.revision = committed.revision.saturating_add(1);
pending_activation_extension(&committed)
}
pub(crate) fn finish_activation(&self, submission_id: &str) {
let mut state = self
.state
.lock()
.expect("pending activation state poisoned");
if state
.activating
.as_ref()
.map(|item| item.submission_id.as_str())
== Some(submission_id)
{
if let Some(receipt) = state
.receipts
.iter_mut()
.find(|receipt| receipt.submission_id == submission_id)
{
receipt.disposition = protocol::SubmissionDisposition::Started;
}
state.activating = None;
state.revision = state.revision.saturating_add(1);
}
}
pub(crate) fn notification_activation_extension(&self) -> SessionExtension {
let state = self
.state
.lock()
.expect("pending activation state poisoned");
let mut committed = state.clone();
committed.activating_notification = None;
committed.revision = committed.revision.saturating_add(1);
pending_activation_extension(&committed)
}
pub(crate) fn finish_notification_activation(&self, notification_request_id: &str) {
let mut state = self
.state
.lock()
.expect("pending activation state poisoned");
if state
.activating_notification
.as_ref()
.map(|item| item.notification_request_id.as_str())
== Some(notification_request_id)
{
state.activating_notification = None;
state.revision = state.revision.saturating_add(1);
}
}
pub(crate) fn snapshot(&self) -> protocol::PendingSubmissionsSnapshot {
self.state
.lock()
.expect("pending activation state poisoned")
.snapshot()
}
pub(crate) fn cancel(
&self,
submission_id: &str,
) -> Result<protocol::PendingSubmissionsSnapshot, PendingSubmissionError> {
let _append_guard = self
.writer
.state
.append_lock
.lock()
.expect("segment append lock poisoned");
let mut state = self
.state
.lock()
.expect("pending activation state poisoned");
let original = state.clone();
let Some(index) = state
.pending
.iter()
.position(|pending| pending.submission_id == submission_id)
else {
return Err(PendingSubmissionError::NotFound(submission_id.to_owned()));
};
state.pending.remove(index);
state.revision = state.revision.saturating_add(1);
if let Err(error) = self.persist_locked(&state) {
*state = original;
return Err(error);
}
Ok(state.snapshot())
}
pub(crate) fn clear(
&self,
) -> Result<protocol::PendingSubmissionsSnapshot, PendingSubmissionError> {
let _append_guard = self
.writer
.state
.append_lock
.lock()
.expect("segment append lock poisoned");
let mut state = self
.state
.lock()
.expect("pending activation state poisoned");
let original = state.clone();
state.pending.clear();
state.pending_notifications.clear();
state.revision = state.revision.saturating_add(1);
if let Err(error) = self.persist_locked(&state) {
*state = original;
return Err(error);
}
Ok(state.snapshot())
}
}
impl PendingSubmissionHandle<session_store::FsStore> {
#[cfg(test)]
pub(crate) fn for_test(root: &std::path::Path) -> Self {
let store = session_store::FsStore::new(root).expect("test session store");
let session_id = session_store::new_session_id();
let segment_id = session_store::new_segment_id();
store
.create_segment(session_id, segment_id, &[])
.expect("test session segment");
Self {
state: Arc::new(Mutex::new(PendingActivationState::default())),
writer: LogWriterHandle {
store,
state: SegmentState::new(session_id, segment_id, 0),
sink: SegmentLogSink::new(),
in_flight: None,
},
}
}
}
/// Type-erased commit handle for the interceptor. Lets the interceptor commit `SystemItem`s without being generic over the
/// concrete `Store` type. /// concrete `Store` type.
pub trait SystemItemCommitter: Send + Sync { pub trait SystemItemCommitter: Send + Sync {
fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError>; fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError>;
fn commit_system_item( fn commit_system_item_with_extensions(
&self, &self,
item: SystemItem, item: SystemItem,
extensions: Vec<SessionExtension>,
) -> Result<HistoryEntry<SessionHistoryMetadata>, StoreError> { ) -> Result<HistoryEntry<SessionHistoryMetadata>, StoreError> {
let metadata = new_history_metadata( let metadata = new_history_metadata(
WorkerHistoryProvenance::BackendInstruction { operation_id: None }, WorkerHistoryProvenance::BackendInstruction { operation_id: None },
@@ -991,6 +1590,7 @@ pub trait SystemItemCommitter: Send + Sync {
item, item,
metadata: metadata.clone(), metadata: metadata.clone(),
}, },
extensions,
})?; })?;
Ok(HistoryEntry::new(history_item, metadata)) Ok(HistoryEntry::new(history_item, metadata))
} }
@@ -1027,8 +1627,6 @@ where
} }
} }
pub const WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN: &str = "worker.input-submission.v1";
#[derive(Clone)] #[derive(Clone)]
struct PreparedFlowProjection { struct PreparedFlowProjection {
selector: String, selector: String,
@@ -1049,6 +1647,7 @@ pub struct WorkerSession {
session_id: SessionId, session_id: SessionId,
revision: u64, revision: u64,
history: History<SessionHistoryMetadata>, history: History<SessionHistoryMetadata>,
pending_activations: Arc<Mutex<PendingActivationState>>,
} }
impl WorkerSession { impl WorkerSession {
@@ -1058,9 +1657,39 @@ impl WorkerSession {
session_id, session_id,
revision, revision,
history: History::from_entries(entries), history: History::from_entries(entries),
pending_activations: Arc::new(Mutex::new(PendingActivationState::default())),
} }
} }
fn restore_pending_activations(&mut self, extensions: &[(String, serde_json::Value)]) {
let Some(payload) = extensions.iter().rev().find_map(|(domain, payload)| {
(domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN).then_some(payload)
}) else {
return;
};
if let Ok(mut state) = serde_json::from_value::<PendingActivationState>(payload.clone()) {
if let Some(activating) = state.activating.take() {
state.pending.push_front(activating);
state.revision = state.revision.saturating_add(1);
}
if let Some(activating) = state.activating_notification.take() {
state.pending_notifications.push_front(activating);
state.revision = state.revision.saturating_add(1);
}
*self
.pending_activations
.lock()
.expect("pending activation state poisoned") = state;
}
}
pub fn pending_submissions(&self) -> protocol::PendingSubmissionsSnapshot {
self.pending_activations
.lock()
.expect("pending activation state poisoned")
.snapshot()
}
pub fn session_id(&self) -> SessionId { pub fn session_id(&self) -> SessionId {
self.session_id self.session_id
} }
@@ -1309,6 +1938,21 @@ impl<C: LlmClient + 'static, St: Store + Clone + 'static> Worker<C, St> {
} }
} }
pub(crate) fn pending_activation_state(&self) -> Arc<Mutex<PendingActivationState>> {
self.session.pending_activations.clone()
}
pub(crate) fn pending_submission_handle(&self) -> PendingSubmissionHandle<St> {
PendingSubmissionHandle {
state: self.session.pending_activations.clone(),
writer: self.log_writer_handle(),
}
}
pub fn pending_submissions(&self) -> protocol::PendingSubmissionsSnapshot {
self.session.pending_submissions()
}
/// Attach a type-erased system-item commit handle. The controller /// Attach a type-erased system-item commit handle. The controller
/// calls this once during spawn so the interceptor can commit /// calls this once during spawn so the interceptor can commit
/// `SystemItem`s directly without owning a generic store handle. /// `SystemItem`s directly without owning a generic store handle.
@@ -1670,6 +2314,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
}, },
metadata: skill_metadata.clone(), metadata: skill_metadata.clone(),
}, },
extensions: Vec::new(),
})?; })?;
let history_entry = HistoryEntry::new(agen::Item::system_message(body), skill_metadata); let history_entry = HistoryEntry::new(agen::Item::system_message(body), skill_metadata);
let mut annotate = history_annotator( let mut annotate = history_annotator(
@@ -1960,6 +2605,28 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
.truncate(loc.session_id, loc.segment_id, truncate_entries)?; .truncate(loc.session_id, loc.segment_id, truncate_entries)?;
self.segment_state.set_entries_written(truncate_entries); self.segment_state.set_entries_written(truncate_entries);
self.sink.truncate_silent(truncate_entries); self.sink.truncate_silent(truncate_entries);
let pending_state = self
.session
.pending_activations
.lock()
.expect("pending activation state poisoned")
.clone();
if !pending_state.pending.is_empty()
|| pending_state.activating.is_some()
|| pending_state.activating_notification.is_some()
|| !pending_state.receipts.is_empty()
{
let checkpoint = LogEntry::Extension {
ts: segment_log::now_millis(),
domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(),
payload: serde_json::to_value(&pending_state).map_err(|error| {
RewindError::Invalid(format!(
"serialize pending submissions during rewind: {error}"
))
})?,
};
self.commit_entry(checkpoint)?;
}
let history_entries = restore_history_entries(loc.session_id, loc.segment_id, &retained) let history_entries = restore_history_entries(loc.session_id, loc.segment_id, &retained)
.map_err(|error| RewindError::Invalid(error.into()))?; .map_err(|error| RewindError::Invalid(error.into()))?;
@@ -2525,7 +3192,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
/// Convenience: run with a single `Segment::Text`. /// Convenience: run with a single `Segment::Text`.
/// ///
/// Equivalent to `run(vec![Segment::text(s)])`. The dumb-client /// Equivalent to `run(vec![Segment::text(s)])`. The dumb-client
/// counterpart of [`protocol::Method::run_text`]; primarily for /// counterpart of [`protocol::Method::submit_text`]; primarily for
/// tests and tools that have only a string in hand. /// tests and tools that have only a string in hand.
pub async fn run_text(&mut self, s: impl Into<String>) -> Result<WorkerRunResult, WorkerError> pub async fn run_text(&mut self, s: impl Into<String>) -> Result<WorkerRunResult, WorkerError>
where where
@@ -3042,6 +3709,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
}, },
metadata: interrupt_metadata.clone(), metadata: interrupt_metadata.clone(),
}, },
extensions: Vec::new(),
})?; })?;
let interrupt_entry = let interrupt_entry =
HistoryEntry::new(agen::Item::system_message(system_note), interrupt_metadata); HistoryEntry::new(agen::Item::system_message(system_note), interrupt_metadata);
@@ -4428,6 +5096,22 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
{ {
initial_entries.push(checkpoint); initial_entries.push(checkpoint);
} }
initial_entries.push(LogEntry::Extension {
ts: segment_log::now_millis(),
domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(),
payload: serde_json::to_value(
&*self
.session
.pending_activations
.lock()
.expect("pending activation state poisoned"),
)
.map_err(|error| {
WorkerError::InvalidState(format!(
"serialize pending submissions during compaction: {error}"
))
})?,
});
if let Some(flow_state) = self if let Some(flow_state) = self
.flow_runtime_state .flow_runtime_state
.lock() .lock()
@@ -5248,6 +5932,9 @@ where
history_persistence_wired: false, history_persistence_wired: false,
log_writer: None, log_writer: None,
}; };
worker
.session
.restore_pending_activations(&state.extensions);
worker.apply_permissions_from_manifest(); worker.apply_permissions_from_manifest();
worker.apply_prune_from_manifest(); worker.apply_prune_from_manifest();
worker.write_worker_metadata_active(SegmentLocation { worker.write_worker_metadata_active(SegmentLocation {
@@ -8453,6 +9140,164 @@ mod build_summary_prompt_tests {
); );
} }
#[test]
fn pending_submission_queue_is_durable_idempotent_and_bounded() {
let temp = tempfile::tempdir().unwrap();
let handle = PendingSubmissionHandle::for_test(temp.path());
let input = vec![Segment::text("queued")];
let accepted = handle
.accept("request-1".into(), input.clone(), false)
.unwrap();
assert_eq!(
accepted.disposition,
protocol::SubmissionDisposition::Queued
);
assert_eq!(handle.snapshot().submissions.len(), 1);
let replay = handle
.accept("request-1".into(), input.clone(), false)
.unwrap();
assert_eq!(replay.submission_id, accepted.submission_id);
assert!(replay.activation.is_none());
assert_eq!(handle.snapshot().submissions.len(), 1);
assert!(matches!(
handle.accept("request-1".into(), vec![Segment::text("different")], false),
Err(PendingSubmissionError::IdempotencyConflict)
));
let entries = handle
.writer
.store
.read_all(
handle.writer.state.session_id(),
handle.writer.state.segment_id(),
)
.unwrap();
let payload = entries
.iter()
.rev()
.find_map(|entry| match entry {
LogEntry::Extension {
domain, payload, ..
} if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN => {
Some(payload.clone())
}
_ => None,
})
.unwrap();
let restored: PendingActivationState = serde_json::from_value(payload).unwrap();
assert_eq!(restored.pending.len(), 1);
assert_eq!(restored.pending[0].submission_id, accepted.submission_id);
let snapshot = handle.cancel(&accepted.submission_id).unwrap();
assert!(snapshot.submissions.is_empty());
assert!(matches!(
handle.cancel(&accepted.submission_id),
Err(PendingSubmissionError::NotFound(_))
));
for index in 0..MAX_PENDING_SUBMISSIONS {
handle
.accept(
format!("limit-{index}"),
vec![Segment::text(format!("value-{index}"))],
false,
)
.unwrap();
}
assert!(matches!(
handle.accept("over-limit".into(), vec![Segment::text("too much")], false),
Err(PendingSubmissionError::CountLimit)
));
assert_eq!(handle.snapshot().submissions.len(), MAX_PENDING_SUBMISSIONS);
let cleared = handle.clear().unwrap();
assert!(cleared.submissions.is_empty());
assert_eq!(cleared.notification_count, 0);
}
#[test]
fn notification_and_submit_share_activation_order_and_notification_dedupes() {
let temp = tempfile::tempdir().unwrap();
let handle = PendingSubmissionHandle::for_test(temp.path());
assert!(
handle
.accept_notification("notification-1".into(), "notice".into())
.unwrap()
);
assert!(
!handle
.accept_notification("notification-1".into(), "notice".into())
.unwrap()
);
assert!(matches!(
handle.accept_notification("notification-1".into(), "different".into()),
Err(PendingSubmissionError::IdempotencyConflict)
));
handle
.accept("request-1".into(), vec![Segment::text("submit")], false)
.unwrap();
let first = handle.prepare_next_activation().unwrap().unwrap();
assert!(matches!(
first,
PendingActivation::Notification(PendingNotification { ref message, .. })
if message == "notice"
));
let committed = handle.notification_activation_extension();
let committed_state: PendingActivationState =
serde_json::from_value(committed.payload).unwrap();
assert!(committed_state.pending_notifications.is_empty());
assert!(committed_state.activating_notification.is_none());
handle.finish_notification_activation("notification-1");
let second = handle.prepare_next_activation().unwrap().unwrap();
assert!(matches!(second, PendingActivation::Submission(_)));
}
#[test]
fn restoring_an_in_flight_activation_requeues_it_at_the_fifo_head() {
let mut session = WorkerSession::new(session_store::new_session_id(), Vec::new());
let state = PendingActivationState {
revision: 4,
next_activation_sequence: 2,
activating: Some(PendingSubmission {
submission_request_id: "request-1".into(),
submission_id: "submission-1".into(),
payload_digest: submission_payload_digest(&[Segment::text("first")]),
accepted_at_ms: 1,
activation_sequence: 0,
provenance: WorkerHistoryProvenance::LegacyUnknown,
was_queued: false,
input: vec![Segment::text("first")],
}),
activating_notification: None,
pending: VecDeque::from([PendingSubmission {
submission_request_id: "request-2".into(),
submission_id: "submission-2".into(),
payload_digest: submission_payload_digest(&[Segment::text("second")]),
accepted_at_ms: 2,
activation_sequence: 1,
provenance: WorkerHistoryProvenance::LegacyUnknown,
was_queued: true,
input: vec![Segment::text("second")],
}]),
pending_notifications: VecDeque::new(),
receipts: VecDeque::new(),
notification_receipts: VecDeque::new(),
};
session.restore_pending_activations(&[(
SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.into(),
serde_json::to_value(state).unwrap(),
)]);
let state = session
.pending_activations
.lock()
.expect("pending activation state poisoned");
assert!(state.activating.is_none());
assert_eq!(state.pending.len(), 2);
assert_eq!(state.pending[0].submission_id, "submission-1");
assert_eq!(state.pending[1].submission_id, "submission-2");
}
fn minimal_manifest() -> WorkerManifest { fn minimal_manifest() -> WorkerManifest {
let toml_str = r#" let toml_str = r#"
[worker] [worker]
+4 -1
View File
@@ -630,7 +630,10 @@ async fn controller_compact_method_emits_start_and_done() {
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle handle
.send(Method::run_text("seed history")) .send(Method::submit_text(
protocol::new_submission_request_id(),
"seed history",
))
.await .await
.expect("send run"); .expect("send run");
loop { loop {
+235 -57
View File
@@ -617,7 +617,13 @@ async fn feature_flags_default_to_core_tool_surface_only() {
let worker = make_worker(client).await; let worker = make_worker(client).await;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Idle).await; wait_for_status(&handle, WorkerStatus::Idle).await;
let request = wait_for_captured_request(&client_for_assert).await; let request = wait_for_captured_request(&client_for_assert).await;
@@ -672,7 +678,13 @@ permission = "write"
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0; let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Idle).await; wait_for_status(&handle, WorkerStatus::Idle).await;
let request = wait_for_captured_request(&client_for_assert).await; let request = wait_for_captured_request(&client_for_assert).await;
@@ -758,7 +770,13 @@ permission = "write"
let worker = make_worker_with_pwd_and_manifest(client, &manifest).await.0; let worker = make_worker_with_pwd_and_manifest(client, &manifest).await.0;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Idle).await; wait_for_status(&handle, WorkerStatus::Idle).await;
let request = wait_for_captured_request(&client_for_assert).await; let request = wait_for_captured_request(&client_for_assert).await;
@@ -814,7 +832,13 @@ async fn builtin_orchestrator_exposes_worker_remove_and_workdir_delete() {
.await; .await;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Idle).await; wait_for_status(&handle, WorkerStatus::Idle).await;
let request = wait_for_captured_request(&client_for_assert).await; let request = wait_for_captured_request(&client_for_assert).await;
let installed = request_tool_names(&request); let installed = request_tool_names(&request);
@@ -863,7 +887,13 @@ permission = "write"
.0; .0;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Idle).await; wait_for_status(&handle, WorkerStatus::Idle).await;
let request = wait_for_captured_request(&client_for_assert).await; let request = wait_for_captured_request(&client_for_assert).await;
@@ -916,7 +946,13 @@ permission = "write"
) )
.await; .await;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Idle).await; wait_for_status(&handle, WorkerStatus::Idle).await;
let request = wait_for_captured_request(&client_for_assert).await; let request = wait_for_captured_request(&client_for_assert).await;
let names = request_tool_names(&request); let names = request_tool_names(&request);
@@ -963,7 +999,13 @@ async fn run_end_returns_to_idle_without_busy_status() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
let mut saw_run_end = false; let mut saw_run_end = false;
let mut saw_idle_status = false; let mut saw_idle_status = false;
@@ -1005,7 +1047,13 @@ async fn provider_stream_error_records_run_errored() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::run_text("ping")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"ping",
))
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
@@ -1054,7 +1102,10 @@ async fn snapshot_includes_user_input_for_in_flight_turn() {
let mut events = handle.subscribe(); let mut events = handle.subscribe();
handle handle
.send(Method::run_text("hello in-flight")) .send(Method::submit_text(
protocol::new_submission_request_id(),
"hello in-flight",
))
.await .await
.unwrap(); .unwrap();
tokio::time::timeout(std::time::Duration::from_secs(2), async { tokio::time::timeout(std::time::Duration::from_secs(2), async {
@@ -1119,7 +1170,13 @@ async fn attach_snapshot_includes_current_status() {
let worker = make_worker(client).await; let worker = make_worker(client).await;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Running).await; wait_for_status(&handle, WorkerStatus::Running).await;
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path()) let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
@@ -1157,7 +1214,13 @@ async fn run_updates_shared_state_to_idle_after_completion() {
let worker = make_worker(client).await; let worker = make_worker(client).await;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
// Wait for the run to complete // Wait for the run to complete
tokio::time::sleep(std::time::Duration::from_millis(100)).await; tokio::time::sleep(std::time::Duration::from_millis(100)).await;
@@ -1171,7 +1234,13 @@ async fn run_populates_history() {
let worker = make_worker(client).await; let worker = make_worker(client).await;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
tokio::time::sleep(std::time::Duration::from_millis(100)).await; tokio::time::sleep(std::time::Duration::from_millis(100)).await;
@@ -1189,7 +1258,13 @@ async fn events_are_broadcast() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::run_text("Hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
let mut saw_turn_start = false; let mut saw_turn_start = false;
let mut saw_text_delta = false; let mut saw_text_delta = false;
@@ -1224,10 +1299,8 @@ async fn events_are_broadcast() {
} }
#[tokio::test] #[tokio::test]
async fn double_run_returns_error() { async fn submit_while_running_is_durably_queued() {
// Keep the first turn in-flight until the test drops the handle. A // Keep the first turn in-flight until the second Submit is accepted.
// finite stream can finish before the second Method reaches the
// controller in the full test suite, making this assertion racy.
let events = vec![ let events = vec![
LlmEvent::text_block_start(0), LlmEvent::text_block_start(0),
LlmEvent::text_delta(0, "slow..."), LlmEvent::text_delta(0, "slow..."),
@@ -1237,35 +1310,44 @@ async fn double_run_returns_error() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
// Send first run and wait until the controller has entered Running. handle
handle.send(Method::run_text("first")).await.unwrap(); .send(Method::submit_text("request-first", "first"))
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Running).await; wait_for_status(&handle, WorkerStatus::Running).await;
handle
.send(Method::submit_text("request-second", "second"))
.await
.unwrap();
// Now the second run must be rejected by drive_turn's live Method arm.
handle.send(Method::run_text("second")).await.unwrap();
// Look for the error event
let mut saw_already_running = false;
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop { let mut accepted = None;
tokio::select! { let mut pending_count = None;
event = rx.recv() => { while tokio::time::Instant::now() < deadline {
match event { match tokio::time::timeout(std::time::Duration::from_millis(100), rx.recv()).await {
Ok(Event::Error { code, .. }) => { Ok(Ok(Event::SubmissionAccepted {
if code == worker::ErrorCode::AlreadyRunning { submission_request_id,
saw_already_running = true; disposition,
..
})) if submission_request_id == "request-second" => accepted = Some(disposition),
Ok(Ok(Event::PendingSubmissionsChanged { pending }))
if pending.submissions.len() == 1 =>
{
pending_count = Some(1)
}
Ok(Ok(Event::Error { code, message })) if code == worker::ErrorCode::AlreadyRunning => {
panic!("Submit was busy-rejected: {message}")
}
_ => {}
}
if accepted.is_some() && pending_count.is_some() {
break; break;
} }
} }
Err(_) => break,
_ => {}
}
}
_ = tokio::time::sleep_until(deadline) => break,
}
}
assert!(saw_already_running, "should see already_running error"); assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued));
assert_eq!(pending_count, Some(1));
handle.send(Method::Pause).await.unwrap();
} }
#[tokio::test] #[tokio::test]
@@ -1353,7 +1435,8 @@ async fn run_with_paste_segment_inlines_content_and_emits_typed_user_message() {
protocol::Segment::text(" thanks"), protocol::Segment::text(" thanks"),
]; ];
handle handle
.send(Method::Run { .send(Method::Submit {
submission_request_id: protocol::new_submission_request_id(),
input: segments.clone(), input: segments.clone(),
}) })
.await .await
@@ -1425,7 +1508,13 @@ async fn run_with_resolvable_file_ref_attaches_system_message_after_user() {
path: "notes.md".into(), path: "notes.md".into(),
}, },
]; ];
handle.send(Method::Run { input: segments }).await.unwrap(); handle
.send(Method::Submit {
submission_request_id: protocol::new_submission_request_id(),
input: segments,
})
.await
.unwrap();
// Wait for the turn to complete. // Wait for the turn to complete.
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
@@ -1473,7 +1562,8 @@ async fn run_with_file_ref_uses_manifest_file_upload_limit() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
handle handle
.send(Method::Run { .send(Method::Submit {
submission_request_id: protocol::new_submission_request_id(),
input: vec![protocol::Segment::FileRef { input: vec![protocol::Segment::FileRef {
path: "long.txt".into(), path: "long.txt".into(),
}], }],
@@ -1526,7 +1616,13 @@ async fn run_with_unresolved_segment_emits_alert_and_placeholder() {
path: "src/lib.rs".into(), path: "src/lib.rs".into(),
}, },
]; ];
handle.send(Method::Run { input: segments }).await.unwrap(); handle
.send(Method::Submit {
submission_request_id: protocol::new_submission_request_id(),
input: segments,
})
.await
.unwrap();
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
let mut saw_alert_for_file_ref = false; let mut saw_alert_for_file_ref = false;
@@ -1574,6 +1670,7 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() {
handle handle
.send(Method::Notify { .send(Method::Notify {
notification_request_id: protocol::new_submission_request_id(),
message: "turn finished".into(), message: "turn finished".into(),
auto_run: true, auto_run: true,
}) })
@@ -1614,6 +1711,19 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() {
saw_notify_in_mirror, saw_notify_in_mirror,
"Method::Notify should commit a SystemItem::Notification entry; mirror = {entries:?}" "Method::Notify should commit a SystemItem::Notification entry; mirror = {entries:?}"
); );
let queue_checkpoint_is_atomic = entries.iter().any(|entry| match entry {
LogEntry::AnnotatedSystemItem { extensions, .. } => extensions.iter().any(|extension| {
extension.domain == "worker.pending_activations.v1"
&& extension.payload["pending_notifications"]
.as_array()
.is_some_and(Vec::is_empty)
}),
_ => false,
});
assert!(
queue_checkpoint_is_atomic,
"notification history and queue claim must share one log entry"
);
// Exactly one request was made; it must contain the formatted // Exactly one request was made; it must contain the formatted
// notification as one of the items (committed to history by // notification as one of the items (committed to history by
@@ -1662,6 +1772,7 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
handle handle
.send(Method::Notify { .send(Method::Notify {
notification_request_id: protocol::new_submission_request_id(),
message: "progress snapshot".into(), message: "progress snapshot".into(),
auto_run: false, auto_run: false,
}) })
@@ -1675,7 +1786,13 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
"weak Notify must not stage RunForNotification while idle" "weak Notify must not stage RunForNotification while idle"
); );
handle.send(Method::run_text("continue")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"continue",
))
.await
.unwrap();
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop { loop {
if !client_for_assert.captured_requests().is_empty() { if !client_for_assert.captured_requests().is_empty() {
@@ -1855,9 +1972,16 @@ async fn notify_while_running_does_not_emit_already_running_error() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::run_text("start")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"start",
))
.await
.unwrap();
handle handle
.send(Method::Notify { .send(Method::Notify {
notification_request_id: protocol::new_submission_request_id(),
message: "ping".into(), message: "ping".into(),
auto_run: true, auto_run: true,
}) })
@@ -1924,7 +2048,13 @@ async fn socket_run_receives_events() {
let mut writer = JsonLineWriter::new(writer); let mut writer = JsonLineWriter::new(writer);
// Send run method via socket // Send run method via socket
writer.write(&Method::run_text("Hello")).await.unwrap(); writer
.write(&Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
.await
.unwrap();
// Collect events // Collect events
let mut saw_turn_start = false; let mut saw_turn_start = false;
@@ -2231,7 +2361,13 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::run_text("hello")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"hello",
))
.await
.unwrap();
// Wait for the partial text_delta to confirm the first stream is // Wait for the partial text_delta to confirm the first stream is
// live before we pause. // live before we pause.
@@ -2320,7 +2456,7 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
assert!(!has_tool_call, "no orphan tool_call in history"); assert!(!has_tool_call, "no orphan tool_call in history");
} }
/// Paused with an orphan `tool_use` in history + a fresh `Method::Run` /// Paused with an orphan `tool_use` in history + a fresh `Method::Submit`
/// must produce a wire-valid next LLM request: the orphan is closed /// must produce a wire-valid next LLM request: the orphan is closed
/// with a synthetic `tool_result`, a system note is inserted, and the /// with a synthetic `tool_result`, a system note is inserted, and the
/// new user input is appended. /// new user input is appended.
@@ -2357,7 +2493,13 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::run_text("first")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"first",
))
.await
.unwrap();
// Wait for ToolCallDone — the ToolCall is committed to history // Wait for ToolCallDone — the ToolCall is committed to history
// right before the Engine enters tool execution and pends. // right before the Engine enters tool execution and pends.
@@ -2388,7 +2530,13 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
// `last_run_interrupted` and runs its interrupt-prep step, which // `last_run_interrupted` and runs its interrupt-prep step, which
// closes the orphan + injects a system note before the fresh user // closes the orphan + injects a system note before the fresh user
// message. // message.
handle.send(Method::run_text("new request")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"new request",
))
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
e, e,
@@ -2519,7 +2667,13 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::run_text("first")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"first",
))
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
e, e,
@@ -2587,7 +2741,10 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
); );
handle handle
.send(Method::run_text("fresh request")) .send(Method::submit_text(
protocol::new_submission_request_id(),
"fresh request",
))
.await .await
.unwrap(); .unwrap();
assert!( assert!(
@@ -2676,7 +2833,13 @@ async fn empty_turn_cancel_rolls_back_submit_entries_and_emits_signal() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::run_text("rollback me")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"rollback me",
))
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Running).await; wait_for_status(&handle, WorkerStatus::Running).await;
handle.send(Method::Cancel).await.unwrap(); handle.send(Method::Cancel).await.unwrap();
@@ -2709,7 +2872,10 @@ async fn empty_turn_pause_rolls_back_and_snapshot_does_not_restore_input() {
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle handle
.send(Method::run_text("pause rollback")) .send(Method::submit_text(
protocol::new_submission_request_id(),
"pause rollback",
))
.await .await
.unwrap(); .unwrap();
wait_for_status(&handle, WorkerStatus::Running).await; wait_for_status(&handle, WorkerStatus::Running).await;
@@ -2743,7 +2909,13 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() {
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::run_text("first kept")).await.unwrap(); handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"first kept",
))
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
e, e,
@@ -2757,7 +2929,10 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() {
wait_for_status(&handle, WorkerStatus::Idle).await; wait_for_status(&handle, WorkerStatus::Idle).await;
handle handle
.send(Method::run_text("second rolled back")) .send(Method::submit_text(
protocol::new_submission_request_id(),
"second rolled back",
))
.await .await
.unwrap(); .unwrap();
wait_for_status(&handle, WorkerStatus::Running).await; wait_for_status(&handle, WorkerStatus::Running).await;
@@ -2804,7 +2979,10 @@ async fn pause_after_assistant_token_does_not_rollback() {
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle handle
.send(Method::run_text("keep this turn")) .send(Method::submit_text(
protocol::new_submission_request_id(),
"keep this turn",
))
.await .await
.unwrap(); .unwrap();
assert!( assert!(
+3 -3
View File
@@ -533,7 +533,7 @@ fn initial_worker_input(segments: &[Segment]) -> Option<EmbeddedWorkerInput> {
Some(EmbeddedWorkerInput { Some(EmbeddedWorkerInput {
kind: EmbeddedWorkerInputKind::User, kind: EmbeddedWorkerInputKind::User,
content: Segment::flatten_to_text(segments), content: Segment::flatten_to_text(segments),
submission_id: None, submission_request_id: None,
segments: Some(segments.to_vec()), segments: Some(segments.to_vec()),
}) })
} }
@@ -2625,7 +2625,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer, WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
}, },
content: request.content, content: request.content,
submission_id: None, submission_request_id: None,
segments: request.segments, segments: request.segments,
}; };
match self.runtime.send_input(&worker_ref, input) { match self.runtime.send_input(&worker_ref, input) {
@@ -3726,7 +3726,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer, WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
}, },
content: request.content, content: request.content,
submission_id: None, submission_request_id: None,
segments: request.segments, segments: request.segments,
}; };
match self.post_json::<_, RuntimeHttpWorkerInputResponse>( match self.post_json::<_, RuntimeHttpWorkerInputResponse>(
+1 -1
View File
@@ -286,7 +286,7 @@ User triggers a Ticket action in yoi panel
-> client Ticket role launcher reads .yoi/workspace.toml [ticket] settings -> client Ticket role launcher reads .yoi/workspace.toml [ticket] settings
-> launcher selects the role Profile -> launcher selects the role Profile
-> launcher spawns the role Worker -> launcher spawns the role Worker
-> launcher sends Method::Run with Text segments -> launcher sends Method::Submit with Text segments
-> launcher waits for run-acceptance evidence -> launcher waits for run-acceptance evidence
-> Dashboard reports success/failure -> Dashboard reports success/failure
``` ```
+1 -1
View File
@@ -34,7 +34,7 @@
- いくつかの async/socket tests はまだ fixed sleeps と process-wide environment mutation (`YOI_RUNTIME_DIR`, `YOI_HOME`, `XDG_RUNTIME_DIR`) に依存している。一部ファイルでは env-changing tests を guards で serialize しているが、fixed timing と global env は高負荷 CI や parallel execution 下で flakiness risk のまま。 - いくつかの async/socket tests はまだ fixed sleeps と process-wide environment mutation (`YOI_RUNTIME_DIR`, `YOI_HOME`, `XDG_RUNTIME_DIR`) に依存している。一部ファイルでは env-changing tests を guards で serialize しているが、fixed timing と global env は高負荷 CI や parallel execution 下で flakiness risk のまま。
- Real provider wire behavior は意図的に `pod` の外側だが、crate と streaming edge cases の interaction はまだ大部分が mock されている。tests は重要な `Worker` outcomes をカバーしているが、malformed/partial provider streams を real provider adapter 経由では exercise していない。 - Real provider wire behavior は意図的に `pod` の外側だが、crate と streaming edge cases の interaction はまだ大部分が mock されている。tests は重要な `Worker` outcomes をカバーしているが、malformed/partial provider streams を real provider adapter 経由では exercise していない。
- Prompt tests は数が多く有用だが、一部は behavior-coupled というより prose-coupled である。critical safety wording に対して、その文字列を意図的に stable contract として扱う場合だけこれは許容できる。そうでなければ maintenance noise になる。 - Prompt tests は数が多く有用だが、一部は behavior-coupled というより prose-coupled である。critical safety wording に対して、その文字列を意図的に stable contract として扱う場合だけこれは許容できる。そうでなければ maintenance noise になる。
- startup profile resolution、socket server、`Method::Run`、session persistence、shutdown、restore を跨ぐ full lifecycle integration は slice ごとにしかカバーされておらず、1 つの scenario としてはカバーされていない。 - startup profile resolution、socket server、`Method::Submit`、session persistence、shutdown、restore を跨ぐ full lifecycle integration は slice ごとにしかカバーされておらず、1 つの scenario としてはカバーされていない。
## 追加を提案するもの ## 追加を提案するもの
+9 -3
View File
@@ -103,7 +103,13 @@ entry_id: string,
*/ */
timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array<string>, } & ({ "kind": "user_input", segments: Array<Segment>, } | { "kind": "message", role: SessionMessageRole, content: Array<SessionContentPart>, } | { "kind": "tool_call", call_id: string, name: string, arguments: string, } | { "kind": "tool_result", call_id: string, summary: string, content?: string | null, is_error: boolean, attachments?: Array<SessionToolAttachment>, } | { "kind": "system_item", item_kind: string, content: string, data?: unknown, } | { "kind": "run_error", message: string, }); timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array<string>, } & ({ "kind": "user_input", segments: Array<Segment>, } | { "kind": "message", role: SessionMessageRole, content: Array<SessionContentPart>, } | { "kind": "tool_call", call_id: string, name: string, arguments: string, } | { "kind": "tool_result", call_id: string, summary: string, content?: string | null, is_error: boolean, attachments?: Array<SessionToolAttachment>, } | { "kind": "system_item", item_kind: string, content: string, data?: unknown, } | { "kind": "run_error", message: string, });
export type SessionSnapshot = { entries: Array<SessionSnapshotEntry>, }; export type PendingSubmissionSummary = { submission_id: string, accepted_at_ms: number, segment_count: number, byte_len: number, };
export type PendingSubmissionsSnapshot = { revision: number, notification_count: number, submissions: Array<PendingSubmissionSummary>, };
export type SubmissionDisposition = "started" | "queued";
export type SessionSnapshot = { pending_submissions: PendingSubmissionsSnapshot, entries: Array<SessionSnapshotEntry>, };
export type InternalWorkerKind = "sub_worker" | { "service": { kind: string, } }; export type InternalWorkerKind = "sub_worker" | { "service": { kind: string, } };
@@ -225,9 +231,9 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip
export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod }); export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod });
export type Method = { "method": "run", "params": { input: Array<Segment>, } } | { "method": "notify", "params": { message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, } } | { "method": "clear_pending_submissions" } | { "method": "continue_pending" } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
export type Event = { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number, export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number,
/** /**
* The attempt that just failed. 1 origin. * The attempt that just failed. 1 origin.
*/ */
@@ -2150,6 +2150,7 @@ Deno.test("snapshot restores TaskStore state from system history", () => {
const event = snapshotEvent("/repo"); const event = snapshotEvent("/repo");
if (event.event !== "snapshot") throw new Error("snapshot fixture expected"); if (event.event !== "snapshot") throw new Error("snapshot fixture expected");
event.data.session = { event.data.session = {
pending_submissions: { revision: 0, notification_count: 0, submissions: [] },
entries: [{ entries: [{
entry_id: "task-reminder-1", entry_id: "task-reminder-1",
timestamp: 1, timestamp: 1,
@@ -1059,3 +1059,25 @@ Deno.test("Web Console switches main and direct SubWorker views from the Tasks r
"Worker view selection should expose only direct SubWorker session identities with main fallback", "Worker view selection should expose only direct SubWorker session identities with main fallback",
); );
}); });
Deno.test("Web Console uses Notify while running and exposes durable pending controls", async () => {
const consolePage = await Deno.readTextFile(
new URL(
"./../../../routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte",
import.meta.url,
),
);
for (const token of [
'method: "submit"',
'method: "notify"',
"notification_request_id: crypto.randomUUID()",
"submission_request_id: crypto.randomUUID()",
'payload.event === "pending_submissions_changed"',
'method: "cancel_pending_submission"',
'method: "clear_pending_submissions"',
'method: "continue_pending"',
]) {
assert(consolePage.includes(token), `missing durable pending control token: ${token}`);
}
});
@@ -31,7 +31,13 @@
type ConsoleViewMode, type ConsoleViewMode,
type ConsoleViewScroll, type ConsoleViewScroll,
} from "$lib/workspace/console/model"; } from "$lib/workspace/console/model";
import type { Event as ProtocolEvent, Method as ProtocolMethod, RewindTarget, Segment } from "$lib/generated/protocol"; import type {
Event as ProtocolEvent,
Method as ProtocolMethod,
PendingSubmissionsSnapshot,
RewindTarget,
Segment,
} from "$lib/generated/protocol";
import { import {
MAX_FILES_PER_SUBMISSION, MAX_FILES_PER_SUBMISSION,
uploadAttachment, uploadAttachment,
@@ -152,6 +158,12 @@
"connecting", "connecting",
); );
let protocolSubscription: WorkspaceMultiplexerSubscription | null = null; let protocolSubscription: WorkspaceMultiplexerSubscription | null = null;
let pendingSubmissions = $state<PendingSubmissionsSnapshot>({
revision: 0,
notification_count: 0,
submissions: [],
});
let pendingSubmissionItems = $derived(pendingSubmissions.submissions ?? []);
let pendingCompletionRequest: { let pendingCompletionRequest: {
resolve: (entries: ComposerCompletionEntry[]) => void; resolve: (entries: ComposerCompletionEntry[]) => void;
reject: (error: Error) => void; reject: (error: Error) => void;
@@ -334,6 +346,13 @@
function handleIncomingProtocolEvent(payload: ProtocolEvent) { function handleIncomingProtocolEvent(payload: ProtocolEvent) {
handleProtocolCommandEvent(payload); handleProtocolCommandEvent(payload);
if (payload.event === "snapshot") {
pendingSubmissions = payload.data.session.pending_submissions;
} else if (payload.event === "segment_rotated") {
pendingSubmissions = payload.data.session.pending_submissions;
} else if (payload.event === "pending_submissions_changed") {
pendingSubmissions = payload.data.pending;
}
if (payload.event === "error") { if (payload.event === "error") {
queueObservationDiagnostic({ queueObservationDiagnostic({
code: payload.data.code, code: payload.data.code,
@@ -555,9 +574,20 @@
): ProtocolMethod { ): ProtocolMethod {
switch (request.kind) { switch (request.kind) {
case "user": case "user":
if (workerRunning) {
return { return {
method: "run", method: "notify",
params: { params: {
notification_request_id: crypto.randomUUID(),
message: request.content,
auto_run: true,
},
};
}
return {
method: "submit",
params: {
submission_request_id: crypto.randomUUID(),
input: request.segments ?? [ input: request.segments ?? [
{ kind: "text", content: request.content }, { kind: "text", content: request.content },
], ],
@@ -566,7 +596,11 @@
case "notify": case "notify":
return { return {
method: "notify", method: "notify",
params: { message: request.content, auto_run: true }, params: {
notification_request_id: crypto.randomUUID(),
message: request.content,
auto_run: true,
},
}; };
case "compact": case "compact":
return { method: "compact" }; return { method: "compact" };
@@ -779,7 +813,7 @@
composerInputElement?.recordHistory(value); composerInputElement?.recordHistory(value);
composerInputElement?.clear(); composerInputElement?.clear();
attachments = []; attachments = [];
if (method.method === "run" || method.method === "notify") { if (method.method === "submit" || method.method === "notify") {
liveWorkerState = "running"; liveWorkerState = "running";
} }
composerNotice = "Sent through Worker protocol."; composerNotice = "Sent through Worker protocol.";
@@ -1722,6 +1756,50 @@
</aside> </aside>
{/if} {/if}
{#if pendingSubmissionItems.length > 0 || pendingSubmissions.notification_count > 0}
<details class="pending-submissions">
<summary>
Pending activations ({pendingSubmissionItems.length} submissions · {pendingSubmissions.notification_count} notifications)
</summary>
<ol>
{#each pendingSubmissionItems as submission (submission.submission_id)}
<li>
<code>{submission.submission_id}</code>
<span>{submission.segment_count} segments · {submission.byte_len} bytes</span>
<button
type="button"
onclick={() =>
sendControl(
{
method: "cancel_pending_submission",
params: { submission_id: submission.submission_id },
},
"Pending submission cancellation",
)}
>Cancel</button>
</li>
{/each}
</ol>
<button
type="button"
disabled={workerRunning}
onclick={() =>
sendControl(
{ method: "continue_pending" },
"Pending activation continue",
)}
>Continue next</button>
<button
type="button"
onclick={() =>
sendControl(
{ method: "clear_pending_submissions" },
"Pending submissions clear",
)}
>Clear all</button>
</details>
{/if}
{#if workerRunning} {#if workerRunning}
<WorkerRunStatus <WorkerRunStatus
startedAtMs={consoleProjection.runActivity.startedAtMs} startedAtMs={consoleProjection.runActivity.startedAtMs}
@@ -2035,6 +2113,31 @@
display: none; display: none;
} }
.pending-submissions {
margin: 0 var(--space-3);
color: var(--muted);
font-size: 0.75rem;
}
.pending-submissions ol {
display: grid;
gap: var(--space-1);
margin: var(--space-2) 0;
padding-left: var(--space-5);
}
.pending-submissions li {
display: flex;
gap: var(--space-2);
align-items: center;
}
.pending-submissions code {
max-width: 16rem;
overflow: hidden;
text-overflow: ellipsis;
}
.console-log { .console-log {
display: grid; display: grid;
align-content: start; align-content: start;