Merge branch 'develop' into work/T-595-runtime-trust-key-management
# Conflicts: # web/workspace/deno.json
This commit is contained in:
Generated
+1
@@ -6650,6 +6650,7 @@ dependencies = [
|
||||
"serial_test",
|
||||
"session-metrics",
|
||||
"session-store",
|
||||
"sha2 0.11.0",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"ticket",
|
||||
|
||||
@@ -120,12 +120,15 @@ mod tests {
|
||||
let mut client = Client::new(socket);
|
||||
|
||||
client
|
||||
.send(&Method::run_text("hello"))
|
||||
.send(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"hello",
|
||||
))
|
||||
.await
|
||||
.expect("send method");
|
||||
assert!(matches!(
|
||||
decode_method(&client.socket.sent[0]),
|
||||
Ok(Method::Run { .. })
|
||||
Ok(Method::Submit { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
client.next_event().await,
|
||||
|
||||
@@ -89,12 +89,15 @@ mod tests {
|
||||
let mut client = Client::new(socket);
|
||||
|
||||
client
|
||||
.send(&Method::run_text("hello"))
|
||||
.send(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"hello",
|
||||
))
|
||||
.await
|
||||
.expect("send method");
|
||||
assert!(matches!(
|
||||
peer.next().await.as_deref().map(decode_method),
|
||||
Some(Ok(Method::Run { .. }))
|
||||
Some(Ok(Method::Submit { .. }))
|
||||
));
|
||||
|
||||
peer.send(
|
||||
|
||||
@@ -147,12 +147,18 @@ mod tests {
|
||||
|
||||
let mut client = Client::new(Socket::connect(&socket_path).await.unwrap());
|
||||
client
|
||||
.send(&Method::run_text("hello"))
|
||||
.send(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"hello",
|
||||
))
|
||||
.await
|
||||
.expect("send method");
|
||||
|
||||
let received = server.await.unwrap().expect("method message");
|
||||
assert!(matches!(decode_method(&received), Ok(Method::Run { .. })));
|
||||
assert!(matches!(
|
||||
decode_method(&received),
|
||||
Ok(Method::Submit { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -114,7 +114,7 @@ mod tests {
|
||||
assert!(matches!(
|
||||
message,
|
||||
Message::Text(ref text)
|
||||
if matches!(decode_method(text), Ok(Method::Run { .. }))
|
||||
if matches!(decode_method(text), Ok(Method::Submit { .. }))
|
||||
));
|
||||
let event = encode_event(&Event::Status {
|
||||
status: WorkerStatus::Idle,
|
||||
@@ -126,7 +126,10 @@ mod tests {
|
||||
let request = format!("ws://{address}").into_client_request().unwrap();
|
||||
let mut client = Client::new(Socket::connect(request).await.unwrap());
|
||||
client
|
||||
.send(&Method::run_text("hello"))
|
||||
.send(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"hello",
|
||||
))
|
||||
.await
|
||||
.expect("send method");
|
||||
assert!(matches!(
|
||||
|
||||
+211
-52
@@ -11,6 +11,11 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
pub use identity::{WorkerId, WorkerIdParseError};
|
||||
|
||||
/// Allocate an opaque idempotency key for one client Submit request.
|
||||
pub fn new_submission_request_id() -> String {
|
||||
uuid::Uuid::now_v7().to_string()
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
@@ -27,21 +32,80 @@ fn is_false(value: &bool) -> bool {
|
||||
// Method (Client → Worker via Unix Socket)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Trusted Server → Runtime transport header carrying the authenticated
|
||||
/// browser Account identity for one Worker protocol connection.
|
||||
///
|
||||
/// Runtime accepts this only after its normal HTTP authentication succeeds;
|
||||
/// serialized [`Method`] payloads cannot set authenticated source identity.
|
||||
pub const AUTHENTICATED_ACCOUNT_ID_HEADER: &str = "x-yoi-authenticated-account-id";
|
||||
|
||||
/// Trusted source identity attached by an authenticated transport boundary.
|
||||
///
|
||||
/// Public clients cannot select this value directly. Runtime/Backend adapters
|
||||
/// stamp it before forwarding an accepted Submit or Notify to a Worker.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum AuthenticatedInputSource {
|
||||
/// Assigned whenever a serialized tracked method crosses an untrusted
|
||||
/// protocol boundary. Receivers must handle it exactly like public input.
|
||||
UntrustedWire,
|
||||
Account {
|
||||
account_id: String,
|
||||
},
|
||||
Worker {
|
||||
runtime_id: String,
|
||||
worker_id: String,
|
||||
},
|
||||
SubWorker {
|
||||
session_id: String,
|
||||
},
|
||||
Backend {
|
||||
operation_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for AuthenticatedInputSource {
|
||||
fn default() -> Self {
|
||||
Self::UntrustedWire
|
||||
}
|
||||
}
|
||||
|
||||
impl AuthenticatedInputSource {
|
||||
pub fn namespace(&self) -> String {
|
||||
match self {
|
||||
Self::UntrustedWire => "untrusted-wire".into(),
|
||||
Self::Account { account_id } => format!("account:{account_id}"),
|
||||
Self::Worker {
|
||||
runtime_id,
|
||||
worker_id,
|
||||
} => format!("worker:{runtime_id}:{worker_id}"),
|
||||
Self::SubWorker { session_id } => format!("sub_worker:{session_id}"),
|
||||
Self::Backend { operation_id } => format!("backend:{operation_id}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(tag = "method", content = "params", rename_all = "snake_case")]
|
||||
pub enum Method {
|
||||
Run {
|
||||
/// Durably accept typed input for immediate activation or the session FIFO.
|
||||
///
|
||||
/// `submission_request_id` is generated by the authenticated caller and is
|
||||
/// used only for idempotent retry. Worker allocates the durable
|
||||
/// `submission_id` returned by [`Event::SubmissionAccepted`].
|
||||
Submit {
|
||||
submission_request_id: String,
|
||||
input: Vec<Segment>,
|
||||
},
|
||||
/// Runtime-internal Run carrying an opaque correlation id that is committed
|
||||
/// with the resulting UserInput entry. This variant is not serializable on
|
||||
/// the public Client → Worker protocol.
|
||||
#[serde(skip)]
|
||||
/// Authenticated transport form of Submit. Trusted adapters replace
|
||||
/// public Submit before forwarding it to the Worker.
|
||||
#[cfg_attr(feature = "typescript", ts(skip))]
|
||||
RunTracked {
|
||||
SubmitTracked {
|
||||
submission_request_id: String,
|
||||
input: Vec<Segment>,
|
||||
submission_id: String,
|
||||
#[serde(skip_deserializing, default)]
|
||||
source: AuthenticatedInputSource,
|
||||
},
|
||||
/// Human-readable text injected into the target Worker's LLM context
|
||||
/// as a non-blocking system message. `auto_run` controls whether an
|
||||
@@ -50,25 +114,54 @@ pub enum Method {
|
||||
/// No side effects beyond LLM context; use `WorkerEvent` for typed
|
||||
/// lifecycle reports.
|
||||
Notify {
|
||||
notification_request_id: String,
|
||||
message: String,
|
||||
#[serde(default = "default_true", skip_serializing_if = "is_true")]
|
||||
auto_run: bool,
|
||||
},
|
||||
/// Authenticated transport form of Notify.
|
||||
#[cfg_attr(feature = "typescript", ts(skip))]
|
||||
NotifyTracked {
|
||||
notification_request_id: String,
|
||||
message: String,
|
||||
#[serde(default = "default_true", skip_serializing_if = "is_true")]
|
||||
auto_run: bool,
|
||||
#[serde(skip_deserializing, default)]
|
||||
source: AuthenticatedInputSource,
|
||||
},
|
||||
/// Typed lifecycle report from a child Worker to its direct parent.
|
||||
WorkerEvent(WorkerEvent),
|
||||
/// Return the authoritative FIFO summary without exposing queued payloads.
|
||||
ListPendingSubmissions,
|
||||
/// Remove one queued submission. Running or already activated submissions
|
||||
/// are immutable and therefore cannot be cancelled here.
|
||||
CancelPendingSubmission {
|
||||
submission_id: String,
|
||||
expected_revision: u64,
|
||||
},
|
||||
/// Remove every queued submission while preserving the active run.
|
||||
ClearPendingSubmissions {
|
||||
expected_revision: u64,
|
||||
},
|
||||
/// Activate the next queued submission while the Worker is idle. This is an
|
||||
/// explicit recovery operation and never resumes a paused run implicitly.
|
||||
ContinuePending {
|
||||
expected_revision: u64,
|
||||
expected_head_id: String,
|
||||
},
|
||||
Resume,
|
||||
Cancel,
|
||||
/// Stop the in-flight turn and transition to `Paused`.
|
||||
///
|
||||
/// Unlike `Cancel` (which discards and returns to `Idle`), a paused
|
||||
/// Worker can resume the interrupted work via `Resume`, or start a
|
||||
/// fresh turn via `Run` (orphan `tool_use` items are closed with a
|
||||
/// Worker can resume the interrupted work via `Resume`, or accept a
|
||||
/// fresh `Submit` (orphan `tool_use` items are closed with a
|
||||
/// synthetic tool result before the new user message is appended).
|
||||
Pause,
|
||||
/// Request an explicit compaction while the Worker is otherwise idle.
|
||||
///
|
||||
/// This is a typed control method: clients must not send `compact` as a
|
||||
/// `Method::Run` user message.
|
||||
/// `Method::Submit` user message.
|
||||
Compact,
|
||||
/// Ask the Worker to list valid rewind targets from its authoritative session log.
|
||||
ListRewindTargets,
|
||||
@@ -181,7 +274,7 @@ impl WorkerEvent {
|
||||
|
||||
/// One typed piece of a user submission.
|
||||
///
|
||||
/// `Method::Run` and `Event::UserMessage` carry `Vec<Segment>`. Dumb
|
||||
/// `Method::Submit` and `Event::UserMessage` carry `Vec<Segment>`. Dumb
|
||||
/// clients (CLI piping, scripts) only need to produce a single
|
||||
/// `Segment::Text`; richer clients (TUI / GUI) construct typed atoms
|
||||
/// (paste chips, file refs) and
|
||||
@@ -404,12 +497,13 @@ impl Segment {
|
||||
}
|
||||
|
||||
impl Method {
|
||||
/// Convenience: a `Run` carrying a single `Segment::Text`.
|
||||
/// Convenience: a `Submit` carrying a single `Segment::Text`.
|
||||
/// Used by dumb clients, inter-Worker tools, and tests that only have
|
||||
/// a string to forward.
|
||||
pub fn run_text(s: impl Into<String>) -> Self {
|
||||
Self::Run {
|
||||
input: vec![Segment::text(s)],
|
||||
pub fn submit_text(submission_request_id: impl Into<String>, text: impl Into<String>) -> Self {
|
||||
Self::Submit {
|
||||
submission_request_id: submission_request_id.into(),
|
||||
input: vec![Segment::text(text)],
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -503,6 +597,39 @@ pub enum ToolResultDisposition {
|
||||
OutcomeUnknown,
|
||||
}
|
||||
|
||||
/// Durable acceptance result for one idempotent Submit request.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SubmissionDisposition {
|
||||
Started,
|
||||
Queued,
|
||||
}
|
||||
|
||||
/// Bounded public projection of one pending submission. Payload segments and
|
||||
/// provenance remain in the session log and are intentionally not exposed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
pub struct PendingSubmissionSummary {
|
||||
pub submission_id: String,
|
||||
pub accepted_at_ms: u64,
|
||||
pub segment_count: u32,
|
||||
pub byte_len: u64,
|
||||
}
|
||||
|
||||
/// Revisioned session-owned FIFO projection used by snapshots and live events.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
pub struct PendingSubmissionsSnapshot {
|
||||
pub revision: u64,
|
||||
#[serde(default)]
|
||||
pub notification_count: u32,
|
||||
#[serde(default)]
|
||||
pub head_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub submissions: Vec<PendingSubmissionSummary>,
|
||||
}
|
||||
|
||||
/// Canonical, storage-independent projection of committed session history.
|
||||
///
|
||||
/// Worker protocols expose this DTO instead of append-log records. New
|
||||
@@ -511,6 +638,8 @@ pub enum ToolResultDisposition {
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
pub struct SessionSnapshot {
|
||||
#[serde(default)]
|
||||
pub pending_submissions: PendingSubmissionsSnapshot,
|
||||
pub entries: Vec<SessionSnapshotEntry>,
|
||||
}
|
||||
|
||||
@@ -609,16 +738,27 @@ pub struct SessionToolAttachment {
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(tag = "event", content = "data", rename_all = "snake_case")]
|
||||
pub enum Event {
|
||||
/// A user input message was accepted, persisted as
|
||||
/// `LogEntry::AnnotatedUserInput`, and is about to start a new turn.
|
||||
/// Broadcast to every subscribed client so TUI / GUI instances show
|
||||
/// the same user line that reconnect snapshots would replay from
|
||||
/// history; clients must not synthesize a separate pending/fake
|
||||
/// message for accepted runs.
|
||||
///
|
||||
/// Fires exactly once per committed user input, after
|
||||
/// `InvokeStart { kind: UserSend }` and before the first
|
||||
/// `TurnStart`. Rejected runs (e.g. `AlreadyRunning`) do not emit.
|
||||
/// Durable Submit acceptance. A `Started` receipt follows the atomic
|
||||
/// UserInput commit; a `Queued` receipt follows the durable FIFO checkpoint.
|
||||
/// Repeating the same request id and exact payload returns the same receipt
|
||||
/// without appending or activating twice.
|
||||
SubmissionAccepted {
|
||||
submission_request_id: String,
|
||||
submission_id: String,
|
||||
disposition: SubmissionDisposition,
|
||||
},
|
||||
/// Correlated rejection before durable acceptance.
|
||||
SubmissionRejected {
|
||||
submission_request_id: String,
|
||||
message: String,
|
||||
},
|
||||
/// Revisioned FIFO replacement following enqueue, activation, cancel, or clear.
|
||||
PendingSubmissionsChanged {
|
||||
pending: PendingSubmissionsSnapshot,
|
||||
},
|
||||
/// A user input message persisted as `LogEntry::AnnotatedUserInput` and
|
||||
/// activated for a turn. Broadcast to every subscribed client so TUI / GUI
|
||||
/// instances show the same user line that reconnect snapshots replay.
|
||||
UserMessage {
|
||||
segments: Vec<Segment>,
|
||||
},
|
||||
@@ -641,7 +781,7 @@ pub enum Event {
|
||||
///
|
||||
/// Marker event for the start of an Invoke range; the range extends
|
||||
/// implicitly until the next `InvokeStart`. Fires for every accepted
|
||||
/// `Method::Run` (kind=`UserSend`), `Method::Notify` (kind=`Notify`),
|
||||
/// `Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`),
|
||||
/// `Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other
|
||||
/// IDLE-breaking trigger. Mid-run interrupts (e.g. hook output,
|
||||
/// typed system reminder insertion that doesn't break IDLE) do not
|
||||
@@ -1193,7 +1333,7 @@ pub enum TurnResult {
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum InvokeKind {
|
||||
/// `Method::Run` — a user submission.
|
||||
/// `Method::Submit` — a user submission.
|
||||
UserSend,
|
||||
/// `Method::Notify` — free-text notification injected into history.
|
||||
Notify,
|
||||
@@ -1216,7 +1356,7 @@ pub enum RunResult {
|
||||
Finished,
|
||||
Paused,
|
||||
LimitReached,
|
||||
/// The accepted Method::Run produced no assistant/tool output before
|
||||
/// The accepted Method::Submit produced no assistant/tool output before
|
||||
/// user interruption, so the Worker rolled the submit-time turn state back
|
||||
/// to its pre-submit snapshot. Clients should treat the Worker as Idle and
|
||||
/// restore the just-submitted input into the editable composer if desired.
|
||||
@@ -1285,26 +1425,30 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn method_run_json_roundtrip() {
|
||||
let json = r#"{"method":"run","params":{"input":[{"kind":"text","content":"Hello"}]}}"#;
|
||||
fn method_submit_json_roundtrip_and_run_is_rejected() {
|
||||
let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#;
|
||||
let method: Method = serde_json::from_str(json).unwrap();
|
||||
match &method {
|
||||
Method::Run { input } => {
|
||||
Method::Submit { input, .. } => {
|
||||
assert_eq!(input.len(), 1);
|
||||
match &input[0] {
|
||||
Segment::Text { content } => assert_eq!(content, "Hello"),
|
||||
other => panic!("expected Text, got {other:?}"),
|
||||
}
|
||||
}
|
||||
other => panic!("expected Run, got {other:?}"),
|
||||
other => panic!("expected Submit, got {other:?}"),
|
||||
}
|
||||
let serialized = serde_json::to_string(&method).unwrap();
|
||||
assert_eq!(serialized, json);
|
||||
assert!(
|
||||
serde_json::from_str::<Method>(r#"{"method":"run","params":{"input":[]}}"#).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_run_paste_segment_roundtrip() {
|
||||
let method = Method::Run {
|
||||
fn method_submit_paste_segment_roundtrip() {
|
||||
let method = Method::Submit {
|
||||
submission_request_id: "request-1".to_string(),
|
||||
input: vec![
|
||||
Segment::text("see "),
|
||||
Segment::Paste {
|
||||
@@ -1318,7 +1462,7 @@ mod tests {
|
||||
let json = serde_json::to_string(&method).unwrap();
|
||||
let decoded: Method = serde_json::from_str(&json).unwrap();
|
||||
match decoded {
|
||||
Method::Run { input } => {
|
||||
Method::Submit { input, .. } => {
|
||||
assert_eq!(input.len(), 2);
|
||||
match &input[1] {
|
||||
Segment::Paste {
|
||||
@@ -1335,7 +1479,7 @@ mod tests {
|
||||
other => panic!("expected Paste, got {other:?}"),
|
||||
}
|
||||
}
|
||||
other => panic!("expected Run, got {other:?}"),
|
||||
other => panic!("expected Submit, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1389,8 +1533,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_run_flow_segment_roundtrip() {
|
||||
let method = Method::Run {
|
||||
fn method_submit_flow_segment_roundtrip() {
|
||||
let method = Method::Submit {
|
||||
submission_request_id: "request-1".to_string(),
|
||||
input: vec![
|
||||
Segment::Flow {
|
||||
selector: "builtin:coder-review".to_string(),
|
||||
@@ -1404,7 +1549,7 @@ mod tests {
|
||||
let decoded = serde_json::from_str::<Method>(&json).unwrap();
|
||||
assert!(matches!(
|
||||
decoded,
|
||||
Method::Run { input }
|
||||
Method::Submit { input, .. }
|
||||
if matches!(
|
||||
input.as_slice(),
|
||||
[
|
||||
@@ -1416,15 +1561,26 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_tracked_run_is_not_public_protocol_json() {
|
||||
let method = Method::RunTracked {
|
||||
fn authenticated_submit_replaces_wire_source_with_transport_identity() {
|
||||
let method = Method::SubmitTracked {
|
||||
input: vec![Segment::text("private")],
|
||||
submission_id: "submission-1".to_string(),
|
||||
submission_request_id: "request-1".to_string(),
|
||||
source: AuthenticatedInputSource::Account {
|
||||
account_id: "account-1".into(),
|
||||
},
|
||||
};
|
||||
assert!(serde_json::to_string(&method).is_err());
|
||||
let json = serde_json::to_string(&method).unwrap();
|
||||
let decoded = serde_json::from_str::<Method>(&json).unwrap();
|
||||
assert!(matches!(
|
||||
decoded,
|
||||
Method::SubmitTracked {
|
||||
source: AuthenticatedInputSource::UntrustedWire,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert!(
|
||||
serde_json::from_str::<Method>(
|
||||
r#"{"method":"run_tracked","input":[],"submission_id":"forged"}"#,
|
||||
r#"{"method":"submit_tracked","input":[],"submission_request_id":"forged"}"#,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
@@ -1442,16 +1598,16 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_run_with_unknown_segment_decodes() {
|
||||
let json = r#"{"method":"run","params":{"input":[{"kind":"text","content":"hi"},{"kind":"future_thing","x":1}]}}"#;
|
||||
fn method_submit_with_unknown_segment_decodes() {
|
||||
let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"hi"},{"kind":"future_thing","x":1}]}}"#;
|
||||
let method: Method = serde_json::from_str(json).unwrap();
|
||||
match method {
|
||||
Method::Run { input } => {
|
||||
Method::Submit { input, .. } => {
|
||||
assert_eq!(input.len(), 2);
|
||||
assert!(matches!(input[0], Segment::Text { .. }));
|
||||
assert!(matches!(input[1], Segment::Unknown));
|
||||
}
|
||||
other => panic!("expected Run, got {other:?}"),
|
||||
other => panic!("expected Submit, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1648,11 +1804,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn method_notify_json_roundtrip_defaults_to_auto_run() {
|
||||
let json = r#"{"method":"notify","params":{"message":"turn done"}}"#;
|
||||
let json = r#"{"method":"notify","params":{"notification_request_id":"notification-1","message":"turn done"}}"#;
|
||||
let method: Method = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(
|
||||
method,
|
||||
Method::Notify { ref message, auto_run: true } if message == "turn done"
|
||||
Method::Notify { ref message, auto_run: true, .. } if message == "turn done"
|
||||
));
|
||||
let serialized = serde_json::to_string(&method).unwrap();
|
||||
assert_eq!(serialized, json);
|
||||
@@ -1660,11 +1816,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn method_notify_weak_json_roundtrip_serializes_auto_run_false() {
|
||||
let json = r#"{"method":"notify","params":{"message":"progress","auto_run":false}}"#;
|
||||
let json = r#"{"method":"notify","params":{"notification_request_id":"notification-1","message":"progress","auto_run":false}}"#;
|
||||
let method: Method = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(
|
||||
method,
|
||||
Method::Notify { ref message, auto_run: false } if message == "progress"
|
||||
Method::Notify { ref message, auto_run: false, .. } if message == "progress"
|
||||
));
|
||||
assert_eq!(serde_json::to_string(&method).unwrap(), json);
|
||||
}
|
||||
@@ -1725,6 +1881,7 @@ mod tests {
|
||||
fn event_snapshot_format() {
|
||||
let event = Event::Snapshot {
|
||||
session: SessionSnapshot {
|
||||
pending_submissions: PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![SessionSnapshotEntry {
|
||||
entry_id: "entry-1".into(),
|
||||
timestamp: 1,
|
||||
@@ -1776,6 +1933,7 @@ mod tests {
|
||||
|
||||
let event = Event::Snapshot {
|
||||
session: SessionSnapshot {
|
||||
pending_submissions: PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: Greeting {
|
||||
@@ -1844,6 +2002,7 @@ mod tests {
|
||||
fn event_segment_rotated_roundtrip() {
|
||||
let event = Event::SegmentRotated {
|
||||
session: SessionSnapshot {
|
||||
pending_submissions: PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
};
|
||||
|
||||
@@ -8,11 +8,11 @@ use crate::{
|
||||
CompletionKind, ErrorCode, Event, Greeting, InFlightBlock, InFlightSnapshot,
|
||||
InFlightToolCallState, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnapshot,
|
||||
InvokeKind, MemoryWorkerEvent, Method, PasteArtifactAvailability, PasteArtifactMediaType,
|
||||
PasteArtifactRef, Permission, RewindSummary, RewindTarget, RewindTargetId, RunResult,
|
||||
ScopeRule, Segment, SessionContentPart, SessionEntryProvenance, SessionMessageRole,
|
||||
SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment,
|
||||
ToolResultDisposition, TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent,
|
||||
WorkerStatus,
|
||||
PasteArtifactRef, PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission,
|
||||
RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart,
|
||||
SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry,
|
||||
SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition,
|
||||
TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, WorkerStatus,
|
||||
subscription::{
|
||||
EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame,
|
||||
SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest,
|
||||
@@ -75,6 +75,9 @@ pub fn generated_protocol_types() -> String {
|
||||
push_decl::<SessionToolAttachment>(&cfg, &mut output);
|
||||
push_decl::<SessionSnapshotEntryData>(&cfg, &mut output);
|
||||
push_decl::<SessionSnapshotEntry>(&cfg, &mut output);
|
||||
push_decl::<PendingSubmissionSummary>(&cfg, &mut output);
|
||||
push_decl::<PendingSubmissionsSnapshot>(&cfg, &mut output);
|
||||
push_decl::<SubmissionDisposition>(&cfg, &mut output);
|
||||
push_decl::<SessionSnapshot>(&cfg, &mut output);
|
||||
push_decl::<InternalWorkerKind>(&cfg, &mut output);
|
||||
push_decl::<InternalWorkerRef>(&cfg, &mut output);
|
||||
|
||||
@@ -21,8 +21,10 @@ use crate::segment_log::LogEntry;
|
||||
use crate::store::{Store, StoreError};
|
||||
use crate::uploaded_file::{
|
||||
bind_uploaded_file, clear_uploaded_file_binding, copy_committed_uploaded_files,
|
||||
delete_uncommitted_uploaded_files, delete_uploaded_file, list_uploaded_file_refs,
|
||||
read_uploaded_file, read_uploaded_file_by_id, write_uploaded_file,
|
||||
delete_uncommitted_uploaded_files, delete_uploaded_file, finalize_uploaded_file_binding,
|
||||
list_uploaded_file_refs, pin_uploaded_file, read_uploaded_file, read_uploaded_file_by_id,
|
||||
reconcile_uploaded_file_pins, release_uploaded_file_pin, uploaded_file_has_pending_owner,
|
||||
write_uploaded_file,
|
||||
};
|
||||
use crate::{
|
||||
PasteArtifactLimits, SegmentId, SessionId, UploadedFileLimits, UploadedFileUploadContext,
|
||||
@@ -518,6 +520,61 @@ impl Store for FsStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn pin_uploaded_file(
|
||||
&self,
|
||||
session_id: SessionId,
|
||||
reference: &UploadedFileRef,
|
||||
owner_id: &str,
|
||||
) -> Result<(), StoreError> {
|
||||
let _guard = self
|
||||
.append_lock
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
|
||||
pin_uploaded_file(&self.paste_artifact_dir(session_id), reference, owner_id)
|
||||
}
|
||||
|
||||
fn release_uploaded_file_pin(
|
||||
&self,
|
||||
session_id: SessionId,
|
||||
artifact_id: &str,
|
||||
owner_id: &str,
|
||||
) -> Result<(), StoreError> {
|
||||
let _guard = self
|
||||
.append_lock
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
|
||||
release_uploaded_file_pin(&self.paste_artifact_dir(session_id), artifact_id, owner_id)
|
||||
}
|
||||
|
||||
fn finalize_uploaded_file_binding(
|
||||
&self,
|
||||
session_id: SessionId,
|
||||
artifact_id: &str,
|
||||
source_entry_id: &str,
|
||||
) -> Result<(), StoreError> {
|
||||
let _guard = self
|
||||
.append_lock
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
|
||||
finalize_uploaded_file_binding(
|
||||
&self.paste_artifact_dir(session_id),
|
||||
artifact_id,
|
||||
source_entry_id,
|
||||
)
|
||||
}
|
||||
|
||||
fn reconcile_uploaded_file_pins(
|
||||
&self,
|
||||
session_id: SessionId,
|
||||
live_owner_ids: &[String],
|
||||
) -> Result<u64, StoreError> {
|
||||
let _guard = self
|
||||
.append_lock
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
|
||||
reconcile_uploaded_file_pins(&self.paste_artifact_dir(session_id), live_owner_ids)
|
||||
}
|
||||
|
||||
fn delete_uploaded_file(
|
||||
&self,
|
||||
session_id: SessionId,
|
||||
@@ -541,7 +598,13 @@ impl Store for FsStore {
|
||||
let Some(source_entry_id) = reference.source_entry_id.as_deref() else {
|
||||
continue;
|
||||
};
|
||||
if !self.uploaded_file_is_referenced(session_id, &reference.artifact_id)? {
|
||||
if self.uploaded_file_is_referenced(session_id, &reference.artifact_id)? {
|
||||
finalize_uploaded_file_binding(&dir, &reference.artifact_id, source_entry_id)?;
|
||||
continue;
|
||||
}
|
||||
if uploaded_file_has_pending_owner(&dir, &reference.artifact_id)? {
|
||||
continue;
|
||||
}
|
||||
clear_uploaded_file_binding(&dir, &reference.artifact_id, source_entry_id)?;
|
||||
if delete_uploaded_file(&dir, &reference.artifact_id)? {
|
||||
removed = removed
|
||||
@@ -549,7 +612,6 @@ impl Store for FsStore {
|
||||
.ok_or(StoreError::ArtifactQuotaExceeded)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
@@ -865,6 +927,106 @@ mod tests {
|
||||
assert!(store.read_uploaded_file(owner, &reference).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_upload_pin_survives_cleanup_until_release_or_history_binding() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let store = FsStore::new(tmp.path()).unwrap();
|
||||
let session_id = new_session_id();
|
||||
let limits = UploadedFileLimits {
|
||||
max_file_bytes: 64,
|
||||
max_session_bytes: 128,
|
||||
};
|
||||
let pending = store
|
||||
.write_uploaded_file(session_id, "pending.txt", "text/plain", b"pending", limits)
|
||||
.unwrap();
|
||||
store
|
||||
.pin_uploaded_file(session_id, &pending, "submission-1")
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
store.pin_uploaded_file(session_id, &pending, "submission-other"),
|
||||
Err(StoreError::ArtifactAlreadyCommitted)
|
||||
));
|
||||
drop(store);
|
||||
let store = FsStore::new(tmp.path()).unwrap();
|
||||
assert_eq!(
|
||||
store.delete_uncommitted_uploaded_files(session_id).unwrap(),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.read_uploaded_file_by_id(session_id, &pending.artifact_id)
|
||||
.unwrap()
|
||||
.1,
|
||||
b"pending"
|
||||
);
|
||||
|
||||
let fork_session_id = new_session_id();
|
||||
assert_eq!(
|
||||
store
|
||||
.copy_committed_uploaded_files(session_id, fork_session_id)
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.read_uploaded_file_by_id(fork_session_id, &pending.artifact_id)
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let committed = store
|
||||
.bind_uploaded_file(session_id, &pending, "entry-1")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.delete_uncommitted_uploaded_files(session_id).unwrap(),
|
||||
0
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.read_uploaded_file_by_id(session_id, &pending.artifact_id)
|
||||
.is_ok()
|
||||
);
|
||||
store
|
||||
.create_segment(
|
||||
session_id,
|
||||
new_segment_id(),
|
||||
&[LogEntry::InputSegmentsCheckpoint {
|
||||
ts: 1,
|
||||
user_segments: vec![vec![protocol::Segment::UploadedFile {
|
||||
file: committed.clone(),
|
||||
}]],
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.delete_uncommitted_uploaded_files(session_id).unwrap(),
|
||||
0
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.release_uploaded_file_pin(session_id, &pending.artifact_id, "submission-1")
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let releasable = store
|
||||
.write_uploaded_file(session_id, "cancelled.txt", "text/plain", b"cancel", limits)
|
||||
.unwrap();
|
||||
store
|
||||
.pin_uploaded_file(session_id, &releasable, "submission-2")
|
||||
.unwrap();
|
||||
store
|
||||
.release_uploaded_file_pin(session_id, &releasable.artifact_id, "submission-2")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.delete_uncommitted_uploaded_files(session_id).unwrap(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.read_uploaded_file_by_id(session_id, &releasable.artifact_id)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uploaded_file_validation_and_shared_quota_fail_closed() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
|
||||
@@ -183,6 +183,7 @@ fn canonicalize_history_entry(
|
||||
item,
|
||||
metadata: legacy_metadata(segment_id, line_index, 0),
|
||||
},
|
||||
extensions: Vec::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ pub fn project_session_snapshot(session_id: SessionId, log: &[LogEntry]) -> Sess
|
||||
entries.push(history_entry(entry, *ts, data));
|
||||
}
|
||||
}
|
||||
LogEntry::AnnotatedSystemItem { ts, entry } => entries.push(system_entry(
|
||||
LogEntry::AnnotatedSystemItem { ts, entry, .. } => entries.push(system_entry(
|
||||
&entry.item,
|
||||
entry.metadata.entry_id.0.clone(),
|
||||
*ts,
|
||||
@@ -100,7 +100,10 @@ pub fn project_session_snapshot(session_id: SessionId, log: &[LogEntry]) -> Sess
|
||||
}
|
||||
}
|
||||
|
||||
SessionSnapshot { entries }
|
||||
SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries,
|
||||
}
|
||||
}
|
||||
|
||||
fn extend_history(
|
||||
|
||||
@@ -287,6 +287,7 @@ pub fn append_system_item(
|
||||
LogEntry::AnnotatedSystemItem {
|
||||
ts: segment_log::now_millis(),
|
||||
entry,
|
||||
extensions: Vec::new(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -112,6 +112,8 @@ pub enum LogEntry {
|
||||
AnnotatedSystemItem {
|
||||
ts: u64,
|
||||
entry: LoggedSystemHistoryEntry,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
extensions: Vec<SessionExtension>,
|
||||
},
|
||||
|
||||
/// Turn boundary. Records the turn count after increment.
|
||||
@@ -312,12 +314,19 @@ pub fn collect_state(entries: &[LogEntry]) -> RestoredState {
|
||||
state.annotated_history.push(entry.clone());
|
||||
state.history.push(Item::from(entry.item.clone()));
|
||||
}
|
||||
LogEntry::AnnotatedSystemItem { entry, .. } => {
|
||||
LogEntry::AnnotatedSystemItem {
|
||||
entry, extensions, ..
|
||||
} => {
|
||||
state.annotated_history.push(LoggedHistoryEntry {
|
||||
item: LoggedItem::from(entry.item.to_history_item()),
|
||||
metadata: entry.metadata.clone(),
|
||||
});
|
||||
state.history.push(entry.item.to_history_item());
|
||||
state.extensions.extend(
|
||||
extensions
|
||||
.iter()
|
||||
.map(|extension| (extension.domain.clone(), extension.payload.clone())),
|
||||
);
|
||||
}
|
||||
LogEntry::TurnEnd { turn_count, .. } => {
|
||||
if let Some(active_turn_count) = &mut state.active_run_turn_count {
|
||||
|
||||
@@ -226,6 +226,47 @@ pub trait Store: Send + Sync {
|
||||
Err(StoreError::PasteArtifactUnsupported)
|
||||
}
|
||||
|
||||
/// Retain an uploaded file while a durable pending operation owns it.
|
||||
fn pin_uploaded_file(
|
||||
&self,
|
||||
_session_id: SessionId,
|
||||
_reference: &UploadedFileRef,
|
||||
_owner_id: &str,
|
||||
) -> Result<(), StoreError> {
|
||||
Err(StoreError::PasteArtifactUnsupported)
|
||||
}
|
||||
|
||||
/// Release a pending-operation pin without changing committed ownership.
|
||||
fn release_uploaded_file_pin(
|
||||
&self,
|
||||
_session_id: SessionId,
|
||||
_artifact_id: &str,
|
||||
_owner_id: &str,
|
||||
) -> Result<(), StoreError> {
|
||||
Err(StoreError::PasteArtifactUnsupported)
|
||||
}
|
||||
|
||||
/// Complete the pending-to-history handoff after the history entry commits.
|
||||
fn finalize_uploaded_file_binding(
|
||||
&self,
|
||||
_session_id: SessionId,
|
||||
_artifact_id: &str,
|
||||
_source_entry_id: &str,
|
||||
) -> Result<(), StoreError> {
|
||||
Err(StoreError::PasteArtifactUnsupported)
|
||||
}
|
||||
|
||||
/// Clear pending-operation pins that have no owner in restored durable
|
||||
/// Worker Session state. This repairs an interrupted pin-before-checkpoint
|
||||
/// acceptance without disturbing live queue owners or committed history.
|
||||
fn reconcile_uploaded_file_pins(
|
||||
&self,
|
||||
_session_id: SessionId,
|
||||
_live_owner_ids: &[String],
|
||||
) -> Result<u64, StoreError> {
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
/// Delete an uncommitted uploaded file owned by `session_id`.
|
||||
fn delete_uploaded_file(
|
||||
&self,
|
||||
|
||||
@@ -24,6 +24,12 @@ pub const DEFAULT_MAX_FILES_PER_SUBMISSION: usize = 8;
|
||||
pub const DEFAULT_MAX_SESSION_UPLOADED_FILES: u64 = 256;
|
||||
const MAX_FILE_NAME_CHARS: usize = 255;
|
||||
const MAX_MEDIA_TYPE_BYTES: usize = 127;
|
||||
fn validate_pending_owner_id(owner_id: &str) -> Result<()> {
|
||||
if owner_id.is_empty() || owner_id.len() > 256 {
|
||||
return Err(StoreError::ArtifactIntegrityMismatch);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct UploadedFileLimits {
|
||||
@@ -59,6 +65,8 @@ struct StoredUploadedFile {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
source_entry_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pending_owner_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
upload_context: Option<UploadedFileUploadContext>,
|
||||
content_base64: String,
|
||||
}
|
||||
@@ -291,6 +299,7 @@ pub(crate) fn write_uploaded_file(
|
||||
byte_len,
|
||||
sha256: sha256.clone(),
|
||||
source_entry_id: None,
|
||||
pending_owner_id: None,
|
||||
upload_context: context.cloned(),
|
||||
content_base64: BASE64.encode(content),
|
||||
};
|
||||
@@ -338,6 +347,12 @@ pub(crate) fn read_uploaded_file_by_id(
|
||||
Ok((reference, content))
|
||||
}
|
||||
|
||||
pub(crate) fn uploaded_file_has_pending_owner(dir: &Path, artifact_id: &str) -> Result<bool> {
|
||||
let path = record_path(dir, artifact_id)?;
|
||||
let stored: StoredUploadedFile = serde_json::from_slice(&fs::read(path)?)?;
|
||||
Ok(stored.pending_owner_id.is_some())
|
||||
}
|
||||
|
||||
pub(crate) fn read_uploaded_file(dir: &Path, reference: &UploadedFileRef) -> Result<Vec<u8>> {
|
||||
let (stored_reference, content) = read_uploaded_file_by_id(dir, &reference.artifact_id)?;
|
||||
if stored_reference.file_name != reference.file_name
|
||||
@@ -376,6 +391,98 @@ pub(crate) fn clear_uploaded_file_binding(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn pin_uploaded_file(
|
||||
dir: &Path,
|
||||
reference: &UploadedFileRef,
|
||||
owner_id: &str,
|
||||
) -> Result<()> {
|
||||
validate_pending_owner_id(owner_id)?;
|
||||
if reference.source_entry_id.is_some() {
|
||||
return Err(StoreError::ArtifactAlreadyCommitted);
|
||||
}
|
||||
let aggregate_lock = fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(dir.join(".aggregate.lock"))?;
|
||||
FileExt::lock_exclusive(&aggregate_lock)?;
|
||||
let path = record_path(dir, &reference.artifact_id)?;
|
||||
let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?;
|
||||
if stored.file_name != reference.file_name
|
||||
|| stored.media_type != reference.media_type
|
||||
|| stored.created_at_ms != reference.created_at_ms
|
||||
|| stored.byte_len != reference.byte_len
|
||||
|| stored.sha256 != reference.sha256
|
||||
{
|
||||
return Err(StoreError::ArtifactIntegrityMismatch);
|
||||
}
|
||||
if stored.source_entry_id.is_some() {
|
||||
return Err(StoreError::ArtifactAlreadyCommitted);
|
||||
}
|
||||
if let Some(existing_owner) = stored.pending_owner_id.as_deref() {
|
||||
return if existing_owner == owner_id {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StoreError::ArtifactAlreadyCommitted)
|
||||
};
|
||||
}
|
||||
stored.pending_owner_id = Some(owner_id.to_owned());
|
||||
let temp = dir.join(format!(".{}.file.pin.tmp", reference.artifact_id));
|
||||
fs::write(&temp, serde_json::to_vec(&stored)?)?;
|
||||
fs::rename(temp, path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn release_uploaded_file_pin(
|
||||
dir: &Path,
|
||||
artifact_id: &str,
|
||||
owner_id: &str,
|
||||
) -> Result<()> {
|
||||
validate_pending_owner_id(owner_id)?;
|
||||
let aggregate_lock = fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(dir.join(".aggregate.lock"))?;
|
||||
FileExt::lock_exclusive(&aggregate_lock)?;
|
||||
let path = record_path(dir, artifact_id)?;
|
||||
let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?;
|
||||
if stored.pending_owner_id.as_deref() != Some(owner_id) {
|
||||
return Err(StoreError::ArtifactIntegrityMismatch);
|
||||
}
|
||||
stored.pending_owner_id = None;
|
||||
let temp = dir.join(format!(".{artifact_id}.file.unpin.tmp"));
|
||||
fs::write(&temp, serde_json::to_vec(&stored)?)?;
|
||||
fs::rename(temp, path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn finalize_uploaded_file_binding(
|
||||
dir: &Path,
|
||||
artifact_id: &str,
|
||||
source_entry_id: &str,
|
||||
) -> Result<()> {
|
||||
let aggregate_lock = fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(dir.join(".aggregate.lock"))?;
|
||||
FileExt::lock_exclusive(&aggregate_lock)?;
|
||||
let path = record_path(dir, artifact_id)?;
|
||||
let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?;
|
||||
if stored.source_entry_id.as_deref() != Some(source_entry_id) {
|
||||
return Err(StoreError::ArtifactIntegrityMismatch);
|
||||
}
|
||||
if stored.pending_owner_id.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
stored.pending_owner_id = None;
|
||||
let temp = dir.join(format!(".{artifact_id}.file.finalize.tmp"));
|
||||
fs::write(&temp, serde_json::to_vec(&stored)?)?;
|
||||
fs::rename(temp, path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn bind_uploaded_file(
|
||||
dir: &Path,
|
||||
reference: &UploadedFileRef,
|
||||
@@ -479,6 +586,40 @@ pub(crate) fn copy_committed_uploaded_files(source_dir: &Path, target_dir: &Path
|
||||
Ok(copied)
|
||||
}
|
||||
|
||||
pub(crate) fn reconcile_uploaded_file_pins(dir: &Path, live_owner_ids: &[String]) -> Result<u64> {
|
||||
fs::create_dir_all(dir)?;
|
||||
let aggregate_lock = fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(dir.join(".aggregate.lock"))?;
|
||||
FileExt::lock_exclusive(&aggregate_lock)?;
|
||||
let mut reconciled = 0_u64;
|
||||
for entry in fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(artifact_id) = file_name.strip_suffix(".file.json") else {
|
||||
continue;
|
||||
};
|
||||
let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?;
|
||||
let Some(owner_id) = stored.pending_owner_id.as_deref() else {
|
||||
continue;
|
||||
};
|
||||
if live_owner_ids.iter().any(|live| live == owner_id) {
|
||||
continue;
|
||||
}
|
||||
stored.pending_owner_id = None;
|
||||
let temp = dir.join(format!(".{artifact_id}.file.reconcile.tmp"));
|
||||
fs::write(&temp, serde_json::to_vec(&stored)?)?;
|
||||
fs::rename(temp, path)?;
|
||||
reconciled = reconciled.saturating_add(1);
|
||||
}
|
||||
Ok(reconciled)
|
||||
}
|
||||
|
||||
pub(crate) fn delete_uncommitted_uploaded_files(dir: &Path) -> Result<u64> {
|
||||
fs::create_dir_all(dir)?;
|
||||
let aggregate_lock = fs::OpenOptions::new()
|
||||
@@ -499,7 +640,7 @@ pub(crate) fn delete_uncommitted_uploaded_files(dir: &Path) -> Result<u64> {
|
||||
continue;
|
||||
}
|
||||
let stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?;
|
||||
if stored.source_entry_id.is_none() {
|
||||
if stored.source_entry_id.is_none() && stored.pending_owner_id.is_none() {
|
||||
fs::remove_file(path)?;
|
||||
removed = removed
|
||||
.checked_add(1)
|
||||
@@ -523,7 +664,7 @@ pub(crate) fn delete_uploaded_file(dir: &Path, artifact_id: &str) -> Result<bool
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if stored.source_entry_id.is_some() {
|
||||
if stored.source_entry_id.is_some() || stored.pending_owner_id.is_some() {
|
||||
return Err(StoreError::ArtifactAlreadyCommitted);
|
||||
}
|
||||
match fs::remove_file(path) {
|
||||
|
||||
@@ -99,7 +99,10 @@ async fn in_process_host_runs_text_and_read_tool_then_shuts_down() {
|
||||
let mut protocol_client = host.connect();
|
||||
|
||||
protocol_client
|
||||
.send(&Method::run_text("read the probe"))
|
||||
.send(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"read the probe",
|
||||
))
|
||||
.await
|
||||
.expect("submit input");
|
||||
|
||||
@@ -336,11 +339,15 @@ async fn standalone_restore_preserves_history_tasks_notifications_and_cwd_scope(
|
||||
let worker_id = host.worker_id();
|
||||
let mut protocol_client = host.connect();
|
||||
protocol_client
|
||||
.send(&Method::run_text("first request"))
|
||||
.send(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"first request",
|
||||
))
|
||||
.await?;
|
||||
wait_for_run_end(&mut protocol_client).await?;
|
||||
protocol_client
|
||||
.send(&Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message: "persisted notification".to_string(),
|
||||
auto_run: true,
|
||||
})
|
||||
@@ -394,7 +401,10 @@ async fn standalone_restore_preserves_history_tasks_notifications_and_cwd_scope(
|
||||
assert!(snapshot.contains("persisted notification"), "{snapshot}");
|
||||
|
||||
protocol_client
|
||||
.send(&Method::run_text("continue after restore"))
|
||||
.send(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"continue after restore",
|
||||
))
|
||||
.await?;
|
||||
wait_for_run_end(&mut protocol_client).await?;
|
||||
let request = second_inspection
|
||||
|
||||
@@ -118,6 +118,7 @@ impl Tool for BashTool {
|
||||
command: params.command,
|
||||
timeout_secs,
|
||||
output_limit: INLINE_BYTE_BUDGET,
|
||||
cwd: None,
|
||||
spill_dir: Some(self.output_dir.clone()),
|
||||
tool_call_id: Some(call_id.clone()),
|
||||
})
|
||||
|
||||
+124
-153
@@ -102,23 +102,6 @@ struct RollbackSubmitState {
|
||||
turn_before: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct QueuedInput {
|
||||
segments: Vec<Segment>,
|
||||
preview: String,
|
||||
}
|
||||
|
||||
impl QueuedInput {
|
||||
fn new(segments: Vec<Segment>) -> Self {
|
||||
let preview = Segment::flatten_to_text(&segments);
|
||||
Self { segments, preview }
|
||||
}
|
||||
|
||||
pub fn preview(&self) -> &str {
|
||||
&self.preview
|
||||
}
|
||||
}
|
||||
|
||||
struct ComposerInputHistory {
|
||||
entries: VecDeque<Vec<Segment>>,
|
||||
browse: Option<ComposerInputHistoryBrowse>,
|
||||
@@ -272,7 +255,7 @@ pub struct App {
|
||||
/// Current transient actionbar notice. Notices are local UI state only:
|
||||
/// they are never appended to transcript/session history or LLM context.
|
||||
actionbar_notice: Option<ActionbarNotice>,
|
||||
/// Normal composer input that is submitted as `Method::Run`.
|
||||
/// Normal composer input that is submitted as `Method::Submit`.
|
||||
pub input: InputBuffer,
|
||||
/// Separate command-line input. It is never submitted as a user message.
|
||||
pub command_input: InputBuffer,
|
||||
@@ -333,9 +316,8 @@ pub struct App {
|
||||
/// Top entry index of the task pane's visible window. Clamped on
|
||||
/// render so it never points past the end of the list.
|
||||
pub task_pane_scroll: usize,
|
||||
/// TUI-local FIFO of user inputs submitted while the Worker is already running.
|
||||
/// Entries have not been sent to the Worker yet, so they remain editable/cancellable locally.
|
||||
queued_inputs: VecDeque<QueuedInput>,
|
||||
/// Authoritative WorkerSession FIFO summary received from snapshot/live events.
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot,
|
||||
/// TUI-local readline-style composer input history. This is intentionally
|
||||
/// client-side only: recalled entries are plain drafts until submitted again.
|
||||
input_history: ComposerInputHistory,
|
||||
@@ -395,7 +377,7 @@ impl App {
|
||||
text_selection: TextSelectionState::default(),
|
||||
task_pane_open: false,
|
||||
task_pane_scroll: 0,
|
||||
queued_inputs: VecDeque::new(),
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
input_history: ComposerInputHistory::new(),
|
||||
input_history_store: None,
|
||||
pending_submit_rollback: None,
|
||||
@@ -768,18 +750,34 @@ impl App {
|
||||
return None;
|
||||
}
|
||||
self.record_input_history(segments.clone());
|
||||
if self.running {
|
||||
self.queued_inputs.push_back(QueuedInput::new(segments));
|
||||
self.input.clear();
|
||||
self.completion = None;
|
||||
return None;
|
||||
}
|
||||
self.input.clear();
|
||||
Some(self.method_for_run(segments))
|
||||
}
|
||||
|
||||
pub fn submit_notify_input(&mut self) -> Option<Method> {
|
||||
let segments = self.input.submit_segments();
|
||||
if segments_are_blank(&segments) {
|
||||
return None;
|
||||
}
|
||||
if segments
|
||||
.iter()
|
||||
.any(|segment| matches!(segment, Segment::UploadedFile { .. }))
|
||||
{
|
||||
self.push_error("Notify accepts text only; remove attachments or queue a Submit.");
|
||||
return None;
|
||||
}
|
||||
let message = Segment::flatten_to_text(&segments);
|
||||
self.record_input_history(segments);
|
||||
self.input.clear();
|
||||
Some(Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message,
|
||||
auto_run: true,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn restore_unsent_run(&mut self, method: &Method) {
|
||||
let Method::Run { input } = method else {
|
||||
let Method::Submit { input, .. } = method else {
|
||||
return;
|
||||
};
|
||||
self.pending_submit_rollback = None;
|
||||
@@ -787,8 +785,9 @@ impl App {
|
||||
self.input.replace_with_segments(input);
|
||||
self.completion = None;
|
||||
} else {
|
||||
self.queued_inputs
|
||||
.push_front(QueuedInput::new(input.clone()));
|
||||
self.push_error(
|
||||
"Submit transport failed; current Composer was preserved and the unsent input was not queued.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -804,7 +803,10 @@ impl App {
|
||||
block_start: self.blocks.len(),
|
||||
turn_before: self.turn_index,
|
||||
});
|
||||
Method::Run { input: segments }
|
||||
Method::Submit {
|
||||
submission_request_id: protocol::new_submission_request_id(),
|
||||
input: segments,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_input_history(&mut self, segments: Vec<Segment>) {
|
||||
@@ -825,7 +827,7 @@ impl App {
|
||||
}
|
||||
|
||||
pub fn queued_input_count(&self) -> usize {
|
||||
self.queued_inputs.len()
|
||||
self.pending_submissions.submissions.len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -910,36 +912,31 @@ impl App {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn continue_pending_method(&self) -> Option<Method> {
|
||||
Some(Method::ContinuePending {
|
||||
expected_revision: self.pending_submissions.revision,
|
||||
expected_head_id: self.pending_submissions.head_id.clone()?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn clear_pending_method(&self) -> Method {
|
||||
Method::ClearPendingSubmissions {
|
||||
expected_revision: self.pending_submissions.revision,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cancel_pending_method(&self, submission_id: String) -> Method {
|
||||
Method::CancelPendingSubmission {
|
||||
submission_id,
|
||||
expected_revision: self.pending_submissions.revision,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_queued_input_preview(&self) -> Option<&str> {
|
||||
self.queued_inputs.front().map(QueuedInput::preview)
|
||||
}
|
||||
|
||||
pub fn clear_queued_inputs(&mut self) -> usize {
|
||||
let cleared = self.queued_inputs.len();
|
||||
self.queued_inputs.clear();
|
||||
cleared
|
||||
}
|
||||
|
||||
pub fn restore_next_queued_input_to_composer(&mut self) -> bool {
|
||||
if self.queued_inputs.is_empty() {
|
||||
return false;
|
||||
}
|
||||
if !self.input.is_empty() {
|
||||
self.push_error("Composer is not empty; clear it before editing queued input.");
|
||||
return false;
|
||||
}
|
||||
let Some(queued) = self.queued_inputs.pop_front() else {
|
||||
return false;
|
||||
};
|
||||
self.input_history.cancel_browse();
|
||||
self.input.replace_with_segments(&queued.segments);
|
||||
self.completion = None;
|
||||
true
|
||||
}
|
||||
|
||||
fn pop_next_queued_run(&mut self) -> Option<Method> {
|
||||
let queued = self.queued_inputs.pop_front()?;
|
||||
Some(self.method_for_run(queued.segments))
|
||||
self.pending_submissions
|
||||
.submissions
|
||||
.first()
|
||||
.map(|submission| submission.submission_id.as_str())
|
||||
}
|
||||
|
||||
pub fn clear_actionbar_notice(&mut self) {
|
||||
@@ -1123,6 +1120,11 @@ impl App {
|
||||
}
|
||||
|
||||
match event {
|
||||
Event::SubmissionAccepted { .. } => {}
|
||||
Event::SubmissionRejected { message, .. } => self.push_error(message),
|
||||
Event::PendingSubmissionsChanged { pending } => {
|
||||
self.pending_submissions = pending;
|
||||
}
|
||||
Event::UserMessage { segments } => {
|
||||
self.turn_index += 1;
|
||||
self.blocks.push(Block::TurnHeader {
|
||||
@@ -1372,9 +1374,6 @@ impl App {
|
||||
WorkerStatus::Idle
|
||||
}
|
||||
});
|
||||
if matches!(result, RunResult::Finished | RunResult::LimitReached) {
|
||||
return self.pop_next_queued_run();
|
||||
}
|
||||
}
|
||||
}
|
||||
Event::CompactStart { .. } => {
|
||||
@@ -1449,6 +1448,7 @@ impl App {
|
||||
internal_workers,
|
||||
} => {
|
||||
self.rewind_refresh_fence = false;
|
||||
self.pending_submissions = session.pending_submissions.clone();
|
||||
self.restore_snapshot(&session, greeting, in_flight);
|
||||
self.replace_internal_worker_snapshots(internal_workers);
|
||||
self.set_worker_status(status);
|
||||
@@ -2681,7 +2681,10 @@ mod rewind_refresh_tests {
|
||||
});
|
||||
|
||||
app.handle_worker_event(Event::RewindApplied {
|
||||
session: protocol::SessionSnapshot { entries: vec![] },
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![],
|
||||
},
|
||||
input: vec![Segment::text("selected rewind input")],
|
||||
summary: summary(3),
|
||||
});
|
||||
@@ -2700,7 +2703,10 @@ mod rewind_refresh_tests {
|
||||
});
|
||||
|
||||
app.handle_worker_event(Event::RewindApplied {
|
||||
session: protocol::SessionSnapshot { entries: vec![] },
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![],
|
||||
},
|
||||
input: vec![Segment::text("rewound input")],
|
||||
summary: summary(1),
|
||||
});
|
||||
@@ -2743,7 +2749,10 @@ mod rewind_refresh_tests {
|
||||
});
|
||||
|
||||
app.handle_worker_event(Event::RewindApplied {
|
||||
session: protocol::SessionSnapshot { entries: vec![] },
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![],
|
||||
},
|
||||
input: vec![Segment::text("rewound input")],
|
||||
summary: summary(2),
|
||||
});
|
||||
@@ -2877,7 +2886,7 @@ mod composer_history_persistence_tests {
|
||||
path: "src/lib.rs".into(),
|
||||
},
|
||||
]);
|
||||
assert!(matches!(app.submit_input(), Some(Method::Run { .. })));
|
||||
assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
|
||||
|
||||
let mut reloaded = App::new_with_input_history_store("test".into(), store);
|
||||
assert!(reloaded.browse_input_history_older());
|
||||
@@ -2958,7 +2967,7 @@ mod composer_history_persistence_tests {
|
||||
app.insert_char(c);
|
||||
}
|
||||
match app.submit_input() {
|
||||
Some(Method::Run { input }) => input,
|
||||
Some(Method::Submit { input, .. }) => input,
|
||||
other => panic!("expected Run, got {other:?}"),
|
||||
}
|
||||
}
|
||||
@@ -3424,72 +3433,44 @@ mod completion_flow_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn running_submit_is_queued_locally_and_clears_composer() {
|
||||
fn running_submit_is_sent_to_the_worker_and_not_queued_locally() {
|
||||
let mut app = App::new("test".into());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
insert_text(&mut app, "queued turn");
|
||||
|
||||
assert!(app.submit_input().is_none());
|
||||
let method = app.submit_input();
|
||||
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
assert_eq!(app.next_queued_input_preview(), Some("queued turn"));
|
||||
assert!(matches!(method, Some(Method::Submit { .. })));
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
assert_eq!(input_text(&app), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finished_run_auto_sends_next_queued_input() {
|
||||
fn pending_submission_projection_is_worker_authoritative() {
|
||||
let mut app = App::new("test".into());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
insert_text(&mut app, "next turn");
|
||||
assert!(app.submit_input().is_none());
|
||||
|
||||
let method = app.handle_worker_event(Event::RunEnd {
|
||||
result: RunResult::Finished,
|
||||
app.handle_worker_event(Event::PendingSubmissionsChanged {
|
||||
pending: protocol::PendingSubmissionsSnapshot {
|
||||
revision: 3,
|
||||
notification_count: 0,
|
||||
head_id: Some("submission-1".into()),
|
||||
submissions: vec![protocol::PendingSubmissionSummary {
|
||||
submission_id: "submission-1".into(),
|
||||
accepted_at_ms: 7,
|
||||
segment_count: 2,
|
||||
byte_len: 42,
|
||||
}],
|
||||
},
|
||||
});
|
||||
|
||||
match method {
|
||||
Some(Method::Run { input }) => {
|
||||
assert_eq!(Segment::flatten_to_text(&input), "next turn");
|
||||
}
|
||||
other => panic!("expected queued Run, got {other:?}"),
|
||||
}
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit_reached_run_auto_sends_next_queued_input() {
|
||||
let mut app = App::new("test".into());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
insert_text(&mut app, "next after limit");
|
||||
assert!(app.submit_input().is_none());
|
||||
|
||||
let method = app.handle_worker_event(Event::RunEnd {
|
||||
result: RunResult::LimitReached,
|
||||
});
|
||||
|
||||
match method {
|
||||
Some(Method::Run { input }) => {
|
||||
assert_eq!(Segment::flatten_to_text(&input), "next after limit");
|
||||
}
|
||||
other => panic!("expected queued Run, got {other:?}"),
|
||||
}
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paused_and_rolled_back_run_do_not_auto_send_queue() {
|
||||
for result in [RunResult::Paused, RunResult::RolledBack] {
|
||||
let mut app = App::new("test".into());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
insert_text(&mut app, "held turn");
|
||||
assert!(app.submit_input().is_none());
|
||||
|
||||
let method = app.handle_worker_event(Event::RunEnd { result });
|
||||
|
||||
assert!(method.is_none());
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
assert_eq!(app.next_queued_input_preview(), Some("held turn"));
|
||||
}
|
||||
assert_eq!(app.next_queued_input_preview(), Some("submission-1"));
|
||||
assert!(
|
||||
app.handle_worker_event(Event::RunEnd {
|
||||
result: RunResult::Finished,
|
||||
})
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3501,24 +3482,6 @@ mod completion_flow_tests {
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_input_can_be_restored_to_composer_or_cleared() {
|
||||
let mut app = App::new("test".into());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
insert_text(&mut app, "edit me");
|
||||
assert!(app.submit_input().is_none());
|
||||
|
||||
assert!(app.restore_next_queued_input_to_composer());
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
assert_eq!(input_text(&app), "edit me");
|
||||
|
||||
app.input.clear();
|
||||
insert_text(&mut app, "clear me");
|
||||
assert!(app.submit_input().is_none());
|
||||
assert_eq!(app.clear_queued_inputs(), 1);
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
}
|
||||
|
||||
fn insert_text(app: &mut App, text: &str) {
|
||||
for c in text.chars() {
|
||||
app.insert_char(c);
|
||||
@@ -3530,7 +3493,7 @@ mod completion_flow_tests {
|
||||
app.insert_char(c);
|
||||
}
|
||||
match app.submit_input() {
|
||||
Some(Method::Run { input }) => input,
|
||||
Some(Method::Submit { input, .. }) => input,
|
||||
other => panic!("expected Run, got {other:?}"),
|
||||
}
|
||||
}
|
||||
@@ -3675,6 +3638,7 @@ mod completion_flow_tests {
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
greeting: test_greeting(),
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
status: WorkerStatus::Running,
|
||||
@@ -3783,6 +3747,7 @@ mod completion_flow_tests {
|
||||
revision,
|
||||
status: WorkerStatus::Idle,
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
in_flight: protocol::InFlightSnapshot::default(),
|
||||
@@ -4000,6 +3965,7 @@ mod completion_flow_tests {
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
greeting: test_greeting(),
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
@@ -4051,6 +4017,7 @@ mod completion_flow_tests {
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
greeting: test_greeting(),
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
@@ -4064,6 +4031,7 @@ mod completion_flow_tests {
|
||||
},
|
||||
revision: 4,
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
status: WorkerStatus::Running,
|
||||
@@ -4222,6 +4190,7 @@ mod completion_flow_tests {
|
||||
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting,
|
||||
@@ -4437,23 +4406,23 @@ mod completion_flow_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_history_records_queued_inputs_and_suppresses_consecutive_duplicates() {
|
||||
fn input_history_records_running_submits_and_suppresses_consecutive_duplicates() {
|
||||
let mut app = App::new("test".into());
|
||||
app.running = true;
|
||||
|
||||
for c in "repeat".chars() {
|
||||
app.insert_char(c);
|
||||
}
|
||||
assert!(app.submit_input().is_none());
|
||||
assert!(app.submit_input().is_some());
|
||||
assert_eq!(app.input_history_len(), 1);
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
|
||||
for c in "repeat".chars() {
|
||||
app.insert_char(c);
|
||||
}
|
||||
assert!(app.submit_input().is_none());
|
||||
assert!(app.submit_input().is_some());
|
||||
assert_eq!(app.input_history_len(), 1);
|
||||
assert_eq!(app.queued_input_count(), 2);
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
|
||||
app.insert_char(' ');
|
||||
assert!(app.submit_input().is_none());
|
||||
@@ -4481,7 +4450,7 @@ mod completion_flow_tests {
|
||||
},
|
||||
];
|
||||
app.input.replace_with_segments(&original);
|
||||
assert!(matches!(app.submit_input(), Some(Method::Run { .. })));
|
||||
assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
|
||||
|
||||
assert!(app.browse_input_history_older());
|
||||
assert_eq!(app.input.submit_segments(), original);
|
||||
@@ -4493,7 +4462,7 @@ mod completion_flow_tests {
|
||||
for c in "sent".chars() {
|
||||
app.insert_char(c);
|
||||
}
|
||||
assert!(matches!(app.submit_input(), Some(Method::Run { .. })));
|
||||
assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
|
||||
|
||||
for c in "draft".chars() {
|
||||
app.insert_char(c);
|
||||
@@ -4511,7 +4480,7 @@ mod completion_flow_tests {
|
||||
for c in "sent".chars() {
|
||||
app.insert_char(c);
|
||||
}
|
||||
assert!(matches!(app.submit_input(), Some(Method::Run { .. })));
|
||||
assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
|
||||
|
||||
assert!(app.browse_input_history_older());
|
||||
assert!(app.input_history_is_browsing());
|
||||
@@ -4528,17 +4497,19 @@ mod completion_flow_tests {
|
||||
for c in "first".chars() {
|
||||
app.insert_char(c);
|
||||
}
|
||||
assert!(matches!(app.submit_input(), Some(Method::Run { .. })));
|
||||
assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
|
||||
for c in "second".chars() {
|
||||
app.insert_char(c);
|
||||
}
|
||||
assert!(matches!(app.submit_input(), Some(Method::Run { .. })));
|
||||
assert!(matches!(app.submit_input(), Some(Method::Submit { .. })));
|
||||
|
||||
assert!(app.browse_input_history_older());
|
||||
assert!(app.browse_input_history_older());
|
||||
let method = app.submit_input();
|
||||
match method {
|
||||
Some(Method::Run { input }) => assert_eq!(Segment::flatten_to_text(&input), "first"),
|
||||
Some(Method::Submit { input, .. }) => {
|
||||
assert_eq!(Segment::flatten_to_text(&input), "first")
|
||||
}
|
||||
other => panic!("expected recalled run, got {other:?}"),
|
||||
}
|
||||
assert_eq!(app.input_history_len(), 3);
|
||||
|
||||
+155
-109
@@ -270,8 +270,8 @@ impl<T: Socket> ConsoleConnection<T> {
|
||||
async fn send(&mut self, method: &Method) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut prepared = method.clone();
|
||||
let carries_attachments =
|
||||
matches!(prepared, Method::Run { .. }) && !self.pending_attachments.is_empty();
|
||||
if let Method::Run { input } = &mut prepared {
|
||||
matches!(prepared, Method::Submit { .. }) && !self.pending_attachments.is_empty();
|
||||
if let Method::Submit { input, .. } = &mut prepared {
|
||||
input.extend(
|
||||
self.pending_attachments
|
||||
.iter()
|
||||
@@ -569,6 +569,7 @@ async fn run_e2e_rewind_fixture(
|
||||
app.connected = true;
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
@@ -697,6 +698,7 @@ async fn run_e2e_rewind_fixture(
|
||||
if submitted_at.elapsed() >= apply_delay {
|
||||
app.handle_worker_event(Event::RewindApplied {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
input: vec![Segment::text("rewind-live-refresh")],
|
||||
@@ -916,7 +918,7 @@ async fn run_loop<T: Socket>(
|
||||
}
|
||||
|
||||
fn attachment_command_path(method: &Method) -> Option<PathBuf> {
|
||||
let Method::Run { input } = method else {
|
||||
let Method::Submit { input, .. } = method else {
|
||||
return None;
|
||||
};
|
||||
let [Segment::Text { content }] = input.as_slice() else {
|
||||
@@ -927,7 +929,7 @@ fn attachment_command_path(method: &Method) -> Option<PathBuf> {
|
||||
}
|
||||
|
||||
fn is_clear_attachments_command(method: &Method) -> bool {
|
||||
let Method::Run { input } = method else {
|
||||
let Method::Submit { input, .. } = method else {
|
||||
return false;
|
||||
};
|
||||
matches!(
|
||||
@@ -941,7 +943,7 @@ async fn send_console_method<T: Socket>(
|
||||
client: &mut ConsoleConnection<T>,
|
||||
method: &Method,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
if matches!(method, Method::Run { .. }) && client.has_active_uploads() {
|
||||
if matches!(method, Method::Submit { .. }) && client.has_active_uploads() {
|
||||
app.restore_unsent_run(method);
|
||||
app.flash_actionbar_notice(
|
||||
"Attachment upload is still in progress; wait or use /clear-attachments.",
|
||||
@@ -953,7 +955,7 @@ async fn send_console_method<T: Socket>(
|
||||
}
|
||||
|
||||
let sends_attachments =
|
||||
matches!(method, Method::Run { .. }) && !client.pending_attachments.is_empty();
|
||||
matches!(method, Method::Submit { .. }) && !client.pending_attachments.is_empty();
|
||||
if let Err(error) = client.send(method).await {
|
||||
if sends_attachments {
|
||||
app.restore_unsent_run(method);
|
||||
@@ -1148,18 +1150,27 @@ fn handle_key(app: &mut App, key: KeyEvent) -> Option<Method> {
|
||||
app.clear_command_input();
|
||||
Some(None)
|
||||
}
|
||||
KeyCode::Char(c)
|
||||
if c.eq_ignore_ascii_case(&'d') && alt && !ctrl && !app.is_command_mode() =>
|
||||
{
|
||||
Some(
|
||||
app.next_queued_input_preview()
|
||||
.map(str::to_owned)
|
||||
.map(|submission_id| app.cancel_pending_method(submission_id)),
|
||||
)
|
||||
}
|
||||
KeyCode::Char(c)
|
||||
if c.eq_ignore_ascii_case(&'n') && alt && !ctrl && !app.is_command_mode() =>
|
||||
{
|
||||
Some(app.submit_notify_input())
|
||||
}
|
||||
KeyCode::Char(c)
|
||||
if c.eq_ignore_ascii_case(&'q') && alt && !ctrl && !app.is_command_mode() =>
|
||||
{
|
||||
if app.restore_next_queued_input_to_composer() {
|
||||
Some(app.refresh_completion())
|
||||
} else {
|
||||
Some(None)
|
||||
}
|
||||
Some(app.continue_pending_method())
|
||||
}
|
||||
KeyCode::Char(c) if c.eq_ignore_ascii_case(&'c') && alt && !ctrl => {
|
||||
app.clear_queued_inputs();
|
||||
Some(None)
|
||||
Some(Some(app.clear_pending_method()))
|
||||
}
|
||||
KeyCode::Char('c') if ctrl => Some(handle_pause_or_quit(app)),
|
||||
KeyCode::Char('x') if ctrl => Some(handle_cancel_or_shutdown(app)),
|
||||
@@ -1427,7 +1438,6 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
|
||||
WorkerStatus::Running | WorkerStatus::Paused
|
||||
) {
|
||||
app.shutdown_confirm = None;
|
||||
app.clear_queued_inputs();
|
||||
return Some(Method::Cancel);
|
||||
}
|
||||
if let Some(pressed_at) = app.shutdown_confirm
|
||||
@@ -1450,7 +1460,6 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
|
||||
/// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running).
|
||||
fn handle_pause_or_quit(app: &mut App) -> Option<Method> {
|
||||
if app.worker_status == WorkerStatus::Running {
|
||||
app.clear_queued_inputs();
|
||||
return Some(Method::Pause);
|
||||
}
|
||||
if let Some(t) = app.quit_confirm
|
||||
@@ -1476,8 +1485,8 @@ mod tests {
|
||||
use crate::text_selection::{HistoryViewport, SelectionRow};
|
||||
use async_trait::async_trait;
|
||||
use protocol::{
|
||||
Event, RewindTarget, RewindTargetId, RunResult, Segment, UploadedFileAvailability,
|
||||
UploadedFileRef, WorkerStatus,
|
||||
Event, RewindTarget, RewindTargetId, Segment, UploadedFileAvailability, UploadedFileRef,
|
||||
WorkerStatus,
|
||||
};
|
||||
|
||||
#[test]
|
||||
@@ -1490,7 +1499,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn client_local_attachment_commands_are_typed_and_do_not_send_the_path() {
|
||||
let attach = Method::Run {
|
||||
let attach = Method::Submit {
|
||||
submission_request_id: protocol::new_submission_request_id(),
|
||||
input: vec![Segment::text("/attach /tmp/report.md")],
|
||||
};
|
||||
assert_eq!(
|
||||
@@ -1499,7 +1509,8 @@ mod tests {
|
||||
);
|
||||
assert!(!is_clear_attachments_command(&attach));
|
||||
|
||||
let clear = Method::Run {
|
||||
let clear = Method::Submit {
|
||||
submission_request_id: protocol::new_submission_request_id(),
|
||||
input: vec![Segment::text("/clear-attachments")],
|
||||
};
|
||||
assert!(is_clear_attachments_command(&clear));
|
||||
@@ -1605,7 +1616,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queued_attachment_send_failure_restores_draft_without_exiting_console() {
|
||||
async fn running_attachment_submit_failure_restores_draft_without_exiting_console() {
|
||||
let file = UploadedFileRef {
|
||||
artifact_id: "artifact-queued".into(),
|
||||
file_name: "queued.txt".into(),
|
||||
@@ -1631,13 +1642,10 @@ mod tests {
|
||||
let mut app = App::new("worker".into());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
app.input.insert_str("queued inspect");
|
||||
assert!(app.submit_input().is_none());
|
||||
|
||||
let method = app
|
||||
.handle_worker_event(Event::RunEnd {
|
||||
result: RunResult::Finished,
|
||||
})
|
||||
.expect("queued run must be released");
|
||||
.submit_input()
|
||||
.expect("running Submit is sent immediately");
|
||||
|
||||
send_console_method(&mut app, &mut connection, &method)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1960,7 +1968,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn running_enter_queues_instead_of_sending_run() {
|
||||
fn running_enter_sends_submit_to_worker() {
|
||||
let mut app = App::new("agent".to_string());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
for c in "queued".chars() {
|
||||
@@ -1973,102 +1981,128 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none());
|
||||
assert!(matches!(
|
||||
handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)),
|
||||
Some(Method::Submit { .. })
|
||||
));
|
||||
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
assert_eq!(app.next_queued_input_preview(), Some("queued"));
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
assert_eq!(input_text(&app), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_input_keybindings_restore_and_clear() {
|
||||
let mut app = App::new("agent".to_string());
|
||||
fn running_alt_n_sends_explicit_notify_without_implicit_submit_conversion() {
|
||||
let mut app = App::new("test".into());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
for c in "edit queued".chars() {
|
||||
assert!(
|
||||
handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
for character in "progress".chars() {
|
||||
app.insert_char(character);
|
||||
}
|
||||
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none());
|
||||
|
||||
assert!(
|
||||
handle_key(
|
||||
let method = handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('q'), KeyModifiers::ALT)
|
||||
)
|
||||
.is_none()
|
||||
KeyEvent::new(KeyCode::Char('n'), KeyModifiers::ALT),
|
||||
);
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
assert_eq!(input_text(&app), "edit queued");
|
||||
|
||||
app.input.clear();
|
||||
for c in "clear queued".chars() {
|
||||
assert!(
|
||||
handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none());
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
|
||||
assert!(
|
||||
handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::ALT)
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
assert!(matches!(
|
||||
method,
|
||||
Some(Method::Notify {
|
||||
ref message,
|
||||
auto_run: true,
|
||||
..
|
||||
}) if message == "progress"
|
||||
));
|
||||
assert_eq!(input_text(&app), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pause_and_cancel_clear_queued_input() {
|
||||
let mut app = App::new("agent".to_string());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
for c in "queued".chars() {
|
||||
assert!(
|
||||
handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none());
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
fn pending_queue_shortcuts_send_worker_operations() {
|
||||
let mut app = App::new("test".into());
|
||||
app.handle_worker_event(Event::PendingSubmissionsChanged {
|
||||
pending: protocol::PendingSubmissionsSnapshot {
|
||||
revision: 2,
|
||||
notification_count: 0,
|
||||
head_id: Some("submission-1".into()),
|
||||
submissions: vec![protocol::PendingSubmissionSummary {
|
||||
submission_id: "submission-1".into(),
|
||||
accepted_at_ms: 1,
|
||||
segment_count: 1,
|
||||
byte_len: 6,
|
||||
}],
|
||||
},
|
||||
});
|
||||
|
||||
let pause = handle_key(
|
||||
let continue_next = handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL),
|
||||
KeyEvent::new(KeyCode::Char('q'), KeyModifiers::ALT),
|
||||
);
|
||||
assert!(matches!(pause, Some(Method::Pause)));
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
|
||||
for c in "queued again".chars() {
|
||||
assert!(
|
||||
handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE)
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none());
|
||||
assert!(matches!(
|
||||
continue_next,
|
||||
Some(Method::ContinuePending {
|
||||
expected_revision: 2,
|
||||
ref expected_head_id,
|
||||
}) if expected_head_id == "submission-1"
|
||||
));
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
|
||||
let cancel = handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
|
||||
KeyEvent::new(KeyCode::Char('d'), KeyModifiers::ALT),
|
||||
);
|
||||
assert!(matches!(cancel, Some(Method::Cancel)));
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
assert!(matches!(
|
||||
cancel,
|
||||
Some(Method::CancelPendingSubmission {
|
||||
expected_revision: 2,
|
||||
ref submission_id,
|
||||
}) if submission_id == "submission-1"
|
||||
));
|
||||
|
||||
let clear = handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::ALT),
|
||||
);
|
||||
assert!(matches!(
|
||||
clear,
|
||||
Some(Method::ClearPendingSubmissions {
|
||||
expected_revision: 2
|
||||
})
|
||||
));
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pause_and_cancel_preserve_authoritative_pending_queue() {
|
||||
let mut app = App::new("test".into());
|
||||
app.handle_worker_event(Event::PendingSubmissionsChanged {
|
||||
pending: protocol::PendingSubmissionsSnapshot {
|
||||
revision: 2,
|
||||
notification_count: 0,
|
||||
head_id: Some("submission-1".into()),
|
||||
submissions: vec![protocol::PendingSubmissionSummary {
|
||||
submission_id: "submission-1".into(),
|
||||
accepted_at_ms: 1,
|
||||
segment_count: 1,
|
||||
byte_len: 6,
|
||||
}],
|
||||
},
|
||||
});
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
assert!(matches!(
|
||||
handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL),
|
||||
),
|
||||
Some(Method::Pause)
|
||||
));
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
assert!(matches!(
|
||||
handle_key(
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
|
||||
),
|
||||
Some(Method::Cancel)
|
||||
));
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2535,13 +2569,19 @@ mod tests {
|
||||
let mut app = App::new("agent".to_string());
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
greeting: test_greeting(),
|
||||
session: protocol::SessionSnapshot { entries: vec![] },
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![],
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
});
|
||||
app.handle_worker_event(Event::RewindApplied {
|
||||
session: protocol::SessionSnapshot { entries: vec![] },
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![],
|
||||
},
|
||||
input: vec![Segment::Text {
|
||||
content: "retry this".into(),
|
||||
}],
|
||||
@@ -2562,7 +2602,10 @@ mod tests {
|
||||
let mut app = App::new("agent".to_string());
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
greeting: test_greeting(),
|
||||
session: protocol::SessionSnapshot { entries: vec![] },
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![],
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
@@ -2570,7 +2613,10 @@ mod tests {
|
||||
type_keys(&mut app, "draft");
|
||||
|
||||
app.handle_worker_event(Event::RewindApplied {
|
||||
session: protocol::SessionSnapshot { entries: vec![] },
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![],
|
||||
},
|
||||
input: vec![Segment::Text {
|
||||
content: "retry this".into(),
|
||||
}],
|
||||
@@ -2918,12 +2964,12 @@ mod tests {
|
||||
type_keys(&mut app, "first");
|
||||
assert!(matches!(
|
||||
handle_key(&mut app, key(KeyCode::Enter)),
|
||||
Some(Method::Run { .. })
|
||||
Some(Method::Submit { .. })
|
||||
));
|
||||
type_keys(&mut app, "second");
|
||||
assert!(matches!(
|
||||
handle_key(&mut app, key(KeyCode::Enter)),
|
||||
Some(Method::Run { .. })
|
||||
Some(Method::Submit { .. })
|
||||
));
|
||||
|
||||
assert_eq!(input_text(&app), "");
|
||||
@@ -2954,7 +3000,7 @@ mod tests {
|
||||
type_keys(&mut app, "sent");
|
||||
assert!(matches!(
|
||||
handle_key(&mut app, key(KeyCode::Enter)),
|
||||
Some(Method::Run { .. })
|
||||
Some(Method::Submit { .. })
|
||||
));
|
||||
type_keys(&mut app, "draft\nbody");
|
||||
app.move_cursor_start();
|
||||
|
||||
+22
-14
@@ -1880,7 +1880,7 @@ fn actionbar_left_item(app: &App, now: Instant) -> Option<(String, Style)> {
|
||||
}
|
||||
if app.queued_input_count() > 0 {
|
||||
return Some((
|
||||
"Alt-q edit queued Alt-c clear queued".to_string(),
|
||||
"Alt-n notify Alt-q continue Alt-d cancel queued Alt-c clear queued".to_string(),
|
||||
Style::default().fg(Color::DarkGray),
|
||||
));
|
||||
}
|
||||
@@ -2136,9 +2136,25 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::app::{ActionbarNoticeLevel, ActionbarNoticeSource, App};
|
||||
use crate::block::{ToolCallBlock, ToolCallState};
|
||||
use protocol::WorkerStatus;
|
||||
use protocol::Event;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
fn set_pending_submission(app: &mut App, id: &str) {
|
||||
app.handle_worker_event(Event::PendingSubmissionsChanged {
|
||||
pending: protocol::PendingSubmissionsSnapshot {
|
||||
revision: 1,
|
||||
notification_count: 0,
|
||||
head_id: Some(id.into()),
|
||||
submissions: vec![protocol::PendingSubmissionSummary {
|
||||
submission_id: id.into(),
|
||||
accepted_at_ms: 1,
|
||||
segment_count: 1,
|
||||
byte_len: 1,
|
||||
}],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_status_line_matches_console_metrics_and_spinner_frame() {
|
||||
let now = Instant::now();
|
||||
@@ -2251,15 +2267,11 @@ mod tests {
|
||||
#[test]
|
||||
fn queue_status_text_includes_count_and_preview() {
|
||||
let mut app = App::new("test".into());
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
for c in "queued preview".chars() {
|
||||
app.insert_char(c);
|
||||
}
|
||||
assert!(app.submit_input().is_none());
|
||||
set_pending_submission(&mut app, "submission-1");
|
||||
|
||||
assert_eq!(
|
||||
queue_status_text(&app),
|
||||
Some("queued: 1 — queued preview".to_string())
|
||||
Some("queued: 1 — submission-1".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2289,14 +2301,10 @@ mod tests {
|
||||
Some("Worker keeps running. Press Ctrl-C again to exit TUI.".into())
|
||||
);
|
||||
|
||||
app.set_worker_status(WorkerStatus::Running);
|
||||
for c in "queued turn".chars() {
|
||||
app.insert_char(c);
|
||||
}
|
||||
assert!(app.submit_input().is_none());
|
||||
set_pending_submission(&mut app, "submission-1");
|
||||
assert_eq!(
|
||||
actionbar_left_item(&app, now).map(|(text, _)| text),
|
||||
Some("Alt-q edit queued Alt-c clear queued".into())
|
||||
Some("Alt-n notify Alt-q continue Alt-d cancel queued Alt-c clear queued".into())
|
||||
);
|
||||
|
||||
app.enter_command_mode();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -68,12 +68,10 @@ pub enum WorkdirSessionOperation {
|
||||
CommandCancel(CommandHandle),
|
||||
}
|
||||
|
||||
/// Wire envelope for an operation and its optional provider-enforced child scope.
|
||||
/// Wire envelope for one provider operation.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkdirSessionOperationRequest {
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub delegations: Vec<crate::WorkdirDelegationRequest>,
|
||||
pub operation: WorkdirSessionOperation,
|
||||
}
|
||||
|
||||
@@ -289,7 +287,7 @@ mod client {
|
||||
use reqwest::{Client, StatusCode, Url};
|
||||
|
||||
use super::*;
|
||||
use crate::{Workdir, WorkdirSession, WorkdirSessionHandle};
|
||||
use crate::{Workdir, WorkdirSession};
|
||||
|
||||
/// Provides a fresh bearer token for each Runtime request. Backend
|
||||
/// implementations can mint short-lived capability tokens without making a
|
||||
@@ -324,7 +322,6 @@ mod client {
|
||||
workdir: Workdir,
|
||||
session_id: WorkdirSessionId,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
delegations: Vec<crate::WorkdirDelegationRequest>,
|
||||
closed: AtomicBool,
|
||||
}
|
||||
|
||||
@@ -377,7 +374,6 @@ mod client {
|
||||
workdir: Workdir::new(opened.workdir_id.as_str()),
|
||||
session_id: opened.session_id,
|
||||
capabilities: opened.capabilities,
|
||||
delegations: Vec::new(),
|
||||
closed: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
@@ -404,10 +400,7 @@ mod client {
|
||||
"operations",
|
||||
],
|
||||
)?;
|
||||
let operation = WorkdirSessionOperationRequest {
|
||||
delegations: self.delegations.clone(),
|
||||
operation,
|
||||
};
|
||||
let operation = WorkdirSessionOperationRequest { operation };
|
||||
let response = self
|
||||
.client
|
||||
.post(url)
|
||||
@@ -436,37 +429,6 @@ mod client {
|
||||
self.capabilities
|
||||
}
|
||||
|
||||
fn transports_delegation_context(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn capture_delegation_source(
|
||||
&self,
|
||||
request: &crate::WorkdirDelegationRequest,
|
||||
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||
if self.closed.load(Ordering::Acquire) {
|
||||
return Err(WorkdirError::SessionClosed);
|
||||
}
|
||||
let mut delegations = self.delegations.clone();
|
||||
delegations.push(request.clone());
|
||||
let candidate = Arc::new(Self {
|
||||
client: self.client.clone(),
|
||||
base_url: self.base_url.clone(),
|
||||
authorization: self.authorization.clone(),
|
||||
workdir: self.workdir.clone(),
|
||||
session_id: self.session_id.clone(),
|
||||
capabilities: self.capabilities,
|
||||
delegations,
|
||||
closed: AtomicBool::new(false),
|
||||
});
|
||||
candidate
|
||||
.stat(StatRequest {
|
||||
path: fs_operation::FsPath::new("").expect("empty Workdir path is valid"),
|
||||
})
|
||||
.await?;
|
||||
Ok(candidate)
|
||||
}
|
||||
|
||||
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError> {
|
||||
match self.operate(WorkdirSessionOperation::Stat(request)).await? {
|
||||
WorkdirSessionOperationResult::Stat(result) => Ok(result),
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
//! bound to one Worker. Tools consume sessions; they do not own Workdir
|
||||
//! materialization or cleanup.
|
||||
|
||||
mod delegation;
|
||||
pub mod http;
|
||||
mod local;
|
||||
mod operation;
|
||||
mod scope;
|
||||
pub mod workspace;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -18,11 +18,6 @@ use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
pub use delegation::{
|
||||
AppliedWorkdirDelegation, ReadOnlyWorkdirSession, WorkdirDelegation,
|
||||
WorkdirDelegationPermission, WorkdirDelegationRequest, WorkdirDelegationRule,
|
||||
apply_delegation_chain, delegation_capable_session,
|
||||
};
|
||||
pub use fs_operation::{
|
||||
ContentHash, EditRequest, EditResult, EntryKind, FsPath as WorkdirPath, GlobRequest,
|
||||
GlobResult, GrepOutputMode, GrepRequest, GrepResult, ListEntry, ListRequest, ListResult,
|
||||
@@ -32,6 +27,10 @@ pub use local::{
|
||||
LocalWorkdirSession, SymlinkInfo, WorkdirSessionResource, direct_symlink, first_symlink,
|
||||
};
|
||||
pub use operation::*;
|
||||
pub use scope::{
|
||||
ReadOnlyWorkdirSession, WorkdirScopeLease, WorkdirToolBroker, WorkdirToolScope,
|
||||
WorkdirToolScopePermission, WorkdirToolScopeRule,
|
||||
};
|
||||
|
||||
/// Persistent, opaque identity of one materialized Workdir.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
@@ -148,39 +147,6 @@ pub trait WorkdirSession: std::fmt::Debug + Send + Sync {
|
||||
fn workdir(&self) -> &Workdir;
|
||||
fn capabilities(&self) -> WorkdirSessionCapabilities;
|
||||
|
||||
fn is_delegation_capable(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Whether this session transports the delegation chain to another
|
||||
/// provider boundary that will apply logical cwd/path resolution there.
|
||||
fn transports_delegation_context(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Capture a provider-specific source for a delegated child session.
|
||||
/// Remote providers use this boundary to pin attachment identity without
|
||||
/// exposing transport handles or host paths.
|
||||
async fn capture_delegation_source(
|
||||
&self,
|
||||
_request: &WorkdirDelegationRequest,
|
||||
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||
Err(WorkdirError::Denied(
|
||||
"workdir provider does not support delegated sessions".into(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Attenuate this session into a revocable child lease. Only sessions
|
||||
/// created with [`delegation_capable_session`] implement this operation.
|
||||
async fn delegate(
|
||||
&self,
|
||||
_request: WorkdirDelegationRequest,
|
||||
) -> Result<WorkdirDelegation, WorkdirError> {
|
||||
Err(WorkdirError::Denied(
|
||||
"workdir session is not delegation-capable".into(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError>;
|
||||
async fn read(&self, request: ReadRequest) -> Result<ReadResult, WorkdirError>;
|
||||
async fn write(&self, request: WriteRequest) -> Result<WriteResult, WorkdirError>;
|
||||
|
||||
+23
-69
@@ -18,7 +18,7 @@ use std::sync::{Arc, Mutex as StdMutex};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use manifest::{Permission, Scope, ScopeConfig, ScopeRule, SharedScope};
|
||||
use manifest::{Scope, SharedScope};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::{Mutex, broadcast, watch};
|
||||
@@ -28,10 +28,8 @@ use crate::{
|
||||
CommandEvent, CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest,
|
||||
CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice, EditRequest, EditResult,
|
||||
GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, ListResult, ReadRequest,
|
||||
ReadResult, StatRequest, StatResult, Workdir, WorkdirDelegationPermission,
|
||||
WorkdirDelegationRequest, WorkdirError, WorkdirPath, WorkdirSession,
|
||||
WorkdirSessionCapabilities, WorkdirSessionCapability, WorkdirSessionHandle, WriteRequest,
|
||||
WriteResult,
|
||||
ReadResult, StatRequest, StatResult, Workdir, WorkdirError, WorkdirPath, WorkdirSession,
|
||||
WorkdirSessionCapabilities, WorkdirSessionCapability, WriteRequest, WriteResult,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::{EntryKind, WriteOutcome};
|
||||
@@ -558,69 +556,6 @@ impl WorkdirSession for LocalWorkdirSession {
|
||||
self.inner.capabilities
|
||||
}
|
||||
|
||||
async fn capture_delegation_source(
|
||||
&self,
|
||||
request: &WorkdirDelegationRequest,
|
||||
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||
let host_rules = request
|
||||
.rules
|
||||
.iter()
|
||||
.map(|rule| ScopeRule {
|
||||
target: self.inner.root.join(rule.target.as_str()),
|
||||
permission: match rule.permission {
|
||||
WorkdirDelegationPermission::Read => Permission::Read,
|
||||
WorkdirDelegationPermission::Write => Permission::Write,
|
||||
},
|
||||
recursive: rule.recursive,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for (logical, host) in request.rules.iter().zip(&host_rules) {
|
||||
if logical.permission == WorkdirDelegationPermission::Write {
|
||||
let resolved = Scope::resolved_target(host)
|
||||
.map_err(|error| WorkdirError::Denied(error.to_string()))?;
|
||||
if resolved != host.target {
|
||||
return Err(WorkdirError::Denied(format!(
|
||||
"write delegation target `{}` traverses a symlink",
|
||||
logical.target
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
let parent_scope = self.inner.scope.snapshot();
|
||||
for rule in &host_rules {
|
||||
if !parent_scope
|
||||
.allows_rule(rule)
|
||||
.map_err(|error| WorkdirError::Denied(error.to_string()))?
|
||||
{
|
||||
return Err(WorkdirError::Denied(format!(
|
||||
"delegated provider scope `{}` exceeds the parent session",
|
||||
rule.target.display()
|
||||
)));
|
||||
}
|
||||
}
|
||||
let child_scope = Scope::from_config(&ScopeConfig {
|
||||
allow: host_rules,
|
||||
deny: Vec::new(),
|
||||
})
|
||||
.map_err(|error| WorkdirError::Denied(error.to_string()))?;
|
||||
let child_cwd = self.inner.root.join(request.cwd.as_str());
|
||||
if !child_scope.is_readable(&child_cwd)
|
||||
|| !std::fs::metadata(&child_cwd).is_ok_and(|metadata| metadata.is_dir())
|
||||
{
|
||||
return Err(WorkdirError::Denied(format!(
|
||||
"delegated cwd `{}` is not a readable Workdir directory",
|
||||
request.cwd
|
||||
)));
|
||||
}
|
||||
Ok(Arc::new(LocalWorkdirSession::materialized_bound(
|
||||
self.inner.workdir.clone(),
|
||||
self.inner.root.clone(),
|
||||
self.inner.root.clone(),
|
||||
SharedScope::new(child_scope),
|
||||
self.inner.capabilities,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError> {
|
||||
self.ensure_capability(WorkdirSessionCapability::Read)?;
|
||||
let logical = request.path.clone();
|
||||
@@ -694,9 +629,20 @@ impl WorkdirSession for LocalWorkdirSession {
|
||||
{
|
||||
return Err(WorkdirError::OutOfScope(spill_dir.to_path_buf()));
|
||||
}
|
||||
let cwd = if let Some(logical_cwd) = request.cwd.as_ref() {
|
||||
let cwd = self.resolve(logical_cwd);
|
||||
let scope = self.inner.scope.snapshot();
|
||||
if !scope.is_readable(&cwd)
|
||||
|| !std::fs::metadata(&cwd).is_ok_and(|metadata| metadata.is_dir())
|
||||
{
|
||||
return Err(WorkdirError::OutOfScope(cwd));
|
||||
}
|
||||
cwd
|
||||
} else {
|
||||
self.inner.cwd.clone()
|
||||
};
|
||||
let id = self.inner.next_command_id.fetch_add(1, Ordering::Relaxed);
|
||||
let handle = CommandHandle(format!("command-{id}"));
|
||||
let cwd = self.inner.cwd.clone();
|
||||
let (completion_tx, completion) = watch::channel(false);
|
||||
let command_id = handle.0.clone();
|
||||
let telemetry = self.inner.command_telemetry.clone();
|
||||
@@ -1516,6 +1462,7 @@ mod tests {
|
||||
command: "sleep 30".to_owned(),
|
||||
timeout_secs: 60,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: None,
|
||||
},
|
||||
@@ -2043,6 +1990,7 @@ mod tests {
|
||||
command: "pwd && printf provider-command".into(),
|
||||
timeout_secs: 5,
|
||||
output_limit: 4096,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: None,
|
||||
},
|
||||
@@ -2141,6 +2089,7 @@ mod tests {
|
||||
command: "printf hidden".into(),
|
||||
timeout_secs: 5,
|
||||
output_limit: 1,
|
||||
cwd: None,
|
||||
spill_dir: Some(spill.path().to_path_buf()),
|
||||
tool_call_id: None,
|
||||
},
|
||||
@@ -2178,6 +2127,7 @@ mod tests {
|
||||
command: "i=0; while [ $i -lt 200 ]; do printf 'line-%03d\\n' \"$i\"; i=$((i+1)); done; printf 'FINAL-NEEDLE\\n'".into(),
|
||||
timeout_secs: 5,
|
||||
output_limit: 64,
|
||||
cwd: None,
|
||||
spill_dir: Some(spill.path().to_path_buf()),
|
||||
tool_call_id: None,
|
||||
},
|
||||
@@ -2224,6 +2174,7 @@ mod tests {
|
||||
command: "printf 'aéz'".into(),
|
||||
timeout_secs: 5,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: None,
|
||||
},
|
||||
@@ -2449,6 +2400,7 @@ mod tests {
|
||||
command: "printf ready; printf warning >&2; sleep 0.2; printf done".into(),
|
||||
timeout_secs: 5,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: Some("tool-7".into()),
|
||||
},
|
||||
@@ -2553,6 +2505,7 @@ mod tests {
|
||||
command: "sleep 30".into(),
|
||||
timeout_secs: 1,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: None,
|
||||
},
|
||||
@@ -2623,6 +2576,7 @@ mod tests {
|
||||
command: "sleep 30".into(),
|
||||
timeout_secs: 60,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: None,
|
||||
},
|
||||
|
||||
@@ -11,6 +11,10 @@ pub struct CommandRequest {
|
||||
pub command: String,
|
||||
pub timeout_secs: u64,
|
||||
pub output_limit: usize,
|
||||
/// Workdir-relative command directory. Providers validate it against the
|
||||
/// active session before process start.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cwd: Option<fs_operation::FsPath>,
|
||||
/// Provider-local directory where complete output is retained when the
|
||||
/// inline result exceeds `output_limit`.
|
||||
pub spill_dir: Option<PathBuf>,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -104,15 +104,5 @@ mod tests {
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkspaceWorkdirSessionOperationRequest {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub expected_session_fence: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub delegations: Vec<crate::WorkdirDelegationRequest>,
|
||||
pub operation: crate::http::WorkdirSessionOperation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkspaceWorkdirSessionFence {
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
@@ -41,14 +41,12 @@ pub enum WorkerExecutionOperation {
|
||||
Cancel,
|
||||
}
|
||||
|
||||
/// Evidence that a user input reached the durable Worker session boundary.
|
||||
///
|
||||
/// This is intentionally distinct from accepting a method on the Worker's
|
||||
/// in-memory channel. For Flow submissions, the committed UserInput entry also
|
||||
/// carries the initial Flow runtime-state extension.
|
||||
/// Evidence that a Submit request reached the durable Worker session boundary.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct WorkerInputCommitAck {
|
||||
pub struct WorkerSubmissionAck {
|
||||
pub submission_request_id: String,
|
||||
pub submission_id: String,
|
||||
pub disposition: protocol::SubmissionDisposition,
|
||||
}
|
||||
|
||||
/// Typed execution result class. Results are transient operation outcomes and
|
||||
@@ -61,7 +59,7 @@ pub struct WorkerExecutionResult {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub message: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_commit: Option<WorkerInputCommitAck>,
|
||||
pub submission: Option<WorkerSubmissionAck>,
|
||||
}
|
||||
|
||||
/// Backend result class for a Worker execution operation.
|
||||
@@ -85,22 +83,26 @@ impl WorkerExecutionResult {
|
||||
outcome: WorkerExecutionOutcome::Accepted,
|
||||
run_state,
|
||||
message: None,
|
||||
input_commit: None,
|
||||
submission: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn accepted_input_committed(
|
||||
pub fn accepted_submission(
|
||||
operation: WorkerExecutionOperation,
|
||||
run_state: WorkerExecutionRunState,
|
||||
submission_request_id: impl Into<String>,
|
||||
submission_id: impl Into<String>,
|
||||
disposition: protocol::SubmissionDisposition,
|
||||
) -> Self {
|
||||
Self {
|
||||
operation,
|
||||
outcome: WorkerExecutionOutcome::Accepted,
|
||||
run_state,
|
||||
message: None,
|
||||
input_commit: Some(WorkerInputCommitAck {
|
||||
submission: Some(WorkerSubmissionAck {
|
||||
submission_request_id: submission_request_id.into(),
|
||||
submission_id: submission_id.into(),
|
||||
disposition,
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -111,7 +113,7 @@ impl WorkerExecutionResult {
|
||||
outcome: WorkerExecutionOutcome::Busy,
|
||||
run_state: WorkerExecutionRunState::Busy,
|
||||
message: Some(message.into()),
|
||||
input_commit: None,
|
||||
submission: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,7 +123,7 @@ impl WorkerExecutionResult {
|
||||
outcome: WorkerExecutionOutcome::Rejected,
|
||||
run_state: WorkerExecutionRunState::Stopped,
|
||||
message: Some(message.into()),
|
||||
input_commit: None,
|
||||
submission: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,7 +133,7 @@ impl WorkerExecutionResult {
|
||||
outcome: WorkerExecutionOutcome::Errored,
|
||||
run_state: WorkerExecutionRunState::Errored,
|
||||
message: Some(message.into()),
|
||||
input_commit: None,
|
||||
submission: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -141,7 +143,7 @@ impl WorkerExecutionResult {
|
||||
outcome: WorkerExecutionOutcome::Unsupported,
|
||||
run_state: WorkerExecutionRunState::Stopped,
|
||||
message: Some(message.into()),
|
||||
input_commit: None,
|
||||
submission: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -618,14 +620,17 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn input_commit_ack_survives_json_round_trip() {
|
||||
let result = WorkerExecutionResult::accepted_input_committed(
|
||||
fn submission_ack_survives_json_round_trip() {
|
||||
let result = WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
"request-1",
|
||||
"submission-1",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
);
|
||||
|
||||
let json = serde_json::to_string(&result).unwrap();
|
||||
assert!(json.contains("\"submission_request_id\":\"request-1\""));
|
||||
assert!(json.contains("\"submission_id\":\"submission-1\""));
|
||||
assert_eq!(
|
||||
serde_json::from_str::<WorkerExecutionResult>(&json).unwrap(),
|
||||
|
||||
@@ -774,8 +774,7 @@ async fn run_workdir_session_operation(
|
||||
.ok_or_else(RuntimeHttpWorkdirError::not_found)?;
|
||||
record.session.clone()
|
||||
};
|
||||
let applied = workdir::apply_delegation_chain(source, request.delegations).await?;
|
||||
let session = applied.scoped_session.as_ref();
|
||||
let session = source.as_ref();
|
||||
let operation = request.operation;
|
||||
|
||||
let result = match operation {
|
||||
@@ -1240,10 +1239,12 @@ async fn worker_protocol_ws(
|
||||
auth: Option<Extension<RuntimeAuthContext>>,
|
||||
Path(worker_id): Path<String>,
|
||||
Query(query): Query<RuntimeWorkerEventsWsQuery>,
|
||||
headers: HeaderMap,
|
||||
ws: WebSocketUpgrade,
|
||||
) -> Result<Response, RuntimeHttpRestError> {
|
||||
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
|
||||
let scope = auth_workspace_scope(&state, auth.as_ref())?;
|
||||
let input_source = authenticated_protocol_input_source(&headers)?;
|
||||
match scope.as_ref() {
|
||||
Some(scope) => state
|
||||
.runtime
|
||||
@@ -1254,17 +1255,89 @@ async fn worker_protocol_ws(
|
||||
.map_err(RuntimeHttpRestError::runtime)?;
|
||||
Ok(ws
|
||||
.on_upgrade(move |socket| {
|
||||
worker_protocol_ws_session(state.runtime, scope, worker_ref, query, socket)
|
||||
worker_protocol_ws_session(
|
||||
state.runtime,
|
||||
scope,
|
||||
worker_ref,
|
||||
query,
|
||||
input_source,
|
||||
socket,
|
||||
)
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
fn authenticated_protocol_input_source(
|
||||
headers: &HeaderMap,
|
||||
) -> Result<Option<protocol::AuthenticatedInputSource>, RuntimeHttpRestError> {
|
||||
let Some(value) = headers.get(protocol::AUTHENTICATED_ACCOUNT_ID_HEADER) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let account_id = value.to_str().map_err(|_| {
|
||||
RuntimeHttpRestError::new(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"authenticated_input_source_invalid",
|
||||
"authenticated Worker input source is invalid",
|
||||
)
|
||||
})?;
|
||||
if account_id.trim().is_empty() || account_id.len() > 128 {
|
||||
return Err(RuntimeHttpRestError::new(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"authenticated_input_source_invalid",
|
||||
"authenticated Worker input source is invalid",
|
||||
));
|
||||
}
|
||||
Ok(Some(protocol::AuthenticatedInputSource::Account {
|
||||
account_id: account_id.to_owned(),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
fn authorize_runtime_protocol_method(
|
||||
method: protocol::Method,
|
||||
transport_source: Option<&protocol::AuthenticatedInputSource>,
|
||||
) -> protocol::Method {
|
||||
match method {
|
||||
protocol::Method::SubmitTracked {
|
||||
submission_request_id,
|
||||
input,
|
||||
..
|
||||
} => protocol::Method::SubmitTracked {
|
||||
source: transport_source.cloned().unwrap_or_else(|| {
|
||||
protocol::AuthenticatedInputSource::Backend {
|
||||
operation_id: submission_request_id.clone(),
|
||||
}
|
||||
}),
|
||||
submission_request_id,
|
||||
input,
|
||||
},
|
||||
protocol::Method::NotifyTracked {
|
||||
notification_request_id,
|
||||
message,
|
||||
auto_run,
|
||||
..
|
||||
} => protocol::Method::NotifyTracked {
|
||||
source: transport_source.cloned().unwrap_or_else(|| {
|
||||
protocol::AuthenticatedInputSource::Backend {
|
||||
operation_id: notification_request_id.clone(),
|
||||
}
|
||||
}),
|
||||
notification_request_id,
|
||||
message,
|
||||
auto_run,
|
||||
},
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
async fn worker_protocol_ws_session(
|
||||
runtime: Runtime,
|
||||
scope: Option<RuntimeWorkspaceScope>,
|
||||
worker_ref: WorkerRef,
|
||||
query: RuntimeWorkerEventsWsQuery,
|
||||
input_source: Option<protocol::AuthenticatedInputSource>,
|
||||
mut socket: WebSocket,
|
||||
) {
|
||||
let mut cursor = match query.cursor.as_deref() {
|
||||
@@ -1347,6 +1420,8 @@ async fn worker_protocol_ws_session(
|
||||
match inbound {
|
||||
Some(Ok(WsMessage::Text(text))) => match decode_method(&text) {
|
||||
Ok(method) => {
|
||||
let method =
|
||||
authorize_runtime_protocol_method(method, input_source.as_ref());
|
||||
let result = match scope.as_ref() {
|
||||
Some(scope) => {
|
||||
runtime.send_protocol_method_scoped(scope, &worker_ref, method)
|
||||
@@ -2139,8 +2214,8 @@ mod tests {
|
||||
use manifest::{Scope, SharedScope};
|
||||
use tower::ServiceExt;
|
||||
use workdir::{
|
||||
GrepOutputMode, GrepRequest, LocalWorkdirSession, ReadRequest, StatRequest, Workdir,
|
||||
WorkdirPath, WorkdirSessionCapabilities,
|
||||
GrepOutputMode, GrepRequest, LocalWorkdirSession, StatRequest, Workdir, WorkdirPath,
|
||||
WorkdirSessionCapabilities,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2219,6 +2294,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_protocol_replaces_serialized_tracked_source() {
|
||||
let wire = serde_json::to_string(&protocol::Method::SubmitTracked {
|
||||
submission_request_id: "request-1".into(),
|
||||
input: vec![protocol::Segment::text("hello")],
|
||||
source: protocol::AuthenticatedInputSource::Account {
|
||||
account_id: "forged".into(),
|
||||
},
|
||||
})
|
||||
.unwrap();
|
||||
let decoded: protocol::Method = serde_json::from_str(&wire).unwrap();
|
||||
assert!(matches!(
|
||||
decoded,
|
||||
protocol::Method::SubmitTracked {
|
||||
source: protocol::AuthenticatedInputSource::UntrustedWire,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert!(matches!(
|
||||
authorize_runtime_protocol_method(decoded, None),
|
||||
protocol::Method::SubmitTracked {
|
||||
source: protocol::AuthenticatedInputSource::Backend { operation_id },
|
||||
..
|
||||
} if operation_id == "request-1"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_protocol_uses_transport_authenticated_account_source() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
protocol::AUTHENTICATED_ACCOUNT_ID_HEADER,
|
||||
"account-1".parse().unwrap(),
|
||||
);
|
||||
let source = authenticated_protocol_input_source(&headers)
|
||||
.unwrap()
|
||||
.expect("account source header must resolve");
|
||||
let wire = serde_json::to_string(&protocol::Method::NotifyTracked {
|
||||
notification_request_id: "notification-1".into(),
|
||||
message: "hello".into(),
|
||||
auto_run: true,
|
||||
source: protocol::AuthenticatedInputSource::Account {
|
||||
account_id: "forged".into(),
|
||||
},
|
||||
})
|
||||
.unwrap();
|
||||
let decoded: protocol::Method = serde_json::from_str(&wire).unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
authorize_runtime_protocol_method(decoded, Some(&source)),
|
||||
protocol::Method::NotifyTracked {
|
||||
source: protocol::AuthenticatedInputSource::Account { account_id },
|
||||
..
|
||||
} if account_id == "account-1"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attachment_routes_require_worker_input_permission() {
|
||||
assert_eq!(
|
||||
@@ -2637,16 +2769,6 @@ mod tests {
|
||||
async fn workdir_session_operations_enforce_owner_and_close_terminally() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
std::fs::write(temp.path().join("hello.txt"), "hello").expect("write fixture");
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::symlink;
|
||||
std::fs::create_dir(temp.path().join("granted")).expect("granted directory");
|
||||
std::fs::write(temp.path().join("granted/visible"), "visible")
|
||||
.expect("visible fixture");
|
||||
std::fs::create_dir(temp.path().join("secret")).expect("secret directory");
|
||||
std::fs::write(temp.path().join("secret/key"), "hidden").expect("secret fixture");
|
||||
symlink("../secret/key", temp.path().join("granted/link")).expect("symlink fixture");
|
||||
}
|
||||
let scope = SharedScope::new(Scope::writable(temp.path()).expect("scope"));
|
||||
let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound(
|
||||
Workdir::new("wd-1"),
|
||||
@@ -2680,7 +2802,6 @@ mod tests {
|
||||
expires_at: u64::MAX,
|
||||
};
|
||||
let operation = WorkdirSessionOperationRequest {
|
||||
delegations: Vec::new(),
|
||||
operation: WorkdirSessionOperation::Stat(StatRequest {
|
||||
path: WorkdirPath::new("hello.txt").expect("logical path"),
|
||||
}),
|
||||
@@ -2697,7 +2818,6 @@ mod tests {
|
||||
assert!(matches!(result, WorkdirSessionOperationResult::Stat(_)));
|
||||
|
||||
let grep = WorkdirSessionOperationRequest {
|
||||
delegations: Vec::new(),
|
||||
operation: WorkdirSessionOperation::Grep(GrepRequest {
|
||||
pattern: "hello".into(),
|
||||
path: WorkdirPath::new("hello.txt").unwrap(),
|
||||
@@ -2720,78 +2840,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("grep direct file through provider operation");
|
||||
match result {
|
||||
WorkdirSessionOperationResult::Grep(result) => {
|
||||
assert_eq!(result.match_count, 1);
|
||||
assert_eq!(result.matched_files, 1);
|
||||
assert!(result.output.starts_with("hello.txt\n"));
|
||||
assert!(result.output.contains("> 1 │ hello"));
|
||||
}
|
||||
other => panic!("unexpected workdir grep result: {other:?}"),
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let delegated_visible = WorkdirSessionOperationRequest {
|
||||
delegations: vec![workdir::WorkdirDelegationRequest {
|
||||
rules: vec![workdir::WorkdirDelegationRule {
|
||||
target: WorkdirPath::new("granted").unwrap(),
|
||||
permission: workdir::WorkdirDelegationPermission::Read,
|
||||
recursive: true,
|
||||
}],
|
||||
cwd: WorkdirPath::new("granted").unwrap(),
|
||||
}],
|
||||
operation: WorkdirSessionOperation::Read(ReadRequest {
|
||||
path: WorkdirPath::new("visible").unwrap(),
|
||||
offset: 0,
|
||||
limit: 20,
|
||||
max_bytes: 1024,
|
||||
}),
|
||||
};
|
||||
let visible = run_workdir_session_operation(
|
||||
State(state.clone()),
|
||||
Path("session-1".to_string()),
|
||||
Some(Extension(auth.clone())),
|
||||
Ok(Json(delegated_visible)),
|
||||
)
|
||||
.await
|
||||
.expect("non-root delegated cwd should resolve once")
|
||||
.0;
|
||||
assert!(matches!(
|
||||
visible,
|
||||
WorkdirSessionOperationResult::Read(result) if result.bytes == b"visible"
|
||||
));
|
||||
|
||||
let delegated_read = WorkdirSessionOperationRequest {
|
||||
delegations: vec![workdir::WorkdirDelegationRequest {
|
||||
rules: vec![workdir::WorkdirDelegationRule {
|
||||
target: WorkdirPath::new("granted").unwrap(),
|
||||
permission: workdir::WorkdirDelegationPermission::Read,
|
||||
recursive: true,
|
||||
}],
|
||||
cwd: WorkdirPath::new("granted").unwrap(),
|
||||
}],
|
||||
operation: WorkdirSessionOperation::Read(ReadRequest {
|
||||
path: WorkdirPath::new("link").unwrap(),
|
||||
offset: 0,
|
||||
limit: 20,
|
||||
max_bytes: 1024,
|
||||
}),
|
||||
};
|
||||
let error = run_workdir_session_operation(
|
||||
State(state.clone()),
|
||||
Path("session-1".to_string()),
|
||||
Some(Extension(auth.clone())),
|
||||
Ok(Json(delegated_read)),
|
||||
)
|
||||
.await
|
||||
.expect_err("provider must reject delegated symlink escape");
|
||||
assert_ne!(error.status, StatusCode::OK);
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(temp.path().join("secret/key")).unwrap(),
|
||||
"hidden"
|
||||
);
|
||||
}
|
||||
assert!(matches!(result, WorkdirSessionOperationResult::Grep(_)));
|
||||
|
||||
let wrong_owner = RuntimeAuthContext {
|
||||
workspace_id: "workspace-b".to_string(),
|
||||
@@ -2870,11 +2919,13 @@ mod tests {
|
||||
_handle: &WorkerExecutionHandle,
|
||||
input: WorkerInput,
|
||||
) -> WorkerExecutionResult {
|
||||
if let Some(submission_id) = input.submission_id {
|
||||
WorkerExecutionResult::accepted_input_committed(
|
||||
if let Some(submission_id) = input.submission_request_id {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
submission_id.clone(),
|
||||
submission_id,
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
} else {
|
||||
WorkerExecutionResult::accepted(
|
||||
@@ -3194,11 +3245,13 @@ mod ws_tests {
|
||||
_handle: &WorkerExecutionHandle,
|
||||
input: WorkerInput,
|
||||
) -> WorkerExecutionResult {
|
||||
if let Some(submission_id) = input.submission_id {
|
||||
WorkerExecutionResult::accepted_input_committed(
|
||||
if let Some(submission_id) = input.submission_request_id {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
submission_id.clone(),
|
||||
submission_id,
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
} else {
|
||||
WorkerExecutionResult::accepted(
|
||||
|
||||
@@ -25,10 +25,10 @@ impl WorkerInputKind {
|
||||
pub struct WorkerInput {
|
||||
pub kind: WorkerInputKind,
|
||||
pub content: String,
|
||||
/// Runtime-generated correlation id. This is never accepted from public
|
||||
/// JSON input and is consumed only by the execution backend.
|
||||
#[serde(skip)]
|
||||
pub submission_id: Option<String>,
|
||||
/// Authenticated client-generated idempotency key. Runtime generates one
|
||||
/// only for trusted internal callers that omit it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub submission_request_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub segments: Option<Vec<Segment>>,
|
||||
}
|
||||
@@ -38,7 +38,7 @@ impl WorkerInput {
|
||||
Self {
|
||||
kind: WorkerInputKind::User,
|
||||
content: content.into(),
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: None,
|
||||
}
|
||||
}
|
||||
@@ -47,7 +47,7 @@ impl WorkerInput {
|
||||
Self {
|
||||
kind: WorkerInputKind::Notify,
|
||||
content: content.into(),
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: None,
|
||||
}
|
||||
}
|
||||
@@ -57,6 +57,21 @@ impl WorkerInput {
|
||||
mod tests {
|
||||
use super::WorkerInput;
|
||||
|
||||
#[test]
|
||||
fn submission_request_id_round_trips_for_authenticated_client_retry() {
|
||||
let input: WorkerInput = serde_json::from_value(serde_json::json!({
|
||||
"kind": "user",
|
||||
"content": "message",
|
||||
"submission_request_id": "request-1"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(input.submission_request_id.as_deref(), Some("request-1"));
|
||||
assert_eq!(
|
||||
serde_json::to_value(input).unwrap()["submission_request_id"],
|
||||
"request-1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notify_is_an_operation_and_legacy_system_kind_is_rejected() {
|
||||
assert_eq!(
|
||||
@@ -78,4 +93,7 @@ mod tests {
|
||||
pub struct WorkerInteractionAck {
|
||||
pub worker_ref: WorkerRef,
|
||||
pub status: WorkerStatus,
|
||||
/// Present for User Submit and absent for non-Submit interactions.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub submission: Option<crate::execution::WorkerSubmissionAck>,
|
||||
}
|
||||
|
||||
@@ -748,8 +748,12 @@ impl Runtime {
|
||||
let state = self.lock()?;
|
||||
state.worker(&worker_ref)?.request.initial_input.clone()
|
||||
} {
|
||||
let expected_submission_id = Uuid::now_v7().to_string();
|
||||
initial_input.submission_id = Some(expected_submission_id.clone());
|
||||
let expected_submission_id = initial_input
|
||||
.submission_request_id
|
||||
.clone()
|
||||
.filter(|request_id| !request_id.trim().is_empty())
|
||||
.unwrap_or_else(|| Uuid::now_v7().to_string());
|
||||
initial_input.submission_request_id = Some(expected_submission_id.clone());
|
||||
let dispatch_result = backend.dispatch_input(&handle, initial_input.clone());
|
||||
if !dispatch_result.is_accepted() {
|
||||
let _ = backend.stop_worker(&handle);
|
||||
@@ -763,9 +767,9 @@ impl Runtime {
|
||||
});
|
||||
}
|
||||
let has_commit_ack = dispatch_result
|
||||
.input_commit
|
||||
.submission
|
||||
.as_ref()
|
||||
.is_some_and(|ack| ack.submission_id == expected_submission_id);
|
||||
.is_some_and(|ack| ack.submission_request_id == expected_submission_id);
|
||||
if !has_commit_ack {
|
||||
let _ = backend.stop_worker(&handle);
|
||||
self.rollback_failed_create(&worker_ref)?;
|
||||
@@ -1146,9 +1150,14 @@ impl Runtime {
|
||||
mut input: WorkerInput,
|
||||
) -> Result<WorkerInteractionAck, RuntimeError> {
|
||||
validate_worker_input(&input)?;
|
||||
let expected_submission_id = if input.kind == WorkerInputKind::User {
|
||||
let submission_id = Uuid::now_v7().to_string();
|
||||
input.submission_id = Some(submission_id.clone());
|
||||
let expected_submission_id =
|
||||
if matches!(input.kind, WorkerInputKind::User | WorkerInputKind::Notify) {
|
||||
let submission_id = input
|
||||
.submission_request_id
|
||||
.clone()
|
||||
.filter(|request_id| !request_id.trim().is_empty())
|
||||
.unwrap_or_else(|| Uuid::now_v7().to_string());
|
||||
input.submission_request_id = Some(submission_id.clone());
|
||||
Some(submission_id)
|
||||
} else {
|
||||
None
|
||||
@@ -1191,13 +1200,13 @@ impl Runtime {
|
||||
}
|
||||
if let Some(expected_submission_id) = expected_submission_id
|
||||
&& dispatch_result
|
||||
.input_commit
|
||||
.submission
|
||||
.as_ref()
|
||||
.is_none_or(|ack| ack.submission_id != expected_submission_id)
|
||||
.is_none_or(|ack| ack.submission_request_id != expected_submission_id)
|
||||
{
|
||||
let result = WorkerExecutionResult::rejected(
|
||||
WorkerExecutionOperation::Input,
|
||||
"execution backend did not acknowledge the committed Runtime submission id",
|
||||
"execution backend did not acknowledge the committed Runtime submission request id",
|
||||
);
|
||||
self.record_execution_result(worker_ref, result.clone())?;
|
||||
return Err(RuntimeError::WorkerExecutionRejected {
|
||||
@@ -1209,6 +1218,7 @@ impl Runtime {
|
||||
});
|
||||
}
|
||||
|
||||
let submission = dispatch_result.submission.clone();
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
let worker = state.worker_mut(worker_ref)?;
|
||||
@@ -1225,6 +1235,7 @@ impl Runtime {
|
||||
Ok(WorkerInteractionAck {
|
||||
worker_ref: worker_ref.clone(),
|
||||
status,
|
||||
submission,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1706,6 +1717,7 @@ impl Runtime {
|
||||
}
|
||||
Ok(protocol::Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
@@ -3250,17 +3262,9 @@ fn validate_worker_input(input: &WorkerInput) -> Result<(), RuntimeError> {
|
||||
#[cfg(feature = "ws-server")]
|
||||
fn input_protocol_event(input: &WorkerInput) -> Option<protocol::Event> {
|
||||
match input.kind {
|
||||
WorkerInputKind::User => Some(protocol::Event::UserMessage {
|
||||
segments: input.segments.clone().unwrap_or_else(|| {
|
||||
vec![protocol::Segment::Text {
|
||||
content: input.content.clone(),
|
||||
}]
|
||||
}),
|
||||
}),
|
||||
// The committed `SystemItem::Notification` is the sole agent-visible
|
||||
// and Console-visible authority for Notify. A synthetic observation
|
||||
// here would display the same notification twice.
|
||||
WorkerInputKind::Notify => None,
|
||||
// Submit is projected only after the Worker commits UserInput. Queued
|
||||
// payloads must never become model- or client-visible history early.
|
||||
WorkerInputKind::User | WorkerInputKind::Notify => None,
|
||||
WorkerInputKind::Compact
|
||||
| WorkerInputKind::ListRewindTargets
|
||||
| WorkerInputKind::RegisterPeer => Some(protocol::Event::SystemItem {
|
||||
@@ -3435,6 +3439,7 @@ mod tests {
|
||||
);
|
||||
let snapshot = protocol::Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
@@ -3475,7 +3480,7 @@ mod tests {
|
||||
let input = WorkerInput {
|
||||
kind: WorkerInputKind::User,
|
||||
content: String::new(),
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: Some(vec![protocol::Segment::Flow {
|
||||
selector: "builtin:coder-review".to_string(),
|
||||
}]),
|
||||
@@ -3488,7 +3493,7 @@ mod tests {
|
||||
let input = WorkerInput {
|
||||
kind: WorkerInputKind::User,
|
||||
content: String::new(),
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: Some(Vec::new()),
|
||||
};
|
||||
assert!(matches!(
|
||||
@@ -3503,7 +3508,7 @@ mod tests {
|
||||
request.initial_input = Some(WorkerInput {
|
||||
kind: WorkerInputKind::User,
|
||||
content: String::new(),
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: Some(vec![protocol::Segment::Flow {
|
||||
selector: "builtin:coder-review".to_string(),
|
||||
}]),
|
||||
@@ -4005,7 +4010,7 @@ mod tests {
|
||||
_handle: &WorkerExecutionHandle,
|
||||
input: WorkerInput,
|
||||
) -> WorkerExecutionResult {
|
||||
let submission_id = input.submission_id.clone();
|
||||
let submission_id = input.submission_request_id.clone();
|
||||
self.dispatched_inputs.lock().unwrap().push(input);
|
||||
let mut result = self
|
||||
.dispatch_result
|
||||
@@ -4013,19 +4018,21 @@ mod tests {
|
||||
.unwrap()
|
||||
.clone()
|
||||
.unwrap_or_else(|| {
|
||||
WorkerExecutionResult::accepted_input_committed(
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
"request-test",
|
||||
"test-submission",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
});
|
||||
if !self
|
||||
.preserve_commit_ack_submission_id
|
||||
.load(Ordering::SeqCst)
|
||||
&& let (Some(ack), Some(submission_id)) =
|
||||
(result.input_commit.as_mut(), submission_id)
|
||||
(result.submission.as_mut(), submission_id)
|
||||
{
|
||||
ack.submission_id = submission_id;
|
||||
ack.submission_request_id = submission_id;
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -4717,10 +4724,12 @@ mod tests {
|
||||
#[test]
|
||||
fn create_worker_uses_committed_input_ack_run_state() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed(
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
"request-test",
|
||||
"test-submission",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
));
|
||||
let mut request = task_request("committed initial input is already idle");
|
||||
request.initial_input = Some(WorkerInput::user("start the ticket"));
|
||||
@@ -4731,13 +4740,15 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_worker_rejects_mismatched_input_commit_acknowledgement() {
|
||||
fn create_worker_rejects_mismatched_submission_acknowledgement() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
backend.preserve_commit_ack_submission_id();
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed(
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
"request-test",
|
||||
"forged-submission",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
));
|
||||
let mut request = task_request("mismatched initial input commit ack");
|
||||
request.initial_input = Some(WorkerInput::user("start the ticket"));
|
||||
@@ -4866,6 +4877,7 @@ mod tests {
|
||||
&detail.worker_ref,
|
||||
protocol::Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![protocol::SessionSnapshotEntry {
|
||||
entry_id: "restored-log-entry".to_owned(),
|
||||
timestamp: 1,
|
||||
@@ -4937,10 +4949,14 @@ mod tests {
|
||||
_handle: &WorkerExecutionHandle,
|
||||
input: WorkerInput,
|
||||
) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted_input_committed(
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
input.submission_id.expect("Runtime submission id"),
|
||||
"request-test",
|
||||
input
|
||||
.submission_request_id
|
||||
.expect("Runtime submission request id"),
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -5030,7 +5046,7 @@ mod tests {
|
||||
request.initial_input = Some(WorkerInput {
|
||||
kind: WorkerInputKind::User,
|
||||
content: String::new(),
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: Some(vec![
|
||||
protocol::Segment::Flow {
|
||||
selector: "builtin:coder-review".to_string(),
|
||||
@@ -5068,7 +5084,7 @@ mod tests {
|
||||
let input = WorkerInput {
|
||||
kind: WorkerInputKind::User,
|
||||
content: String::new(),
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: Some(vec![protocol::Segment::Flow {
|
||||
selector: "builtin:coder-review".to_string(),
|
||||
}]),
|
||||
@@ -5083,11 +5099,11 @@ mod tests {
|
||||
assert_eq!(dispatched[0].kind, input.kind);
|
||||
assert_eq!(dispatched[0].content, input.content);
|
||||
assert_eq!(dispatched[0].segments, input.segments);
|
||||
let submission_id = dispatched[0]
|
||||
.submission_id
|
||||
let submission_request_id = dispatched[0]
|
||||
.submission_request_id
|
||||
.as_deref()
|
||||
.expect("Runtime submission id");
|
||||
Uuid::parse_str(submission_id).expect("submission id UUID");
|
||||
.expect("Runtime submission request id");
|
||||
Uuid::parse_str(submission_request_id).expect("submission request id UUID");
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
@@ -5113,11 +5129,7 @@ mod tests {
|
||||
let observations = runtime
|
||||
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
|
||||
.unwrap();
|
||||
assert_eq!(observations.len(), 1);
|
||||
assert!(matches!(
|
||||
observations[0].payload,
|
||||
protocol::Event::UserMessage { .. }
|
||||
));
|
||||
assert!(observations.is_empty());
|
||||
|
||||
runtime
|
||||
.observe_worker_event(
|
||||
@@ -5135,8 +5147,8 @@ mod tests {
|
||||
let observations = runtime
|
||||
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
|
||||
.unwrap();
|
||||
assert_eq!(observations.len(), 2);
|
||||
let protocol::Event::SystemItem { item } = &observations[1].payload else {
|
||||
assert_eq!(observations.len(), 1);
|
||||
let protocol::Event::SystemItem { item } = &observations[0].payload else {
|
||||
panic!("committed notification observation must be a system item");
|
||||
};
|
||||
assert_eq!(item["kind"], "notification");
|
||||
|
||||
@@ -39,7 +39,7 @@ use crate::working_directory::{
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus};
|
||||
use session_store::{CombinedStore, LogEntry, WorkerAggregateStore, WorkerSessionStore};
|
||||
use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore};
|
||||
#[cfg(test)]
|
||||
use session_store::{FsStore, FsWorkerStore};
|
||||
use tokio::runtime::Runtime;
|
||||
@@ -57,11 +57,10 @@ use worker::feature::builtin::{
|
||||
#[cfg(feature = "ws-server")]
|
||||
use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session};
|
||||
use worker::{
|
||||
PreparedWorker, PromptCatalogSource, SegmentLogSink, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN,
|
||||
Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout,
|
||||
WorkerControllerTransport, WorkerError, WorkerFilesystemAuthority, WorkerHandle,
|
||||
WorkerSharedState, WorkerWorkspaceContext, WorkspaceClient, WorkspaceId,
|
||||
bash_output_dir_for_worker_id,
|
||||
PreparedWorker, PromptCatalogSource, SegmentLogSink, Worker, WorkerBootstrap,
|
||||
WorkerBootstrapError, WorkerBootstrapLayout, WorkerControllerTransport, WorkerError,
|
||||
WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, WorkerWorkspaceContext,
|
||||
WorkspaceClient, WorkspaceId, bash_output_dir_for_worker_id,
|
||||
};
|
||||
|
||||
const DEFAULT_BACKEND_ID: &str = "worker-crate";
|
||||
@@ -70,17 +69,6 @@ const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
// returns a typed execution error instead of leaving the outer waiter to time out.
|
||||
const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9);
|
||||
|
||||
fn user_input_has_submission(entry: &LogEntry, submission_id: &str) -> bool {
|
||||
let extensions = match entry {
|
||||
LogEntry::AnnotatedUserInput { extensions, .. } => extensions,
|
||||
_ => return false,
|
||||
};
|
||||
extensions.iter().any(|extension| {
|
||||
extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN
|
||||
&& extension.payload["submission_id"].as_str() == Some(submission_id)
|
||||
})
|
||||
}
|
||||
|
||||
pub struct RuntimeWorkerController {
|
||||
pub handle: WorkerHandle,
|
||||
pub shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>,
|
||||
@@ -1342,126 +1330,75 @@ where
|
||||
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
|
||||
}
|
||||
|
||||
fn send_user_input_and_wait_for_commit(
|
||||
fn send_submit_and_wait_for_acceptance(
|
||||
&self,
|
||||
operation: WorkerExecutionOperation,
|
||||
worker: WorkerHandle,
|
||||
method: Method,
|
||||
submission_id: String,
|
||||
submission_request_id: String,
|
||||
accepted_run_state: WorkerExecutionRunState,
|
||||
) -> WorkerExecutionResult {
|
||||
let acknowledged_submission_id = submission_id.clone();
|
||||
let request_id = submission_request_id.clone();
|
||||
self.run_on_adapter_runtime(async move {
|
||||
// Subscribe before enqueueing the input so the acknowledgement cannot
|
||||
// race with a fast Worker commit. The opaque submission id is stored in
|
||||
// the same UserInput entry as the transformed Flow input and its state.
|
||||
let (_, mut committed_entries) = worker.sink.subscribe_with_snapshot();
|
||||
let committed_probe = worker.clone();
|
||||
// Subscribe before enqueueing so a fast durable acceptance cannot
|
||||
// race the Runtime acknowledgement.
|
||||
let mut events = worker.subscribe();
|
||||
worker
|
||||
.send(method)
|
||||
.await
|
||||
.map_err(|err| format!("failed to send Worker method: {err}"))?;
|
||||
|
||||
let timeout_probe = committed_probe.clone();
|
||||
let timeout_submission_id = submission_id.clone();
|
||||
let acknowledgement = tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move {
|
||||
let input_was_committed = || {
|
||||
committed_probe
|
||||
.committed_entries()
|
||||
.iter()
|
||||
.any(|entry| user_input_has_submission(entry, &submission_id))
|
||||
};
|
||||
tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
entry = committed_entries.recv() => {
|
||||
match entry {
|
||||
Ok(entry) if user_input_has_submission(&entry, &submission_id) => {
|
||||
return Ok(());
|
||||
match events.recv().await {
|
||||
Ok(Event::SubmissionAccepted {
|
||||
submission_request_id,
|
||||
submission_id,
|
||||
disposition,
|
||||
}) if submission_request_id == request_id => {
|
||||
return Ok((submission_id, disposition));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
|
||||
if input_was_committed() {
|
||||
return Ok(());
|
||||
Ok(Event::SubmissionRejected {
|
||||
submission_request_id,
|
||||
message,
|
||||
}) if submission_request_id == request_id => {
|
||||
return Err(format!("worker rejected Submit: {message}"));
|
||||
}
|
||||
return Err(format!(
|
||||
"worker input commit acknowledgement lagged by {skipped} entry event(s)"
|
||||
));
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
if input_was_committed() {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(
|
||||
"worker entry stream closed before user input was committed"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
event = events.recv() => {
|
||||
match event {
|
||||
Ok(Event::Error { message, .. }) => {
|
||||
if input_was_committed() {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(format!(
|
||||
"worker rejected user input before session commit: {message}"
|
||||
"worker rejected Submit before durable acceptance: {message}"
|
||||
));
|
||||
}
|
||||
Ok(Event::Shutdown) => {
|
||||
if input_was_committed() {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(
|
||||
"worker shut down before user input was committed".to_string()
|
||||
"worker shut down before Submit was durably accepted".to_string()
|
||||
);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => {
|
||||
if input_was_committed() {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(format!(
|
||||
"worker input commit acknowledgement lagged by {skipped} protocol event(s)"
|
||||
"worker Submit acknowledgement lagged by {skipped} protocol event(s)"
|
||||
));
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
|
||||
if input_was_committed() {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(
|
||||
"worker event stream closed before user input was committed"
|
||||
"worker event stream closed before Submit was durably accepted"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
match acknowledgement {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
if timeout_probe
|
||||
.committed_entries()
|
||||
.iter()
|
||||
.any(|entry| user_input_has_submission(entry, &timeout_submission_id))
|
||||
{
|
||||
Ok(())
|
||||
} else {
|
||||
Err("timed out waiting for worker user input commit".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
.await
|
||||
.map_err(|_| "timed out waiting for durable Worker Submit acceptance".to_string())?
|
||||
})
|
||||
.map(|_| {
|
||||
WorkerExecutionResult::accepted_input_committed(
|
||||
.map(|(submission_id, disposition)| {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
operation,
|
||||
accepted_run_state,
|
||||
acknowledged_submission_id,
|
||||
submission_request_id,
|
||||
submission_id,
|
||||
disposition,
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
|
||||
@@ -1582,9 +1519,10 @@ impl<F> Drop for WorkerRuntimeExecutionBackend<F> {
|
||||
fn method_starts_turn(method: &Method) -> bool {
|
||||
matches!(
|
||||
method,
|
||||
Method::Run { .. }
|
||||
| Method::RunTracked { .. }
|
||||
Method::Submit { .. }
|
||||
| Method::SubmitTracked { .. }
|
||||
| Method::Notify { auto_run: true, .. }
|
||||
| Method::NotifyTracked { auto_run: true, .. }
|
||||
| Method::Resume
|
||||
| Method::Compact
|
||||
)
|
||||
@@ -1609,9 +1547,10 @@ fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExec
|
||||
|
||||
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
|
||||
match method {
|
||||
Method::Run { .. }
|
||||
| Method::RunTracked { .. }
|
||||
Method::Submit { .. }
|
||||
| Method::SubmitTracked { .. }
|
||||
| Method::Notify { auto_run: true, .. }
|
||||
| Method::NotifyTracked { auto_run: true, .. }
|
||||
| Method::Resume
|
||||
| Method::Compact => WorkerExecutionRunState::Busy,
|
||||
Method::Shutdown => WorkerExecutionRunState::Stopped,
|
||||
@@ -1959,12 +1898,19 @@ where
|
||||
&& busy
|
||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||
.is_ok();
|
||||
let notification_request_id = input
|
||||
.submission_request_id
|
||||
.unwrap_or_else(protocol::new_submission_request_id);
|
||||
let result = self.send_method(
|
||||
WorkerExecutionOperation::Input,
|
||||
worker,
|
||||
Method::Notify {
|
||||
Method::NotifyTracked {
|
||||
notification_request_id: notification_request_id.clone(),
|
||||
message: input.content,
|
||||
auto_run: true,
|
||||
source: protocol::AuthenticatedInputSource::Backend {
|
||||
operation_id: notification_request_id,
|
||||
},
|
||||
},
|
||||
accepted_run_state,
|
||||
);
|
||||
@@ -1975,21 +1921,23 @@ where
|
||||
return result;
|
||||
}
|
||||
|
||||
if worker.shared_state.get_status() != WorkerStatus::Idle
|
||||
|| busy
|
||||
let is_user_submit = input.kind == WorkerInputKind::User;
|
||||
let status = worker.shared_state.get_status();
|
||||
let claimed_here = status == WorkerStatus::Idle
|
||||
&& busy
|
||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||
.is_err()
|
||||
{
|
||||
.is_ok();
|
||||
if !is_user_submit && !claimed_here {
|
||||
return WorkerExecutionResult::busy(
|
||||
WorkerExecutionOperation::Input,
|
||||
"Worker is already running; runtime adapter v0 does not queue input",
|
||||
"Worker is already running",
|
||||
);
|
||||
}
|
||||
|
||||
let (method, submission_id) = match input.kind {
|
||||
let (method, submission_request_id) = match input.kind {
|
||||
WorkerInputKind::User => {
|
||||
let Some(submission_id) = input
|
||||
.submission_id
|
||||
.submission_request_id
|
||||
.filter(|submission_id| !submission_id.trim().is_empty())
|
||||
else {
|
||||
busy.store(false, Ordering::SeqCst);
|
||||
@@ -1999,11 +1947,14 @@ where
|
||||
);
|
||||
};
|
||||
(
|
||||
Method::RunTracked {
|
||||
Method::SubmitTracked {
|
||||
submission_request_id: submission_id.clone(),
|
||||
input: input.segments.unwrap_or_else(|| {
|
||||
vec![Segment::text(input.content.trim().to_string())]
|
||||
}),
|
||||
submission_id: submission_id.clone(),
|
||||
source: protocol::AuthenticatedInputSource::Backend {
|
||||
operation_id: submission_id.clone(),
|
||||
},
|
||||
},
|
||||
Some(submission_id),
|
||||
)
|
||||
@@ -2021,21 +1972,22 @@ where
|
||||
),
|
||||
};
|
||||
let accepted_run_state = match method {
|
||||
Method::Run { .. }
|
||||
| Method::RunTracked { .. }
|
||||
Method::Submit { .. }
|
||||
| Method::SubmitTracked { .. }
|
||||
| Method::Notify { .. }
|
||||
| Method::NotifyTracked { .. }
|
||||
| Method::Compact => WorkerExecutionRunState::Busy,
|
||||
_ => WorkerExecutionRunState::Idle,
|
||||
};
|
||||
let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle;
|
||||
let waits_for_user_input_commit = submission_id.is_some();
|
||||
let waits_for_submission_acceptance = submission_request_id.is_some();
|
||||
|
||||
let result = if waits_for_user_input_commit {
|
||||
self.send_user_input_and_wait_for_commit(
|
||||
let result = if waits_for_submission_acceptance {
|
||||
self.send_submit_and_wait_for_acceptance(
|
||||
WorkerExecutionOperation::Input,
|
||||
worker,
|
||||
method,
|
||||
submission_id.expect("tracked Run has submission id"),
|
||||
submission_request_id.expect("Submit must have a submission request id"),
|
||||
accepted_run_state,
|
||||
)
|
||||
} else {
|
||||
@@ -2046,7 +1998,9 @@ where
|
||||
accepted_run_state,
|
||||
)
|
||||
};
|
||||
if accepted_is_idle || result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
|
||||
if accepted_is_idle
|
||||
|| (claimed_here
|
||||
&& result.outcome != crate::execution::WorkerExecutionOutcome::Accepted)
|
||||
{
|
||||
busy.store(false, Ordering::SeqCst);
|
||||
}
|
||||
@@ -2116,8 +2070,12 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
if let Method::Notify { auto_run, .. } = &method {
|
||||
let auto_run = *auto_run;
|
||||
if let Some(auto_run) = match &method {
|
||||
Method::Notify { auto_run, .. } | Method::NotifyTracked { auto_run, .. } => {
|
||||
Some(*auto_run)
|
||||
}
|
||||
_ => None,
|
||||
} {
|
||||
let status = worker.shared_state.get_status();
|
||||
let accepted_run_state = accepted_notify_run_state(status, auto_run);
|
||||
let claimed_here = status == WorkerStatus::Idle
|
||||
@@ -3400,6 +3358,60 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn running_worker_accepts_a_second_submit_as_queued() {
|
||||
let client = MockClient::sequential(vec![MockResponse::Hang(vec![])]);
|
||||
let runtime_base = tempfile::tempdir().unwrap();
|
||||
let cwd = tempfile::tempdir().unwrap();
|
||||
let store = tempfile::tempdir().unwrap();
|
||||
let factory = MockFactory {
|
||||
client,
|
||||
runtime_base: runtime_base.path().to_path_buf(),
|
||||
cwd: cwd.path().to_path_buf(),
|
||||
store_dir: store.path().join("sessions"),
|
||||
worker_metadata_dir: store.path().join("workers"),
|
||||
observed_cwds: Arc::new(Mutex::new(Vec::new())),
|
||||
observed_workspace_clients: Arc::new(Mutex::new(Vec::new())),
|
||||
};
|
||||
let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
|
||||
let runtime =
|
||||
EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend).unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let detail = runtime
|
||||
.create_worker(create_request("queued-submit"))
|
||||
.unwrap();
|
||||
|
||||
let mut first_input = WorkerInput::user("first");
|
||||
first_input.submission_request_id = Some("request-first".into());
|
||||
let first = runtime
|
||||
.send_input(&detail.worker_ref, first_input.clone())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
first.submission.as_ref().map(|ack| ack.disposition),
|
||||
Some(protocol::SubmissionDisposition::Started)
|
||||
);
|
||||
let retry = runtime.send_input(&detail.worker_ref, first_input).unwrap();
|
||||
assert_eq!(retry.submission, first.submission);
|
||||
let mut conflicting_retry = WorkerInput::user("different");
|
||||
conflicting_retry.submission_request_id = Some("request-first".into());
|
||||
assert!(
|
||||
runtime
|
||||
.send_input(&detail.worker_ref, conflicting_retry)
|
||||
.is_err(),
|
||||
"same request id with a different payload must fail"
|
||||
);
|
||||
|
||||
let mut second_input = WorkerInput::user("second");
|
||||
second_input.submission_request_id = Some("request-second".into());
|
||||
let second = runtime
|
||||
.send_input(&detail.worker_ref, second_input)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
second.submission.as_ref().map(|ack| ack.disposition),
|
||||
Some(protocol::SubmissionDisposition::Queued)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_with_initial_input_returns_after_session_commit() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
@@ -3449,8 +3461,10 @@ mod tests {
|
||||
};
|
||||
extensions
|
||||
.iter()
|
||||
.find(|extension| extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN)
|
||||
.and_then(|extension| extension.payload["submission_id"].as_str())
|
||||
.find(|extension| extension.domain == "worker.pending_activations.v1")
|
||||
.and_then(|extension| {
|
||||
extension.payload["receipts"][0]["submission_id"].as_str()
|
||||
})
|
||||
})
|
||||
.expect("committed input submission id");
|
||||
uuid::Uuid::parse_str(submission_id).expect("opaque submission id is a UUID");
|
||||
|
||||
@@ -20,6 +20,7 @@ protocol = { workspace = true, features = ["json-schema"] }
|
||||
client = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
reqwest = { version = "0.13", default-features = false, features = ["blocking", "native-tls"] }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] }
|
||||
|
||||
@@ -101,7 +101,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
|
||||
// Send a run method
|
||||
handle
|
||||
.send(Method::run_text("What is the capital of France?"))
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"What is the capital of France?",
|
||||
))
|
||||
.await?;
|
||||
|
||||
// Wait for completion
|
||||
|
||||
+843
-165
File diff suppressed because it is too large
Load Diff
@@ -1012,7 +1012,19 @@ async fn send_peer_notify(socket_path: &Path, message: String) -> io::Result<()>
|
||||
}
|
||||
|
||||
async fn send_notify(socket_path: &Path, message: String, auto_run: bool) -> io::Result<()> {
|
||||
connect_and_send(socket_path, &Method::Notify { message, auto_run }).await
|
||||
let notification_request_id = protocol::new_submission_request_id();
|
||||
connect_and_send(
|
||||
socket_path,
|
||||
&Method::NotifyTracked {
|
||||
notification_request_id: notification_request_id.clone(),
|
||||
message,
|
||||
auto_run,
|
||||
source: protocol::AuthenticatedInputSource::Backend {
|
||||
operation_id: notification_request_id,
|
||||
},
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn json_content<T: Serialize>(value: &T) -> Result<String, ToolError> {
|
||||
@@ -1482,6 +1494,7 @@ mod tests {
|
||||
writer
|
||||
.write(&Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
@@ -1517,6 +1530,7 @@ mod tests {
|
||||
writer
|
||||
.write(&Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
@@ -1536,7 +1550,10 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
let method = reader.next::<Method>().await.unwrap().unwrap();
|
||||
if let Method::Notify { message, auto_run } = method {
|
||||
if let Method::NotifyTracked {
|
||||
message, auto_run, ..
|
||||
} = method
|
||||
{
|
||||
assert!(auto_run);
|
||||
tx.send(message).await.unwrap();
|
||||
} else {
|
||||
@@ -1608,6 +1625,7 @@ mod tests {
|
||||
writer
|
||||
.write(&Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
@@ -1634,6 +1652,7 @@ mod tests {
|
||||
writer
|
||||
.write(&Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
@@ -1653,7 +1672,10 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
let method = reader.next::<Method>().await.unwrap().unwrap();
|
||||
if let Method::Notify { message, auto_run } = method {
|
||||
if let Method::NotifyTracked {
|
||||
message, auto_run, ..
|
||||
} = method
|
||||
{
|
||||
assert!(!auto_run);
|
||||
tx.send(message).await.unwrap();
|
||||
} else {
|
||||
@@ -1738,6 +1760,7 @@ mod tests {
|
||||
writer
|
||||
.write(&Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
@@ -1790,6 +1813,8 @@ mod tests {
|
||||
let _ = writer
|
||||
.write(&Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(
|
||||
),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
//! endpoints, credentials, materializer handles, and operation sessions stay
|
||||
//! behind [`WorkspaceClient`].
|
||||
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
use agen::tool::{Tool, ToolDefinition, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
|
||||
@@ -12,7 +14,7 @@ use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use workdir::http::{WorkdirSessionOperation, WorkdirSessionOperationResult};
|
||||
use workdir::workspace::{WorkspaceWorkdirSessionFence, WorkspaceWorkdirSessionOperationRequest};
|
||||
use workdir::workspace::WorkspaceWorkdirSessionOperationRequest;
|
||||
use workdir::{
|
||||
CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest, CommandStatus, EditRequest,
|
||||
EditResult, GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, ListResult,
|
||||
@@ -52,16 +54,48 @@ const LIST_DESCRIPTION: &str = "List persistent Workdirs in the current Workspac
|
||||
const CREATE_DESCRIPTION: &str = "Materialize a persistent Workdir on a selected Runtime from a Workspace repository and optional selector. This does not change this Worker's attachment; use WorkdirAttach explicitly after creation.";
|
||||
const ATTACH_DESCRIPTION: &str = "Attach this Worker to one existing Workdir. The Backend enforces one active Workdir per Worker and one active Worker per Workdir, then opens an ephemeral operation session.";
|
||||
const DETACH_DESCRIPTION: &str = "Detach this Worker from its active Workdir and release Workdir occupancy. Any ephemeral operation session is closed.";
|
||||
pub(crate) type BeforeWorkdirRelease =
|
||||
Arc<dyn Fn() -> Pin<Box<dyn Future<Output = std::io::Result<()>> + Send>> + Send + Sync>;
|
||||
pub(crate) type AfterWorkdirAttach = Arc<dyn Fn() + Send + Sync>;
|
||||
|
||||
const DELETE_DESCRIPTION: &str = "Request removal of one persistent Workdir by id through durable Backend Workspace authority. The input includes only the Workdir id and a bounded reason. The result reports removed, retained, or attention_required without exposing operation-table or provider internals.";
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
#[derive(Clone)]
|
||||
pub struct ManageWorkdirFeature {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
before_workdir_release: Option<BeforeWorkdirRelease>,
|
||||
after_workdir_attach: Option<AfterWorkdirAttach>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ManageWorkdirFeature {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ManageWorkdirFeature")
|
||||
.field("client_kind", &self.client.kind())
|
||||
.field("release_guard", &self.before_workdir_release.is_some())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ManageWorkdirFeature {
|
||||
pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
|
||||
Self { client }
|
||||
Self {
|
||||
client,
|
||||
before_workdir_release: None,
|
||||
after_workdir_attach: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_child_lifecycle(
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
before_workdir_release: BeforeWorkdirRelease,
|
||||
after_workdir_attach: AfterWorkdirAttach,
|
||||
) -> Self {
|
||||
Self {
|
||||
client,
|
||||
before_workdir_release: Some(before_workdir_release),
|
||||
after_workdir_attach: Some(after_workdir_attach),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,7 +115,10 @@ impl FeatureModule for ManageWorkdirFeature {
|
||||
}
|
||||
|
||||
fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> {
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(self.client.clone());
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(self.client.clone()).with_child_lifecycle(
|
||||
self.before_workdir_release.clone(),
|
||||
self.after_workdir_attach.clone(),
|
||||
);
|
||||
for (name, definition) in [
|
||||
(
|
||||
LIST_TOOL,
|
||||
@@ -142,9 +179,21 @@ impl FeatureModule for ManageWorkdirFeature {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
#[derive(Clone)]
|
||||
struct WorkspaceHttpWorkdirBackend {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
before_workdir_release: Option<BeforeWorkdirRelease>,
|
||||
after_workdir_attach: Option<AfterWorkdirAttach>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for WorkspaceHttpWorkdirBackend {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("WorkspaceHttpWorkdirBackend")
|
||||
.field("client_kind", &self.client.kind())
|
||||
.field("release_guard", &self.before_workdir_release.is_some())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Worker-local Workdir handle whose operation authority remains in the Workspace Backend.
|
||||
@@ -156,8 +205,6 @@ struct WorkspaceHttpWorkdirBackend {
|
||||
pub struct WorkspaceAttachedWorkdirSession {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
workdir: Workdir,
|
||||
expected_session_fence: Option<String>,
|
||||
delegations: Vec<workdir::WorkdirDelegationRequest>,
|
||||
}
|
||||
|
||||
impl WorkspaceAttachedWorkdirSession {
|
||||
@@ -165,8 +212,6 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
Arc::new(Self {
|
||||
client,
|
||||
workdir: Workdir::new("workspace-attachment"),
|
||||
expected_session_fence: None,
|
||||
delegations: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -183,16 +228,13 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
"/api/w/{}/workers/self/workdir-session/operations",
|
||||
encode_path_segment(workspace_id)
|
||||
),
|
||||
serde_json::to_string(&WorkspaceWorkdirSessionOperationRequest {
|
||||
expected_session_fence: self.expected_session_fence.clone(),
|
||||
delegations: self.delegations.clone(),
|
||||
operation,
|
||||
})
|
||||
.map_err(|error| {
|
||||
serde_json::to_string(&WorkspaceWorkdirSessionOperationRequest { operation }).map_err(
|
||||
|error| {
|
||||
WorkdirError::Transport(format!(
|
||||
"failed to encode Workspace Workdir operation: {error}"
|
||||
))
|
||||
})?,
|
||||
},
|
||||
)?,
|
||||
);
|
||||
let response = self
|
||||
.client
|
||||
@@ -241,59 +283,6 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession {
|
||||
WorkdirSessionCapabilities::ALL
|
||||
}
|
||||
|
||||
fn transports_delegation_context(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn capture_delegation_source(
|
||||
&self,
|
||||
request: &workdir::WorkdirDelegationRequest,
|
||||
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||
let expected_session_fence = if let Some(fence) = &self.expected_session_fence {
|
||||
fence.clone()
|
||||
} else {
|
||||
let workspace_id = self.client.workspace_id().ok_or_else(|| {
|
||||
WorkdirError::Unavailable("Workspace identity is unavailable".to_string())
|
||||
})?;
|
||||
let response = self
|
||||
.client
|
||||
.execute(WorkspaceRequest {
|
||||
method: WorkspaceRequestMethod::Get,
|
||||
path: format!(
|
||||
"/api/w/{}/workers/self/workdir-session/fence",
|
||||
encode_path_segment(workspace_id)
|
||||
),
|
||||
body: None,
|
||||
})
|
||||
.map_err(|error| {
|
||||
WorkdirError::Unavailable(format!(
|
||||
"failed to capture Workdir attachment fence: {error}"
|
||||
))
|
||||
})?;
|
||||
let fence: WorkspaceWorkdirSessionFence = serde_json::from_str(&response.body)
|
||||
.map_err(|error| {
|
||||
WorkdirError::Unavailable(format!(
|
||||
"invalid Workdir attachment fence response: {error}"
|
||||
))
|
||||
})?;
|
||||
fence.value
|
||||
};
|
||||
let mut delegations = self.delegations.clone();
|
||||
delegations.push(request.clone());
|
||||
let candidate = Arc::new(Self {
|
||||
client: self.client.clone(),
|
||||
workdir: self.workdir.clone(),
|
||||
expected_session_fence: Some(expected_session_fence),
|
||||
delegations,
|
||||
});
|
||||
candidate
|
||||
.stat(StatRequest {
|
||||
path: workdir::WorkdirPath::new("").expect("empty Workdir path is valid"),
|
||||
})
|
||||
.await?;
|
||||
Ok(candidate)
|
||||
}
|
||||
|
||||
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError> {
|
||||
match self.operate(WorkdirSessionOperation::Stat(request))? {
|
||||
WorkdirSessionOperationResult::Stat(result) => Ok(result),
|
||||
@@ -387,7 +376,21 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession {
|
||||
|
||||
impl WorkspaceHttpWorkdirBackend {
|
||||
fn new(client: Arc<dyn WorkspaceClient>) -> Self {
|
||||
Self { client }
|
||||
Self {
|
||||
client,
|
||||
before_workdir_release: None,
|
||||
after_workdir_attach: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_child_lifecycle(
|
||||
mut self,
|
||||
before_workdir_release: Option<BeforeWorkdirRelease>,
|
||||
after_workdir_attach: Option<AfterWorkdirAttach>,
|
||||
) -> Self {
|
||||
self.before_workdir_release = before_workdir_release;
|
||||
self.after_workdir_attach = after_workdir_attach;
|
||||
self
|
||||
}
|
||||
|
||||
fn workspace_id(&self) -> Result<&str, ToolError> {
|
||||
@@ -565,11 +568,26 @@ impl Tool for WorkspaceHttpWorkdirTool {
|
||||
parse_input::<WorkdirCreateInput>(input_json)?,
|
||||
ctx.call_id.to_string(),
|
||||
),
|
||||
WorkdirOperation::Attach => self
|
||||
WorkdirOperation::Attach => {
|
||||
let result = self
|
||||
.backend
|
||||
.attach(parse_input::<WorkdirAttachInput>(input_json)?),
|
||||
.attach(parse_input::<WorkdirAttachInput>(input_json)?);
|
||||
if result.is_ok()
|
||||
&& let Some(after_attach) = &self.backend.after_workdir_attach
|
||||
{
|
||||
after_attach();
|
||||
}
|
||||
result
|
||||
}
|
||||
WorkdirOperation::Detach => {
|
||||
let _input = parse_input::<WorkdirDetachInput>(input_json)?;
|
||||
if let Some(before_release) = &self.backend.before_workdir_release {
|
||||
before_release().await.map_err(|error| {
|
||||
ToolError::ExecutionFailed(format!(
|
||||
"stop Internal SubWorkers before Workdir detach: {error}"
|
||||
))
|
||||
})?;
|
||||
}
|
||||
self.backend.detach()
|
||||
}
|
||||
WorkdirOperation::Delete => self
|
||||
@@ -765,6 +783,7 @@ struct WorkdirDeleteInput {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use super::*;
|
||||
use crate::feature::{FeatureModule, FeatureRegistryBuilder};
|
||||
@@ -1155,6 +1174,7 @@ mod tests {
|
||||
command: "true".to_string(),
|
||||
timeout_secs: 120,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: Some("/worker-local/bash-output".into()),
|
||||
tool_call_id: Some("call-1".to_string()),
|
||||
})
|
||||
@@ -1178,83 +1198,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delegated_attached_session_carries_captured_fence_on_operations() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![
|
||||
response(json!({"value": "attachment-fence"})),
|
||||
response(json!({
|
||||
"operation": "stat",
|
||||
"result": {"path": "", "kind": "directory", "size": 0}
|
||||
})),
|
||||
response(json!({
|
||||
"operation": "stat",
|
||||
"result": {"path": "visible.txt", "kind": "file", "size": 8}
|
||||
})),
|
||||
]));
|
||||
let parent = workdir::delegation_capable_session(WorkspaceAttachedWorkdirSession::handle(
|
||||
client.clone(),
|
||||
));
|
||||
let delegation = parent
|
||||
.delegate(workdir::WorkdirDelegationRequest {
|
||||
rules: vec![workdir::WorkdirDelegationRule {
|
||||
target: workdir::WorkdirPath::new("").unwrap(),
|
||||
permission: workdir::WorkdirDelegationPermission::Read,
|
||||
recursive: false,
|
||||
}],
|
||||
cwd: workdir::WorkdirPath::new("").unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
delegation
|
||||
.scoped_session
|
||||
.stat(StatRequest {
|
||||
path: workdir::WorkdirPath::new("visible.txt").unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let requests = client.requests();
|
||||
assert_eq!(requests.len(), 3);
|
||||
assert_eq!(
|
||||
requests[0].path,
|
||||
"/api/w/workspace%2Ftest/workers/self/workdir-session/fence"
|
||||
);
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_str(requests[2].body.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(body["expected_session_fence"], "attachment-fence");
|
||||
assert_eq!(body["operation"]["operation"], "stat");
|
||||
assert_eq!(body["delegations"][0]["rules"][0]["target"], "");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attached_provider_rejection_happens_before_delegation_is_returned() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![
|
||||
response(json!({"value": "attachment-fence"})),
|
||||
response(json!({"error": "provider rejected delegated write target"})),
|
||||
]));
|
||||
let parent = workdir::delegation_capable_session(WorkspaceAttachedWorkdirSession::handle(
|
||||
client.clone(),
|
||||
));
|
||||
let result = parent
|
||||
.delegate(workdir::WorkdirDelegationRequest {
|
||||
rules: vec![workdir::WorkdirDelegationRule {
|
||||
target: workdir::WorkdirPath::new("linked-target").unwrap(),
|
||||
permission: workdir::WorkdirDelegationPermission::Write,
|
||||
recursive: true,
|
||||
}],
|
||||
cwd: workdir::WorkdirPath::new("linked-target").unwrap(),
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(result.is_err(), "provider rejection must fail before lease");
|
||||
let requests = client.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
let validation: serde_json::Value =
|
||||
serde_json::from_str(requests[1].body.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(validation["operation"]["operation"], "stat");
|
||||
assert_eq!(validation["delegations"].as_array().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attached_session_preserves_typed_provider_validation_error() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![error_response(
|
||||
@@ -1298,73 +1241,52 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nested_attached_session_preserves_full_delegation_chain() {
|
||||
async fn scoped_broker_operations_carry_no_child_context() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![
|
||||
response(json!({"value": "attachment-fence"})),
|
||||
response(json!({
|
||||
"operation": "stat",
|
||||
"result": {"path": "", "kind": "directory", "size": 0}
|
||||
"result": {"path": "visible.txt", "kind": "file", "size": 8}
|
||||
})),
|
||||
response(json!({
|
||||
"operation": "stat",
|
||||
"result": {"path": "nested", "kind": "directory", "size": 0}
|
||||
})),
|
||||
response(json!({
|
||||
"operation": "stat",
|
||||
"result": {"path": "nested/file", "kind": "file", "size": 1}
|
||||
"result": {"path": "visible.txt", "kind": "file", "size": 8}
|
||||
})),
|
||||
]));
|
||||
let parent = workdir::delegation_capable_session(WorkspaceAttachedWorkdirSession::handle(
|
||||
let broker = workdir::WorkdirToolBroker::new(WorkspaceAttachedWorkdirSession::handle(
|
||||
client.clone(),
|
||||
));
|
||||
let outer = parent
|
||||
.delegate(workdir::WorkdirDelegationRequest {
|
||||
rules: vec![workdir::WorkdirDelegationRule {
|
||||
let scoped = broker
|
||||
.scope(workdir::WorkdirToolScope {
|
||||
rules: vec![workdir::WorkdirToolScopeRule {
|
||||
target: workdir::WorkdirPath::new("").unwrap(),
|
||||
permission: workdir::WorkdirDelegationPermission::Read,
|
||||
permission: workdir::WorkdirToolScopePermission::Read,
|
||||
recursive: true,
|
||||
}],
|
||||
cwd: workdir::WorkdirPath::new("").unwrap(),
|
||||
command: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let nested = outer
|
||||
.scoped_session
|
||||
.delegate(workdir::WorkdirDelegationRequest {
|
||||
rules: vec![workdir::WorkdirDelegationRule {
|
||||
target: workdir::WorkdirPath::new("nested").unwrap(),
|
||||
permission: workdir::WorkdirDelegationPermission::Read,
|
||||
recursive: true,
|
||||
}],
|
||||
cwd: workdir::WorkdirPath::new("nested").unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
nested
|
||||
.scoped_session
|
||||
scoped
|
||||
.stat(StatRequest {
|
||||
path: workdir::WorkdirPath::new("file").unwrap(),
|
||||
path: workdir::WorkdirPath::new("visible.txt").unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let requests = client.requests();
|
||||
assert_eq!(requests.len(), 4);
|
||||
let outer_validation: serde_json::Value =
|
||||
serde_json::from_str(requests[1].body.as_deref().unwrap()).unwrap();
|
||||
let nested_validation: serde_json::Value =
|
||||
serde_json::from_str(requests[2].body.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(outer_validation["delegations"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(requests.len(), 2);
|
||||
for request in requests {
|
||||
assert_eq!(
|
||||
nested_validation["delegations"].as_array().unwrap().len(),
|
||||
2
|
||||
request.path,
|
||||
"/api/w/workspace%2Ftest/workers/self/workdir-session/operations"
|
||||
);
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_str(requests[3].body.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(body["delegations"].as_array().unwrap().len(), 2);
|
||||
assert_eq!(body["delegations"][0]["rules"][0]["target"], "");
|
||||
assert_eq!(body["delegations"][1]["rules"][0]["target"], "nested");
|
||||
assert_eq!(body["operation"]["request"]["path"], "file");
|
||||
serde_json::from_str(request.body.as_deref().unwrap()).unwrap();
|
||||
assert!(body.get("delegations").is_none());
|
||||
assert!(body.get("child").is_none());
|
||||
assert!(body.get("expected_session_fence").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1416,4 +1338,86 @@ mod tests {
|
||||
assert!(client.requests().is_empty());
|
||||
assert!(parse_input::<WorkdirListInput>(r#"{"path":"/tmp"}"#).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn detach_stops_internal_subworkers_before_backend_release() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||
"workspace_id": "workspace/test",
|
||||
"workdir_id": "wd-attached",
|
||||
"attached": false
|
||||
}))]));
|
||||
let cleanup_calls = Arc::new(AtomicUsize::new(0));
|
||||
let cleanup_calls_for_guard = cleanup_calls.clone();
|
||||
let before_release: BeforeWorkdirRelease = Arc::new(move || {
|
||||
let cleanup_calls = cleanup_calls_for_guard.clone();
|
||||
Box::pin(async move {
|
||||
cleanup_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(())
|
||||
})
|
||||
});
|
||||
let tool = WorkspaceHttpWorkdirTool {
|
||||
backend: WorkspaceHttpWorkdirBackend::new(client.clone())
|
||||
.with_child_lifecycle(Some(before_release), None),
|
||||
operation: WorkdirOperation::Detach,
|
||||
};
|
||||
|
||||
tool.execute("{}", ToolExecutionContext::default())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(client.requests().len(), 1);
|
||||
assert_eq!(
|
||||
client.requests()[0].path,
|
||||
"/api/w/workspace%2Ftest/workers/self/workdir-attachment"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn detach_does_not_release_backend_when_child_cleanup_fails() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(Vec::new()));
|
||||
let before_release: BeforeWorkdirRelease =
|
||||
Arc::new(|| Box::pin(async { Err(std::io::Error::other("child cleanup failed")) }));
|
||||
let tool = WorkspaceHttpWorkdirTool {
|
||||
backend: WorkspaceHttpWorkdirBackend::new(client.clone())
|
||||
.with_child_lifecycle(Some(before_release), None),
|
||||
operation: WorkdirOperation::Detach,
|
||||
};
|
||||
|
||||
let error = tool
|
||||
.execute("{}", ToolExecutionContext::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("stop Internal SubWorkers"));
|
||||
assert!(client.requests().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_attach_reopens_internal_subworker_admission() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||
"workspace_id": "workspace/test",
|
||||
"workdir_id": "wd-attached",
|
||||
"attached": true
|
||||
}))]));
|
||||
let reopen_calls = Arc::new(AtomicUsize::new(0));
|
||||
let reopen_calls_for_hook = reopen_calls.clone();
|
||||
let after_attach: AfterWorkdirAttach = Arc::new(move || {
|
||||
reopen_calls_for_hook.fetch_add(1, Ordering::SeqCst);
|
||||
});
|
||||
let tool = WorkspaceHttpWorkdirTool {
|
||||
backend: WorkspaceHttpWorkdirBackend::new(client)
|
||||
.with_child_lifecycle(None, Some(after_attach)),
|
||||
operation: WorkdirOperation::Attach,
|
||||
};
|
||||
|
||||
tool.execute(
|
||||
r#"{"workdir_id":"wd-attached"}"#,
|
||||
ToolExecutionContext::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(reopen_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -803,7 +803,10 @@ mod tests {
|
||||
.collect();
|
||||
Ok(WorkerSessionCapture {
|
||||
segment_id: "segment".to_string(),
|
||||
session: protocol::SessionSnapshot { entries },
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -709,7 +709,7 @@ pub(crate) fn prepare_internal_worker_from_spec(
|
||||
}
|
||||
|
||||
Box::pin(prepare_internal_worker_session(
|
||||
worker, store, visibility, None, None,
|
||||
worker, store, visibility, None, None, None,
|
||||
))
|
||||
.await
|
||||
})
|
||||
@@ -746,13 +746,16 @@ pub(crate) async fn prepare_internal_worker_session(
|
||||
visibility: InternalWorkerVisibility,
|
||||
child_registry: Option<Arc<SpawnedWorkerRegistry>>,
|
||||
on_turn_end: Option<Arc<dyn Fn(InternalWorkerSessionStatus) + Send + Sync>>,
|
||||
command_event_broker: Option<workdir::WorkdirToolBroker>,
|
||||
) -> Result<InternalWorkerSessionHandle, InternalWorkerSessionError> {
|
||||
let (event_tx, _event_rx) = broadcast::channel(256);
|
||||
let sink = worker.sink();
|
||||
spawn_internal_log_event_bridge(sink.clone(), event_tx.clone());
|
||||
let alerter = Alerter::new(event_tx.clone());
|
||||
let in_flight = InFlightEvents::new(event_tx.clone());
|
||||
if let Some(session) = worker.workdir_session() {
|
||||
if let Some(broker) = command_event_broker.as_ref() {
|
||||
wire_workdir_command_events(&broker.tool_session(), &in_flight);
|
||||
} else if let Some(session) = worker.workdir_session() {
|
||||
wire_workdir_command_events(session, &in_flight);
|
||||
}
|
||||
let actor_in_flight = in_flight.clone();
|
||||
@@ -887,6 +890,7 @@ pub(crate) async fn spawn_prepared_internal_worker_session(
|
||||
InternalWorkerVisibility::ServicePrivate,
|
||||
None,
|
||||
on_turn_end,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
handle.send(input).await?;
|
||||
|
||||
@@ -176,12 +176,23 @@ impl WorkerInterceptor {
|
||||
/// `Item::system_message`s reach the worker via
|
||||
/// `ContinueWith` / `pending_history_appends`, so on-disk order
|
||||
/// matches worker-history order.
|
||||
fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> {
|
||||
fn commit_system_items_with_extensions(
|
||||
&self,
|
||||
items: &[(
|
||||
SystemItem,
|
||||
Vec<session_store::SessionExtension>,
|
||||
Option<session_store::LoggedSessionHistoryOrigin>,
|
||||
)],
|
||||
) -> Result<(), session_store::StoreError> {
|
||||
let Some(writer) = self.log_writer.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
for item in items {
|
||||
let entry = writer.commit_system_item(item.clone())?;
|
||||
for (item, extensions, history_provenance) in items {
|
||||
let entry = writer.commit_system_item_with_extensions(
|
||||
item.clone(),
|
||||
extensions.clone(),
|
||||
history_provenance.clone(),
|
||||
)?;
|
||||
self.pending_committed_history
|
||||
.lock()
|
||||
.expect("pending committed history poisoned")
|
||||
@@ -190,6 +201,16 @@ impl WorkerInterceptor {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> {
|
||||
self.commit_system_items_with_extensions(
|
||||
&items
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|item| (item, Vec::new(), None))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}
|
||||
|
||||
fn current_turn_index(&self) -> usize {
|
||||
self.next_turn_index
|
||||
.load(Ordering::Relaxed)
|
||||
@@ -327,7 +348,11 @@ impl Interceptor<SessionHistoryMetadata> for WorkerInterceptor {
|
||||
projection_digest: projection.catalog_digest.clone(),
|
||||
logical_name: "internal.notify_wrapper".to_string(),
|
||||
};
|
||||
let mut system_items: Vec<SystemItem> = Vec::with_capacity(drained.len());
|
||||
let mut system_items: Vec<(
|
||||
SystemItem,
|
||||
Vec<session_store::SessionExtension>,
|
||||
Option<session_store::LoggedSessionHistoryOrigin>,
|
||||
)> = Vec::with_capacity(drained.len());
|
||||
let mut items: Vec<Item> = Vec::with_capacity(drained.len());
|
||||
for entry in &drained {
|
||||
let system_item = match build_system_item_with_provenance(
|
||||
@@ -345,9 +370,9 @@ impl Interceptor<SessionHistoryMetadata> for WorkerInterceptor {
|
||||
}
|
||||
};
|
||||
items.push(system_item.to_history_item());
|
||||
system_items.push(system_item);
|
||||
system_items.push((system_item, entry.extensions(), entry.history_provenance()));
|
||||
}
|
||||
if let Err(error) = self.commit_system_items(&system_items) {
|
||||
if let Err(error) = self.commit_system_items_with_extensions(&system_items) {
|
||||
self.pending_notifies.requeue_front(drained);
|
||||
return Err(InterceptorError::new(
|
||||
InterceptorErrorCategory::Dependency,
|
||||
|
||||
@@ -25,7 +25,7 @@ use std::collections::VecDeque;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use protocol::WorkerEvent;
|
||||
use session_store::SystemItem;
|
||||
use session_store::{LoggedSessionHistoryOrigin, SessionExtension, SystemItem};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::prompt::catalog::{CatalogError, PromptCatalog};
|
||||
@@ -41,8 +41,33 @@ const CAPACITY: usize = 128;
|
||||
/// is available.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum PendingNotify {
|
||||
Notify { message: String, auto_run: bool },
|
||||
WorkerEvent { event: WorkerEvent },
|
||||
Notify {
|
||||
message: String,
|
||||
auto_run: bool,
|
||||
extensions: Vec<SessionExtension>,
|
||||
history_provenance: Option<LoggedSessionHistoryOrigin>,
|
||||
},
|
||||
WorkerEvent {
|
||||
event: WorkerEvent,
|
||||
},
|
||||
}
|
||||
|
||||
impl PendingNotify {
|
||||
pub(crate) fn extensions(&self) -> Vec<SessionExtension> {
|
||||
match self {
|
||||
PendingNotify::Notify { extensions, .. } => extensions.clone(),
|
||||
PendingNotify::WorkerEvent { .. } => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn history_provenance(&self) -> Option<LoggedSessionHistoryOrigin> {
|
||||
match self {
|
||||
PendingNotify::Notify {
|
||||
history_provenance, ..
|
||||
} => history_provenance.clone(),
|
||||
PendingNotify::WorkerEvent { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared, mutex-guarded buffer of pending entries.
|
||||
@@ -62,7 +87,46 @@ impl NotifyBuffer {
|
||||
/// oldest entry is dropped and a `tracing::warn` is emitted — the
|
||||
/// caller should never hit this in normal operation.
|
||||
pub fn push_notify(&self, message: String, auto_run: bool) {
|
||||
self.push_entry(PendingNotify::Notify { message, auto_run });
|
||||
self.push_entry(PendingNotify::Notify {
|
||||
message,
|
||||
auto_run,
|
||||
extensions: Vec::new(),
|
||||
history_provenance: None,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn push_durable_notify(
|
||||
&self,
|
||||
message: String,
|
||||
auto_run: bool,
|
||||
history_provenance: LoggedSessionHistoryOrigin,
|
||||
extension: SessionExtension,
|
||||
) {
|
||||
self.push_entry(PendingNotify::Notify {
|
||||
message,
|
||||
auto_run,
|
||||
extensions: vec![extension],
|
||||
history_provenance: Some(history_provenance),
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn replace_durable_notification_extension(
|
||||
&self,
|
||||
extension: SessionExtension,
|
||||
) -> bool {
|
||||
let mut queue = self.inner.lock().expect("notify buffer poisoned");
|
||||
let Some(extensions) = queue.iter_mut().rev().find_map(|pending| match pending {
|
||||
PendingNotify::Notify {
|
||||
auto_run: false,
|
||||
extensions,
|
||||
..
|
||||
} if !extensions.is_empty() => Some(extensions),
|
||||
_ => None,
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
*extensions = vec![extension];
|
||||
true
|
||||
}
|
||||
|
||||
/// Push a typed worker-event entry onto the queue.
|
||||
@@ -202,6 +266,8 @@ mod tests {
|
||||
let entry = PendingNotify::Notify {
|
||||
message: "hello".into(),
|
||||
auto_run: false,
|
||||
extensions: Vec::new(),
|
||||
history_provenance: None,
|
||||
};
|
||||
let catalog = PromptCatalog::builtins_only().unwrap();
|
||||
let item = build_system_item(&entry, &catalog).unwrap();
|
||||
|
||||
@@ -57,9 +57,9 @@ pub use session_history::{
|
||||
};
|
||||
pub use shared_state::WorkerSharedState;
|
||||
pub use worker::{
|
||||
LocalWorkingDirectory, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, WorkerError,
|
||||
WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceClient,
|
||||
WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspacePromptCatalogResolution,
|
||||
WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, WorkspaceWorkerDiscoveryRequest,
|
||||
apply_worker_manifest, marker_workspace_client, unavailable_workspace_client,
|
||||
LocalWorkingDirectory, Worker, WorkerError, WorkerFilesystemAuthority, WorkerRunResult,
|
||||
WorkerWorkspaceContext, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
||||
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
|
||||
WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, marker_workspace_client,
|
||||
unavailable_workspace_client,
|
||||
};
|
||||
|
||||
@@ -291,6 +291,7 @@ mod tests {
|
||||
prompt_provenance: None,
|
||||
},
|
||||
),
|
||||
extensions: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -72,6 +72,7 @@ mod tests {
|
||||
fn snapshot(entries: Vec<serde_json::Value>) -> Event {
|
||||
Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: entries
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
|
||||
@@ -12,7 +12,7 @@ use std::collections::{BTreeMap, HashSet};
|
||||
use std::io;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::Instant;
|
||||
|
||||
@@ -23,9 +23,9 @@ use protocol::{Event, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnaps
|
||||
use session_store::{
|
||||
LoggedItem, WorkerMetadataStore, WorkerReclaimedChild, WorkerSpawnedChild, WorkerStoreError,
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
use tokio::sync::{Notify, broadcast};
|
||||
use tracing::warn;
|
||||
use workdir::WorkdirDelegation;
|
||||
use workdir::WorkdirScopeLease;
|
||||
|
||||
use crate::internal_worker::{InternalWorkerSessionHandle, InternalWorkerVisibility};
|
||||
use crate::runtime::dir::{RuntimeDir, SpawnedWorkerRecord};
|
||||
@@ -68,10 +68,11 @@ pub(crate) struct SubWorkerStopSummary {
|
||||
pub(crate) struct InternalSpawnedWorkerRecord {
|
||||
pub worker_name: String,
|
||||
pub scope_delegated: Vec<ScopeRule>,
|
||||
pub workdir_delegation: Arc<WorkdirDelegation>,
|
||||
pub workdir_tool_scope: Arc<WorkdirScopeLease>,
|
||||
#[cfg(test)]
|
||||
pub installed_tools: Arc<[String]>,
|
||||
pub session: InternalWorkerSessionHandle,
|
||||
pub child_registry: Arc<SpawnedWorkerRegistry>,
|
||||
change_tracker: Option<tools::Tracker>,
|
||||
started_at: Instant,
|
||||
stop_lock: Arc<tokio::sync::Mutex<()>>,
|
||||
@@ -86,18 +87,20 @@ impl InternalSpawnedWorkerRecord {
|
||||
pub(crate) fn new(
|
||||
worker_name: String,
|
||||
scope_delegated: Vec<ScopeRule>,
|
||||
workdir_delegation: WorkdirDelegation,
|
||||
workdir_tool_scope: WorkdirScopeLease,
|
||||
#[cfg(test)] installed_tools: Vec<String>,
|
||||
session: InternalWorkerSessionHandle,
|
||||
child_registry: Arc<SpawnedWorkerRegistry>,
|
||||
change_tracker: Option<tools::Tracker>,
|
||||
) -> Self {
|
||||
Self {
|
||||
worker_name,
|
||||
scope_delegated,
|
||||
workdir_delegation: Arc::new(workdir_delegation),
|
||||
workdir_tool_scope: Arc::new(workdir_tool_scope),
|
||||
#[cfg(test)]
|
||||
installed_tools: installed_tools.into(),
|
||||
session,
|
||||
child_registry,
|
||||
change_tracker,
|
||||
started_at: Instant::now(),
|
||||
stop_lock: Arc::new(tokio::sync::Mutex::new(())),
|
||||
@@ -235,18 +238,56 @@ pub(crate) struct InternalSpawnReservation {
|
||||
}
|
||||
|
||||
impl InternalSpawnReservation {
|
||||
pub(crate) fn commit(mut self, record: InternalSpawnedWorkerRecord) -> io::Result<()> {
|
||||
if record.worker_name != self.worker_name {
|
||||
return Err(io::Error::new(
|
||||
pub(crate) async fn commit(mut self, record: InternalSpawnedWorkerRecord) -> io::Result<()> {
|
||||
let rejection = if record.worker_name != self.worker_name {
|
||||
Some(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"internal SubWorker reservation name does not match record name",
|
||||
))
|
||||
} else {
|
||||
match self.registry.internal_records.lock() {
|
||||
Ok(mut records) => {
|
||||
if self.registry.internal_shutting_down.load(Ordering::Acquire) {
|
||||
Some(io::Error::new(
|
||||
io::ErrorKind::Interrupted,
|
||||
"internal SubWorker registry is shutting down",
|
||||
))
|
||||
} else {
|
||||
records.push(record.clone());
|
||||
None
|
||||
}
|
||||
}
|
||||
Err(_) => Some(io::Error::other(
|
||||
"internal spawned-worker registry lock poisoned",
|
||||
)),
|
||||
}
|
||||
};
|
||||
if let Some(error) = rejection {
|
||||
let mut cleanup_failures = Vec::new();
|
||||
if let Err(cleanup) = record.session.stop().await {
|
||||
cleanup_failures.push(format!("stop rejected Internal SubWorker: {cleanup}"));
|
||||
}
|
||||
if let Err(cleanup) = Box::pin(record.child_registry.shutdown_internal()).await {
|
||||
cleanup_failures.push(format!(
|
||||
"stop rejected Internal SubWorker descendants: {cleanup}"
|
||||
));
|
||||
}
|
||||
if let Err(cleanup) = record.workdir_tool_scope.close().await {
|
||||
cleanup_failures.push(format!(
|
||||
"close rejected Internal SubWorker Workdir tools: {cleanup}"
|
||||
));
|
||||
}
|
||||
if cleanup_failures.is_empty() {
|
||||
return Err(error);
|
||||
}
|
||||
self.registry
|
||||
.internal_records
|
||||
.lock()
|
||||
.map_err(|_| io::Error::other("internal spawned-worker registry lock poisoned"))?
|
||||
.push(record.clone());
|
||||
.internal_spawn_cleanup_failed
|
||||
.store(true, Ordering::Release);
|
||||
return Err(io::Error::other(format!(
|
||||
"{error}; {}",
|
||||
cleanup_failures.join("; ")
|
||||
)));
|
||||
}
|
||||
self.registry.start_protocol_forwarding(record);
|
||||
self.committed = true;
|
||||
Ok(())
|
||||
@@ -260,6 +301,10 @@ impl Drop for InternalSpawnReservation {
|
||||
names.remove(&self.worker_name);
|
||||
}
|
||||
}
|
||||
self.registry
|
||||
.pending_internal_spawns
|
||||
.fetch_sub(1, Ordering::AcqRel);
|
||||
self.registry.pending_internal_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -267,6 +312,10 @@ pub struct SpawnedWorkerRegistry {
|
||||
internal_records: std::sync::Mutex<Vec<InternalSpawnedWorkerRecord>>,
|
||||
service_records: std::sync::Mutex<Vec<InternalServiceWorkerRecord>>,
|
||||
internal_names: std::sync::Mutex<HashSet<String>>,
|
||||
internal_shutting_down: AtomicBool,
|
||||
pending_internal_spawns: AtomicUsize,
|
||||
pending_internal_notify: Notify,
|
||||
internal_spawn_cleanup_failed: AtomicBool,
|
||||
parent_scope: Option<SharedScope>,
|
||||
parent_protocol: Mutex<Option<(broadcast::Sender<Event>, String)>>,
|
||||
}
|
||||
@@ -283,6 +332,10 @@ impl SpawnedWorkerRegistry {
|
||||
internal_records: std::sync::Mutex::new(Vec::new()),
|
||||
service_records: std::sync::Mutex::new(Vec::new()),
|
||||
internal_names: std::sync::Mutex::new(HashSet::new()),
|
||||
internal_shutting_down: AtomicBool::new(false),
|
||||
pending_internal_spawns: AtomicUsize::new(0),
|
||||
pending_internal_notify: Notify::new(),
|
||||
internal_spawn_cleanup_failed: AtomicBool::new(false),
|
||||
parent_scope: None,
|
||||
parent_protocol: Mutex::new(None),
|
||||
})
|
||||
@@ -294,6 +347,10 @@ impl SpawnedWorkerRegistry {
|
||||
internal_records: std::sync::Mutex::new(Vec::new()),
|
||||
service_records: std::sync::Mutex::new(Vec::new()),
|
||||
internal_names: std::sync::Mutex::new(HashSet::new()),
|
||||
internal_shutting_down: AtomicBool::new(false),
|
||||
pending_internal_spawns: AtomicUsize::new(0),
|
||||
pending_internal_notify: Notify::new(),
|
||||
internal_spawn_cleanup_failed: AtomicBool::new(false),
|
||||
parent_scope: None,
|
||||
parent_protocol: Mutex::new(None),
|
||||
})
|
||||
@@ -304,6 +361,10 @@ impl SpawnedWorkerRegistry {
|
||||
internal_records: std::sync::Mutex::new(Vec::new()),
|
||||
service_records: std::sync::Mutex::new(Vec::new()),
|
||||
internal_names: std::sync::Mutex::new(HashSet::new()),
|
||||
internal_shutting_down: AtomicBool::new(false),
|
||||
pending_internal_spawns: AtomicUsize::new(0),
|
||||
pending_internal_notify: Notify::new(),
|
||||
internal_spawn_cleanup_failed: AtomicBool::new(false),
|
||||
parent_scope: Some(parent_scope),
|
||||
parent_protocol: Mutex::new(None),
|
||||
})
|
||||
@@ -383,6 +444,10 @@ impl SpawnedWorkerRegistry {
|
||||
internal_records: std::sync::Mutex::new(Vec::new()),
|
||||
service_records: std::sync::Mutex::new(Vec::new()),
|
||||
internal_names: std::sync::Mutex::new(HashSet::new()),
|
||||
internal_shutting_down: AtomicBool::new(false),
|
||||
pending_internal_spawns: AtomicUsize::new(0),
|
||||
pending_internal_notify: Notify::new(),
|
||||
internal_spawn_cleanup_failed: AtomicBool::new(false),
|
||||
parent_scope,
|
||||
parent_protocol: Mutex::new(None),
|
||||
}),
|
||||
@@ -394,6 +459,16 @@ impl SpawnedWorkerRegistry {
|
||||
self: &Arc<Self>,
|
||||
worker_name: String,
|
||||
) -> io::Result<InternalSpawnReservation> {
|
||||
let records = self
|
||||
.internal_records
|
||||
.lock()
|
||||
.map_err(|_| io::Error::other("internal Worker registry lock poisoned"))?;
|
||||
if self.internal_shutting_down.load(Ordering::Acquire) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Interrupted,
|
||||
"internal SubWorker registry is shutting down",
|
||||
));
|
||||
}
|
||||
let mut names = self
|
||||
.internal_names
|
||||
.lock()
|
||||
@@ -404,7 +479,9 @@ impl SpawnedWorkerRegistry {
|
||||
format!("spawned worker `{worker_name}` is already registered"),
|
||||
));
|
||||
}
|
||||
self.pending_internal_spawns.fetch_add(1, Ordering::AcqRel);
|
||||
drop(names);
|
||||
drop(records);
|
||||
Ok(InternalSpawnReservation {
|
||||
registry: Arc::clone(self),
|
||||
worker_name,
|
||||
@@ -679,18 +756,11 @@ impl SpawnedWorkerRegistry {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) fn reclaim_internal_scope(&self, worker_name: &str) -> io::Result<bool> {
|
||||
let record = self.get_internal(worker_name).ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::NotFound, "internal SubWorker not found")
|
||||
})?;
|
||||
self.reclaim_record_scope(&record)
|
||||
}
|
||||
|
||||
fn reclaim_record_scope(&self, record: &InternalSpawnedWorkerRecord) -> io::Result<bool> {
|
||||
if !record.claim_scope_reclaim() {
|
||||
return Ok(false);
|
||||
}
|
||||
record.workdir_delegation.release();
|
||||
record.workdir_tool_scope.revoke();
|
||||
let result = if let Some(parent_scope) = &self.parent_scope {
|
||||
parent_scope
|
||||
.update(|current| current.with_removed_deny_rules(delegated_write_rules(record)))
|
||||
@@ -705,6 +775,58 @@ impl SpawnedWorkerRegistry {
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn close_internal_scope(&self, name: &str) -> io::Result<bool> {
|
||||
let Some(record) = self.get_internal(name) else {
|
||||
return Ok(false);
|
||||
};
|
||||
Box::pin(record.child_registry.shutdown_internal()).await?;
|
||||
record
|
||||
.workdir_tool_scope
|
||||
.close()
|
||||
.await
|
||||
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||
self.reclaim_record_scope(&record)
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown_internal(&self) -> io::Result<()> {
|
||||
let names = {
|
||||
let records = self
|
||||
.internal_records
|
||||
.lock()
|
||||
.map_err(|_| io::Error::other("internal Worker registry lock poisoned"))?;
|
||||
self.internal_shutting_down.store(true, Ordering::Release);
|
||||
records
|
||||
.iter()
|
||||
.map(|record| record.worker_name.clone())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
loop {
|
||||
let notified = self.pending_internal_notify.notified();
|
||||
if self.pending_internal_spawns.load(Ordering::Acquire) == 0 {
|
||||
break;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
let mut first_error = None;
|
||||
for name in names {
|
||||
if let Err(error) = self.remove_internal(&name).await {
|
||||
first_error.get_or_insert(error);
|
||||
}
|
||||
}
|
||||
if first_error.is_none() && self.internal_spawn_cleanup_failed.load(Ordering::Acquire) {
|
||||
first_error = Some(io::Error::other(
|
||||
"an in-flight Internal SubWorker failed cleanup during shutdown",
|
||||
));
|
||||
}
|
||||
first_error.map_or(Ok(()), Err)
|
||||
}
|
||||
|
||||
pub(crate) fn reopen_internal(&self) {
|
||||
self.internal_shutting_down.store(false, Ordering::Release);
|
||||
self.internal_spawn_cleanup_failed
|
||||
.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Stop one direct Internal SubWorker and discard its registry/scope state.
|
||||
///
|
||||
/// The child actor must acknowledge its stop before the registry is removed.
|
||||
@@ -731,6 +853,12 @@ impl SpawnedWorkerRegistry {
|
||||
.stop()
|
||||
.await
|
||||
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||
Box::pin(record.child_registry.shutdown_internal()).await?;
|
||||
record
|
||||
.workdir_tool_scope
|
||||
.close()
|
||||
.await
|
||||
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||
let summary = record.stop_summary();
|
||||
self.reclaim_record_scope(&record)?;
|
||||
let removed =
|
||||
@@ -966,7 +1094,7 @@ mod tests {
|
||||
deny: Vec::new(),
|
||||
})
|
||||
.unwrap();
|
||||
let source = workdir::delegation_capable_session(Arc::new(
|
||||
let source = workdir::WorkdirToolBroker::new(Arc::new(
|
||||
workdir::LocalWorkdirSession::materialized_bound(
|
||||
workdir::Workdir::new("registry-test"),
|
||||
root.clone(),
|
||||
@@ -976,13 +1104,14 @@ mod tests {
|
||||
),
|
||||
));
|
||||
let delegation = source
|
||||
.delegate(workdir::WorkdirDelegationRequest {
|
||||
rules: vec![workdir::WorkdirDelegationRule {
|
||||
.scope(workdir::WorkdirToolScope {
|
||||
rules: vec![workdir::WorkdirToolScopeRule {
|
||||
target: workdir::WorkdirPath::new("").unwrap(),
|
||||
permission: workdir::WorkdirDelegationPermission::Read,
|
||||
permission: workdir::WorkdirToolScopePermission::Read,
|
||||
recursive: true,
|
||||
}],
|
||||
cwd: workdir::WorkdirPath::new("").unwrap(),
|
||||
command: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -993,6 +1122,7 @@ mod tests {
|
||||
delegation,
|
||||
Vec::new(),
|
||||
session,
|
||||
registry(),
|
||||
None,
|
||||
),
|
||||
sender,
|
||||
@@ -1230,6 +1360,143 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parent_shutdown_stops_all_internal_workers_before_returning() {
|
||||
let registry = registry();
|
||||
for name in ["first", "second"] {
|
||||
let (record, _events) = record(name, InternalWorkerVisibility::ParentClient).await;
|
||||
record
|
||||
.session
|
||||
.force_status(InternalWorkerSessionStatus::Running);
|
||||
install_record(®istry, record);
|
||||
}
|
||||
|
||||
registry.shutdown_internal().await.unwrap();
|
||||
|
||||
assert!(registry.list_internal().is_empty());
|
||||
assert!(registry.get_internal("first").is_none());
|
||||
assert!(registry.get_internal("second").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_rejects_new_reservations_until_reopened() {
|
||||
let registry = registry();
|
||||
registry.shutdown_internal().await.unwrap();
|
||||
assert!(registry.reserve_internal_name("late-child".into()).is_err());
|
||||
|
||||
registry.reopen_internal();
|
||||
let reservation = registry.reserve_internal_name("late-child".into()).unwrap();
|
||||
drop(reservation);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_commit_and_shutdown_leave_no_live_internal_worker() {
|
||||
let registry = registry();
|
||||
let reservation = registry
|
||||
.reserve_internal_name("racing-child".into())
|
||||
.unwrap();
|
||||
let (record, _events) =
|
||||
record("racing-child", InternalWorkerVisibility::ParentClient).await;
|
||||
let scope = record.workdir_tool_scope.clone();
|
||||
let barrier = Arc::new(tokio::sync::Barrier::new(2));
|
||||
let commit_barrier = barrier.clone();
|
||||
let commit = tokio::spawn(async move {
|
||||
commit_barrier.wait().await;
|
||||
reservation.commit(record).await
|
||||
});
|
||||
let shutdown_registry = registry.clone();
|
||||
let shutdown = tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
shutdown_registry.shutdown_internal().await
|
||||
});
|
||||
|
||||
let commit = commit.await.unwrap();
|
||||
shutdown.await.unwrap().unwrap();
|
||||
if let Err(error) = commit {
|
||||
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
|
||||
}
|
||||
|
||||
assert!(registry.list_internal().is_empty());
|
||||
assert!(!scope.is_active());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_fences_a_reservation_that_has_not_committed() {
|
||||
let registry = registry();
|
||||
let reservation = registry
|
||||
.reserve_internal_name("racing-child".into())
|
||||
.unwrap();
|
||||
let (record, _events) =
|
||||
record("racing-child", InternalWorkerVisibility::ParentClient).await;
|
||||
|
||||
let mut shutdown = {
|
||||
let registry = registry.clone();
|
||||
tokio::spawn(async move { registry.shutdown_internal().await })
|
||||
};
|
||||
while !registry.internal_shutting_down.load(Ordering::Acquire) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(50), &mut shutdown)
|
||||
.await
|
||||
.is_err(),
|
||||
"shutdown must wait for the pending spawn to roll back"
|
||||
);
|
||||
let error = reservation.commit(record).await.unwrap_err();
|
||||
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
|
||||
shutdown.await.unwrap().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_spawn_cleanup_failure_keeps_shutdown_failed_closed() {
|
||||
let registry = registry();
|
||||
let reservation = registry
|
||||
.reserve_internal_name("cleanup-failure".into())
|
||||
.unwrap();
|
||||
let (record, _events) =
|
||||
record("cleanup-failure", InternalWorkerVisibility::ParentClient).await;
|
||||
record.session.force_stop_failure();
|
||||
let shutdown = {
|
||||
let registry = registry.clone();
|
||||
tokio::spawn(async move { registry.shutdown_internal().await })
|
||||
};
|
||||
while !registry.internal_shutting_down.load(Ordering::Acquire) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
let error = reservation.commit(record).await.unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("stop rejected Internal SubWorker")
|
||||
);
|
||||
let shutdown_error = shutdown.await.unwrap().unwrap_err();
|
||||
assert!(
|
||||
shutdown_error
|
||||
.to_string()
|
||||
.contains("failed cleanup during shutdown")
|
||||
);
|
||||
assert!(registry.internal_shutting_down.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_recursively_stops_grandchildren_before_parent_scope_release() {
|
||||
let registry = registry();
|
||||
let (child, _child_events) = record("child", InternalWorkerVisibility::ParentClient).await;
|
||||
let child_registry = child.child_registry.clone();
|
||||
let (grandchild, _grandchild_events) =
|
||||
record("grandchild", InternalWorkerVisibility::ParentClient).await;
|
||||
let grandchild_scope = grandchild.workdir_tool_scope.clone();
|
||||
install_record(&child_registry, grandchild);
|
||||
install_record(®istry, child);
|
||||
|
||||
registry.shutdown_internal().await.unwrap();
|
||||
|
||||
assert!(registry.list_internal().is_empty());
|
||||
assert!(child_registry.list_internal().is_empty());
|
||||
assert!(!grandchild_scope.is_active());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn running_worker_is_stopped_before_removal() {
|
||||
let registry = registry();
|
||||
|
||||
+158
-194
@@ -22,8 +22,7 @@ use manifest::{
|
||||
use serde::Deserialize;
|
||||
use tokio::sync::mpsc;
|
||||
use workdir::{
|
||||
WorkdirDelegationPermission, WorkdirDelegationRequest, WorkdirDelegationRule, WorkdirPath,
|
||||
WorkdirSessionHandle,
|
||||
WorkdirToolBroker, WorkdirToolScope, WorkdirToolScopePermission, WorkdirToolScopeRule,
|
||||
};
|
||||
|
||||
use crate::PromptCatalogSource;
|
||||
@@ -58,12 +57,15 @@ struct SubWorkerSpawnInput {
|
||||
/// a host path and grants no authority. When omitted, the Workdir root is used.
|
||||
#[serde(default)]
|
||||
cwd: Option<String>,
|
||||
/// First message sent to the spawned SubWorker via `Method::Run`.
|
||||
/// First message sent to the spawned SubWorker via `Method::Submit`.
|
||||
task: String,
|
||||
/// Allow rules delegated to the spawned SubWorker. Must be a subset of the
|
||||
/// spawner's explicit delegation authority; direct tool scope alone is not
|
||||
/// sufficient. Omit `recursive` for normal workspace/worktree delegation; it defaults to true.
|
||||
scope: Vec<ScopeRuleInput>,
|
||||
/// Explicitly grant command execution through the parent-owned Workdir tool broker.
|
||||
#[serde(default)]
|
||||
command: bool,
|
||||
/// Binds an actual read-only builtin Reviewer child to the current Merge Request candidate.
|
||||
/// Review capability material is generated by the trusted spawn layer.
|
||||
#[serde(default)]
|
||||
@@ -219,33 +221,50 @@ fn parse_spawn_profile_selector(raw: Option<&str>) -> Result<SpawnProfileSelecto
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) enum ParentNotificationTarget {
|
||||
Controller(mpsc::WeakSender<Method>),
|
||||
Buffer(crate::ipc::notify_buffer::NotifyBuffer),
|
||||
Controller {
|
||||
sender: mpsc::WeakSender<Method>,
|
||||
fallback: Arc<dyn Fn(Method) + Send + Sync>,
|
||||
},
|
||||
Durable(Arc<dyn Fn(Method) + Send + Sync>),
|
||||
}
|
||||
|
||||
impl ParentNotificationTarget {
|
||||
fn notify(&self, message: String, auto_run: bool) {
|
||||
pub(crate) fn with_controller_fallback(
|
||||
sender: mpsc::WeakSender<Method>,
|
||||
fallback: ParentNotificationTarget,
|
||||
) -> Self {
|
||||
let ParentNotificationTarget::Durable(fallback) = fallback else {
|
||||
unreachable!("controller fallback must use durable pending authority");
|
||||
};
|
||||
Self::Controller { sender, fallback }
|
||||
}
|
||||
|
||||
pub(crate) fn notify(&self, child_session_id: String, message: String, auto_run: bool) {
|
||||
let method = Method::NotifyTracked {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message,
|
||||
auto_run,
|
||||
source: protocol::AuthenticatedInputSource::SubWorker {
|
||||
session_id: child_session_id,
|
||||
},
|
||||
};
|
||||
match self {
|
||||
Self::Controller(parent_method_tx) => {
|
||||
let Some(parent_method_tx) = parent_method_tx.upgrade() else {
|
||||
tracing::warn!(
|
||||
"parent Worker controller closed before Internal SubWorker completion notification"
|
||||
);
|
||||
Self::Controller { sender, fallback } => {
|
||||
let Some(parent_method_tx) = sender.upgrade() else {
|
||||
fallback(method);
|
||||
return;
|
||||
};
|
||||
let fallback = fallback.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(error) = parent_method_tx
|
||||
.send(Method::Notify { message, auto_run })
|
||||
.await
|
||||
{
|
||||
if let Err(error) = parent_method_tx.send(method).await {
|
||||
tracing::warn!(
|
||||
%error,
|
||||
"failed to notify parent Worker about Internal SubWorker completion"
|
||||
"failed to notify parent Controller; using durable pending authority"
|
||||
);
|
||||
fallback(error.0);
|
||||
}
|
||||
});
|
||||
}
|
||||
Self::Buffer(parent_notifies) => parent_notifies.push_notify(message, auto_run),
|
||||
Self::Durable(notify) => notify(method),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -267,8 +286,8 @@ pub struct SubWorkerSpawnTool {
|
||||
workspace_root: PathBuf,
|
||||
/// Directory the spawned SubWorker's tools should use when the LLM did not
|
||||
/// override it. Defaults to the spawner's cwd.
|
||||
/// Active provider-backed Workdir session from which child leases are captured.
|
||||
source_workdir_session: Option<WorkdirSessionHandle>,
|
||||
/// Parent-owned broker for scoped Workdir tool execution.
|
||||
workdir_tool_broker: Option<WorkdirToolBroker>,
|
||||
/// Parent-owned in-memory registry shared by the five SubWorker tools.
|
||||
registry: Arc<SpawnedWorkerRegistry>,
|
||||
/// Spawner's resolved Manifest. `profile = "inherit"` derives the
|
||||
@@ -295,7 +314,7 @@ impl SubWorkerSpawnTool {
|
||||
runtime_base: PathBuf,
|
||||
bash_output_dir: PathBuf,
|
||||
workspace_root: PathBuf,
|
||||
source_workdir_session: Option<WorkdirSessionHandle>,
|
||||
workdir_tool_broker: Option<WorkdirToolBroker>,
|
||||
registry: Arc<SpawnedWorkerRegistry>,
|
||||
spawner_manifest: WorkerManifest,
|
||||
prompt_loader: PromptCatalogSource,
|
||||
@@ -308,7 +327,7 @@ impl SubWorkerSpawnTool {
|
||||
runtime_base,
|
||||
bash_output_dir,
|
||||
workspace_root,
|
||||
source_workdir_session,
|
||||
workdir_tool_broker,
|
||||
registry,
|
||||
spawner_manifest,
|
||||
prompt_loader,
|
||||
@@ -341,6 +360,11 @@ fn validate_reviewer_handoff(input: &SubWorkerSpawnInput) -> Result<(), ToolErro
|
||||
"Merge Request Reviewer SubWorkers must include writable delegated scope".to_string(),
|
||||
));
|
||||
}
|
||||
if !input.command {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"Merge Request Reviewer SubWorkers require an explicit command grant".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -370,7 +394,7 @@ impl Tool for SubWorkerSpawnTool {
|
||||
.reserve_internal_name(input.name.clone())
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
|
||||
let mut workdir_rules = parse_workdir_scope(&input.scope)?;
|
||||
let workdir_rules = parse_workdir_scope(&input.scope)?;
|
||||
let child_bash_output_dir = self.bash_output_dir.join("sub-workers").join(&input.name);
|
||||
tokio::fs::create_dir_all(&child_bash_output_dir)
|
||||
.await
|
||||
@@ -380,28 +404,15 @@ impl Tool for SubWorkerSpawnTool {
|
||||
child_bash_output_dir.display()
|
||||
))
|
||||
})?;
|
||||
let source_workdir_session =
|
||||
require_active_workdir_session(self.source_workdir_session.as_ref())?;
|
||||
let transports_delegation_context = source_workdir_session.transports_delegation_context();
|
||||
// Provider-transported sessions resolve every delegation rule in the
|
||||
// receiving Workdir namespace. The Bash spill directory instead belongs
|
||||
// to this Worker host, so forwarding it would widen the request with a
|
||||
// foreign absolute path and fail the provider's existing scope check.
|
||||
if !transports_delegation_context {
|
||||
workdir_rules.push(WorkdirDelegationRule {
|
||||
target: WorkdirPath::new_scoped(child_bash_output_dir.to_string_lossy())
|
||||
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?,
|
||||
permission: WorkdirDelegationPermission::Read,
|
||||
recursive: true,
|
||||
});
|
||||
}
|
||||
let delegation_request = workdir_delegation_request(input.cwd.as_deref(), workdir_rules)?;
|
||||
let workdir_delegation = source_workdir_session
|
||||
.delegate(delegation_request)
|
||||
let workdir_tool_broker = require_workdir_tool_broker(self.workdir_tool_broker.as_ref())?;
|
||||
let tool_scope = workdir_tool_scope(input.cwd.as_deref(), workdir_rules, input.command)?;
|
||||
let workdir_scope = workdir_tool_broker
|
||||
.scope(tool_scope)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
ToolError::InvalidArgument(format!("delegate Workdir session: {error}"))
|
||||
ToolError::InvalidArgument(format!("scope parent-owned Workdir tools: {error}"))
|
||||
})?;
|
||||
let child_workdir_tool_broker = workdir_scope.broker();
|
||||
|
||||
let spawn_selector =
|
||||
parse_spawn_profile_selector(input.profile.as_deref()).map_err(|msg| {
|
||||
@@ -490,7 +501,6 @@ impl Tool for SubWorkerSpawnTool {
|
||||
)
|
||||
.await
|
||||
.map_err(|error| ToolError::ExecutionFailed(format!("build Internal Worker: {error}")))?;
|
||||
child.bind_workdir_session(Some(workdir_delegation.scoped_session.clone()));
|
||||
child
|
||||
.add_scope_rules([ScopeRule {
|
||||
target: child_bash_output_dir.clone(),
|
||||
@@ -510,6 +520,7 @@ impl Tool for SubWorkerSpawnTool {
|
||||
self.runtime_base.clone(),
|
||||
child_registry.clone(),
|
||||
None,
|
||||
Some(child_workdir_tool_broker.clone()),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
@@ -538,20 +549,24 @@ impl Tool for SubWorkerSpawnTool {
|
||||
InternalWorkerSessionStatus::Failed | InternalWorkerSessionStatus::Stopped
|
||||
) {
|
||||
if let Some(registry) = registry.upgrade() {
|
||||
if let Err(error) = registry.reclaim_internal_scope(&child_name) {
|
||||
let child_name = child_name.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(error) = registry.close_internal_scope(&child_name).await {
|
||||
tracing::warn!(
|
||||
child_name,
|
||||
%error,
|
||||
"failed to reclaim delegated scope after Internal SubWorker failure"
|
||||
"failed to close parent-owned Workdir tools after Internal SubWorker failure"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
let message = format!(
|
||||
"SubWorker `{child_name}` turn ended with status {status:?}. Inspect its committed session with worker-observation tools before making completion decisions."
|
||||
);
|
||||
parent_notifications.notify(message, true);
|
||||
parent_notifications.notify(child_name.clone(), message, true);
|
||||
})),
|
||||
Some(child_workdir_tool_broker.clone()),
|
||||
)
|
||||
.await;
|
||||
let session = session_result.map_err(|error| {
|
||||
@@ -602,15 +617,19 @@ impl Tool for SubWorkerSpawnTool {
|
||||
),
|
||||
body.to_string(),
|
||||
);
|
||||
let response = self
|
||||
.workspace_context
|
||||
.client()
|
||||
.execute(request)
|
||||
.map_err(|error| {
|
||||
ToolError::ExecutionFailed(format!("register review capability: {error}"))
|
||||
})?;
|
||||
let response = match self.workspace_context.client().execute(request) {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
let _ = session.stop().await;
|
||||
let _ = workdir_scope.close().await;
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"register review capability: {error}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
if !response.is_success() {
|
||||
let _ = session.stop().await;
|
||||
let _ = workdir_scope.close().await;
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"register review capability failed with status {}: {}",
|
||||
response.status, response.body
|
||||
@@ -621,14 +640,14 @@ impl Tool for SubWorkerSpawnTool {
|
||||
let record = crate::spawn::registry::InternalSpawnedWorkerRecord::new(
|
||||
input.name.clone(),
|
||||
scope_allow,
|
||||
workdir_delegation,
|
||||
workdir_scope,
|
||||
#[cfg(test)]
|
||||
installed_tools,
|
||||
session.clone(),
|
||||
child_registry,
|
||||
child_change_tracker,
|
||||
);
|
||||
if let Err(error) = name_reservation.commit(record) {
|
||||
let _ = session.stop().await;
|
||||
if let Err(error) = name_reservation.commit(record).await {
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"register Internal Worker session: {error}"
|
||||
)));
|
||||
@@ -674,18 +693,18 @@ fn logical_workdir_path(value: &str, field: &str) -> Result<FsPath, ToolError> {
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_workdir_scope(rules: &[ScopeRuleInput]) -> Result<Vec<WorkdirDelegationRule>, ToolError> {
|
||||
fn parse_workdir_scope(rules: &[ScopeRuleInput]) -> Result<Vec<WorkdirToolScopeRule>, ToolError> {
|
||||
if rules.is_empty() {
|
||||
return Err(ToolError::InvalidArgument("scope must not be empty".into()));
|
||||
}
|
||||
rules
|
||||
.iter()
|
||||
.map(|rule| {
|
||||
Ok(WorkdirDelegationRule {
|
||||
Ok(WorkdirToolScopeRule {
|
||||
target: logical_workdir_path(&rule.target, "scope.target")?,
|
||||
permission: match rule.permission {
|
||||
PermissionInput::Read => WorkdirDelegationPermission::Read,
|
||||
PermissionInput::Write => WorkdirDelegationPermission::Write,
|
||||
PermissionInput::Read => WorkdirToolScopePermission::Read,
|
||||
PermissionInput::Write => WorkdirToolScopePermission::Write,
|
||||
},
|
||||
recursive: rule.recursive,
|
||||
})
|
||||
@@ -693,22 +712,24 @@ fn parse_workdir_scope(rules: &[ScopeRuleInput]) -> Result<Vec<WorkdirDelegation
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn workdir_delegation_request(
|
||||
fn workdir_tool_scope(
|
||||
cwd: Option<&str>,
|
||||
rules: Vec<WorkdirDelegationRule>,
|
||||
) -> Result<WorkdirDelegationRequest, ToolError> {
|
||||
Ok(WorkdirDelegationRequest {
|
||||
rules: Vec<WorkdirToolScopeRule>,
|
||||
command: bool,
|
||||
) -> Result<WorkdirToolScope, ToolError> {
|
||||
Ok(WorkdirToolScope {
|
||||
rules,
|
||||
cwd: logical_workdir_path(cwd.unwrap_or("."), "cwd")?,
|
||||
command,
|
||||
})
|
||||
}
|
||||
|
||||
fn require_active_workdir_session(
|
||||
session: Option<&WorkdirSessionHandle>,
|
||||
) -> Result<&WorkdirSessionHandle, ToolError> {
|
||||
session.ok_or_else(|| {
|
||||
fn require_workdir_tool_broker(
|
||||
broker: Option<&WorkdirToolBroker>,
|
||||
) -> Result<&WorkdirToolBroker, ToolError> {
|
||||
broker.ok_or_else(|| {
|
||||
ToolError::InvalidArgument(
|
||||
"SubWorkerSpawn requires an active Workdir session; attach a Workdir before delegating filesystem access"
|
||||
"SubWorkerSpawn requires parent-owned Workdir tools; attach a Workdir before granting filesystem access"
|
||||
.to_string(),
|
||||
)
|
||||
})
|
||||
@@ -946,7 +967,7 @@ pub(crate) fn sub_worker_spawn_tool(
|
||||
runtime_base: PathBuf,
|
||||
bash_output_dir: PathBuf,
|
||||
workspace_root: PathBuf,
|
||||
source_workdir_session: Option<WorkdirSessionHandle>,
|
||||
workdir_tool_broker: Option<WorkdirToolBroker>,
|
||||
registry: Arc<SpawnedWorkerRegistry>,
|
||||
spawner_manifest: WorkerManifest,
|
||||
prompts: Arc<ArcSwap<PromptCatalog>>,
|
||||
@@ -958,7 +979,7 @@ pub(crate) fn sub_worker_spawn_tool(
|
||||
runtime_base,
|
||||
bash_output_dir,
|
||||
workspace_root,
|
||||
source_workdir_session,
|
||||
workdir_tool_broker,
|
||||
registry,
|
||||
spawner_manifest,
|
||||
prompts,
|
||||
@@ -972,7 +993,7 @@ fn sub_worker_spawn_tool_impl(
|
||||
runtime_base: PathBuf,
|
||||
bash_output_dir: PathBuf,
|
||||
workspace_root: PathBuf,
|
||||
source_workdir_session: Option<WorkdirSessionHandle>,
|
||||
workdir_tool_broker: Option<WorkdirToolBroker>,
|
||||
registry: Arc<SpawnedWorkerRegistry>,
|
||||
spawner_manifest: WorkerManifest,
|
||||
prompts: Arc<ArcSwap<PromptCatalog>>,
|
||||
@@ -1004,7 +1025,7 @@ fn sub_worker_spawn_tool_impl(
|
||||
runtime_base.clone(),
|
||||
bash_output_dir.clone(),
|
||||
workspace_root.clone(),
|
||||
source_workdir_session.clone(),
|
||||
workdir_tool_broker.clone(),
|
||||
registry.clone(),
|
||||
spawner_manifest.clone(),
|
||||
prompts.load_full().source(),
|
||||
@@ -1037,12 +1058,12 @@ mod tests {
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn missing_active_workdir_session_fails_deterministically() {
|
||||
let error = require_active_workdir_session(None).unwrap_err();
|
||||
fn missing_parent_workdir_tool_broker_fails_deterministically() {
|
||||
let error = require_workdir_tool_broker(None).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
ToolError::InvalidArgument(message)
|
||||
if message.contains("requires an active Workdir session")
|
||||
if message.contains("requires parent-owned Workdir tools")
|
||||
));
|
||||
}
|
||||
|
||||
@@ -1079,6 +1100,7 @@ mod tests {
|
||||
let valid: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({
|
||||
"name":"reviewer","task":"review","profile":"builtin:reviewer",
|
||||
"scope":[{"target":"work","permission":"write"}],
|
||||
"command":true,
|
||||
"review":{"ticket_id":"T1"}
|
||||
}))
|
||||
.unwrap();
|
||||
@@ -1134,12 +1156,41 @@ enabled = false
|
||||
#[tokio::test]
|
||||
async fn parent_controller_notification_target_does_not_keep_channel_open() {
|
||||
let (parent_method_tx, mut parent_method_rx) = mpsc::channel(1);
|
||||
let target = ParentNotificationTarget::Controller(parent_method_tx.downgrade());
|
||||
let captured = Arc::new(std::sync::Mutex::new(false));
|
||||
let captured_for_fallback = captured.clone();
|
||||
let target = ParentNotificationTarget::with_controller_fallback(
|
||||
parent_method_tx.downgrade(),
|
||||
ParentNotificationTarget::Durable(Arc::new(move |_| {
|
||||
*captured_for_fallback.lock().unwrap() = true;
|
||||
})),
|
||||
);
|
||||
|
||||
drop(parent_method_tx);
|
||||
|
||||
assert!(parent_method_rx.recv().await.is_none());
|
||||
target.notify("late completion".to_string(), true);
|
||||
target.notify("child-session".into(), "late completion".to_string(), true);
|
||||
assert!(*captured.lock().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_parent_notification_target_preserves_child_source() {
|
||||
let captured = Arc::new(std::sync::Mutex::new(None));
|
||||
let captured_for_target = captured.clone();
|
||||
let target = ParentNotificationTarget::Durable(Arc::new(move |method| {
|
||||
*captured_for_target.lock().unwrap() = Some(method);
|
||||
}));
|
||||
|
||||
target.notify("child-session".into(), "completed".into(), true);
|
||||
|
||||
assert!(matches!(
|
||||
captured.lock().unwrap().take(),
|
||||
Some(Method::NotifyTracked {
|
||||
message,
|
||||
auto_run: true,
|
||||
source: protocol::AuthenticatedInputSource::SubWorker { session_id },
|
||||
..
|
||||
}) if session_id == "child-session" && message == "completed"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1173,7 +1224,7 @@ enabled = false
|
||||
let fail_requests = Arc::new(AtomicBool::new(false));
|
||||
let prompt_loader = PromptCatalogSource::builtins_only();
|
||||
let (parent_method_tx, mut parent_method_rx) = mpsc::channel(8);
|
||||
let source_workdir_session = workdir::delegation_capable_session(Arc::new(
|
||||
let workdir_tool_broker = workdir::WorkdirToolBroker::new(Arc::new(
|
||||
workdir::LocalWorkdirSession::materialized_bound(
|
||||
workdir::Workdir::new("test-workdir"),
|
||||
workspace_root.clone(),
|
||||
@@ -1185,11 +1236,14 @@ enabled = false
|
||||
let tool = SubWorkerSpawnTool::new(
|
||||
"parent".into(),
|
||||
workspace_context,
|
||||
ParentNotificationTarget::Controller(parent_method_tx.downgrade()),
|
||||
ParentNotificationTarget::with_controller_fallback(
|
||||
parent_method_tx.downgrade(),
|
||||
ParentNotificationTarget::Durable(Arc::new(|_| {})),
|
||||
),
|
||||
runtime.path().to_path_buf(),
|
||||
bash_output_dir.clone(),
|
||||
workspace_root.clone(),
|
||||
Some(source_workdir_session),
|
||||
Some(workdir_tool_broker),
|
||||
registry.clone(),
|
||||
manifest.clone(),
|
||||
prompt_loader,
|
||||
@@ -1212,7 +1266,8 @@ enabled = false
|
||||
"target": ".",
|
||||
"permission": "write",
|
||||
"recursive": true
|
||||
}]
|
||||
}],
|
||||
"command": true
|
||||
});
|
||||
|
||||
assert!(spawner_scope.snapshot().is_writable(&workspace_root));
|
||||
@@ -1247,15 +1302,6 @@ enabled = false
|
||||
let record = registry
|
||||
.get_internal("reviewer-child")
|
||||
.expect("Internal reviewer registry record");
|
||||
let child_bash_output_dir = bash_output_dir.join("sub-workers").join("reviewer-child");
|
||||
record
|
||||
.workdir_delegation
|
||||
.scoped_session
|
||||
.stat(workdir::StatRequest {
|
||||
path: WorkdirPath::new_scoped(child_bash_output_dir.to_string_lossy()).unwrap(),
|
||||
})
|
||||
.await
|
||||
.expect("local child retains read scope for its Bash output directory");
|
||||
for required in ["Read", "Write", "Edit", "Glob", "Grep", "Bash"] {
|
||||
assert!(
|
||||
record.installed_tools.iter().any(|name| name == required),
|
||||
@@ -1282,10 +1328,13 @@ enabled = false
|
||||
.expect("parent method channel remains open");
|
||||
assert!(matches!(
|
||||
completion,
|
||||
Method::Notify {
|
||||
Method::NotifyTracked {
|
||||
message,
|
||||
auto_run: true,
|
||||
} if message.contains("SubWorker `reviewer-child` turn ended with status Idle")
|
||||
source: protocol::AuthenticatedInputSource::SubWorker { session_id },
|
||||
..
|
||||
} if session_id == "reviewer-child"
|
||||
&& message.contains("SubWorker `reviewer-child` turn ended with status Idle")
|
||||
));
|
||||
assert!(!runtime.path().join("reviewer-child/sock").exists());
|
||||
|
||||
@@ -1371,7 +1420,7 @@ enabled = false
|
||||
"Stopped terminal child must release its delegated Workdir session"
|
||||
);
|
||||
assert!(
|
||||
!record.workdir_delegation.is_active(),
|
||||
!record.workdir_tool_scope.is_active(),
|
||||
"stopped child must revoke cloned scoped sessions"
|
||||
);
|
||||
assert!(registry.get_internal("reviewer-child").is_some());
|
||||
@@ -1426,7 +1475,7 @@ enabled = false
|
||||
Arc::new(AvailableWorkspaceClient),
|
||||
);
|
||||
let remote_client = Arc::new(StrictRemoteWorkdirWorkspaceClient::default());
|
||||
let source_workdir_session = workdir::delegation_capable_session(
|
||||
let workdir_tool_broker = workdir::WorkdirToolBroker::new(
|
||||
WorkspaceAttachedWorkdirSession::handle(remote_client.clone()),
|
||||
);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
@@ -1434,11 +1483,14 @@ enabled = false
|
||||
let tool = SubWorkerSpawnTool::new(
|
||||
"parent".into(),
|
||||
workspace_context,
|
||||
ParentNotificationTarget::Controller(parent_method_tx.downgrade()),
|
||||
ParentNotificationTarget::with_controller_fallback(
|
||||
parent_method_tx.downgrade(),
|
||||
ParentNotificationTarget::Durable(Arc::new(|_| {})),
|
||||
),
|
||||
runtime.path().to_path_buf(),
|
||||
bash_output_dir.clone(),
|
||||
workspace_root.clone(),
|
||||
Some(source_workdir_session),
|
||||
Some(workdir_tool_broker),
|
||||
registry.clone(),
|
||||
manifest,
|
||||
PromptCatalogSource::builtins_only(),
|
||||
@@ -1478,51 +1530,12 @@ enabled = false
|
||||
record.session.wait_until_idle().await,
|
||||
crate::internal_worker::InternalWorkerSessionStatus::Idle
|
||||
);
|
||||
assert!(record.installed_tools.iter().any(|tool| tool == "Write"));
|
||||
assert!(!record.installed_tools.iter().any(|tool| tool == "Bash"));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
remote_client
|
||||
.foreign_scope_rejections
|
||||
.load(Ordering::SeqCst),
|
||||
0
|
||||
);
|
||||
let child_bash_output_dir = bash_output_dir.join("sub-workers").join("remote-child");
|
||||
assert!(child_bash_output_dir.is_dir());
|
||||
for required in ["Read", "Write", "Edit", "Glob", "Grep", "Bash"] {
|
||||
assert!(
|
||||
record.installed_tools.iter().any(|name| name == required),
|
||||
"remote write-scoped child is missing {required}: {:?}",
|
||||
record.installed_tools
|
||||
);
|
||||
}
|
||||
|
||||
let remote_requests = remote_client.requests();
|
||||
let operate_requests = remote_requests
|
||||
.iter()
|
||||
.filter(|request| request.body.is_some())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
operate_requests.len(),
|
||||
1,
|
||||
"remote requests: {remote_requests:?}"
|
||||
);
|
||||
let operation_body: serde_json::Value = serde_json::from_str(
|
||||
operate_requests[0]
|
||||
.body
|
||||
.as_deref()
|
||||
.expect("remote operation body"),
|
||||
)
|
||||
.unwrap();
|
||||
let rules = operation_body["delegations"][0]["rules"]
|
||||
.as_array()
|
||||
.expect("delegation rules");
|
||||
assert_eq!(rules.len(), 1, "remote operation body: {operation_body}");
|
||||
assert_eq!(rules[0]["target"], "");
|
||||
assert!(
|
||||
!operation_body.to_string().contains(
|
||||
child_bash_output_dir
|
||||
.to_str()
|
||||
.expect("UTF-8 test output directory")
|
||||
)
|
||||
remote_client.requests().is_empty(),
|
||||
"spawning a child must not open or delegate a provider Workdir session"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1534,6 +1547,7 @@ enabled = false
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("schema properties");
|
||||
assert!(properties.contains_key("cwd"), "schema: {schema}");
|
||||
assert!(properties.contains_key("command"), "schema: {schema}");
|
||||
let required = schema
|
||||
.get("required")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
@@ -1663,7 +1677,6 @@ enabled = false
|
||||
#[derive(Debug, Default)]
|
||||
struct StrictRemoteWorkdirWorkspaceClient {
|
||||
requests: Mutex<Vec<WorkspaceRequest>>,
|
||||
foreign_scope_rejections: AtomicUsize,
|
||||
}
|
||||
|
||||
impl StrictRemoteWorkdirWorkspaceClient {
|
||||
@@ -1695,59 +1708,10 @@ enabled = false
|
||||
self.requests
|
||||
.lock()
|
||||
.expect("remote Workdir request lock")
|
||||
.push(request.clone());
|
||||
if request.path.ends_with("/fence") {
|
||||
return Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: serde_json::json!({ "value": "remote-fence-1" }).to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let body: serde_json::Value = serde_json::from_str(
|
||||
request
|
||||
.body
|
||||
.as_deref()
|
||||
.ok_or_else(|| WorkspaceClientError::Request("missing request body".into()))?,
|
||||
)
|
||||
.map_err(|error| WorkspaceClientError::Request(error.to_string()))?;
|
||||
let has_foreign_scope = body
|
||||
.get("delegations")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.flat_map(|delegation| {
|
||||
delegation
|
||||
.get("rules")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
})
|
||||
.filter_map(|rule| rule.get("target").and_then(serde_json::Value::as_str))
|
||||
.any(|target| Path::new(target).is_absolute());
|
||||
if has_foreign_scope {
|
||||
self.foreign_scope_rejections.fetch_add(1, Ordering::SeqCst);
|
||||
return Ok(WorkspaceResponse {
|
||||
status: 403,
|
||||
body: serde_json::json!({
|
||||
"code": "out_of_scope",
|
||||
"message": "Worker-host path is outside the remote Workdir namespace"
|
||||
})
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: serde_json::json!({
|
||||
"operation": "stat",
|
||||
"result": {
|
||||
"path": "",
|
||||
"kind": "directory",
|
||||
"size": 0
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
})
|
||||
.push(request);
|
||||
Err(WorkspaceClientError::Request(
|
||||
"SubWorker spawn must not call the remote Workdir provider".into(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1669
-19
File diff suppressed because it is too large
Load Diff
@@ -630,7 +630,10 @@ async fn controller_compact_method_emits_start_and_done() {
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle
|
||||
.send(Method::run_text("seed history"))
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"seed history",
|
||||
))
|
||||
.await
|
||||
.expect("send run");
|
||||
loop {
|
||||
|
||||
@@ -332,6 +332,7 @@ async fn shutdown_closes_bound_workdir_session() {
|
||||
command: "sleep 30".to_owned(),
|
||||
timeout_secs: 60,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: None,
|
||||
})
|
||||
@@ -376,6 +377,7 @@ async fn controller_projects_workdir_command_events_and_snapshot_state() {
|
||||
command: "printf ready; sleep 0.3; printf done".to_owned(),
|
||||
timeout_secs: 5,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: Some("tool-command-1".into()),
|
||||
})
|
||||
@@ -484,6 +486,7 @@ async fn controller_refreshes_command_snapshot_after_high_output_provider_lag()
|
||||
.to_owned(),
|
||||
timeout_secs: 10,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: Some("tool-high-output".into()),
|
||||
})
|
||||
@@ -560,6 +563,7 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
||||
command: "printf unreachable".to_owned(),
|
||||
timeout_secs: 5,
|
||||
output_limit: 1024,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: None,
|
||||
})
|
||||
@@ -617,7 +621,13 @@ async fn feature_flags_default_to_core_tool_surface_only() {
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
|
||||
let request = wait_for_captured_request(&client_for_assert).await;
|
||||
@@ -672,7 +682,13 @@ permission = "write"
|
||||
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
|
||||
let request = wait_for_captured_request(&client_for_assert).await;
|
||||
@@ -758,7 +774,13 @@ permission = "write"
|
||||
let worker = make_worker_with_pwd_and_manifest(client, &manifest).await.0;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
|
||||
let request = wait_for_captured_request(&client_for_assert).await;
|
||||
@@ -826,7 +848,13 @@ async fn builtin_orchestrator_exposes_worker_remove_and_workdir_delete() {
|
||||
.await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
let request = wait_for_captured_request(&client_for_assert).await;
|
||||
let installed = request_tool_names(&request);
|
||||
@@ -875,7 +903,13 @@ permission = "write"
|
||||
.0;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
|
||||
let request = wait_for_captured_request(&client_for_assert).await;
|
||||
@@ -928,7 +962,13 @@ permission = "write"
|
||||
)
|
||||
.await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
let request = wait_for_captured_request(&client_for_assert).await;
|
||||
let names = request_tool_names(&request);
|
||||
@@ -975,7 +1015,13 @@ async fn run_end_returns_to_idle_without_busy_status() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut saw_run_end = false;
|
||||
let mut saw_idle_status = false;
|
||||
@@ -1017,7 +1063,13 @@ async fn provider_stream_error_records_run_errored() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::run_text("ping")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"ping",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
@@ -1066,7 +1118,10 @@ async fn snapshot_includes_user_input_for_in_flight_turn() {
|
||||
let mut events = handle.subscribe();
|
||||
|
||||
handle
|
||||
.send(Method::run_text("hello in-flight"))
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"hello in-flight",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
@@ -1131,7 +1186,13 @@ async fn attach_snapshot_includes_current_status() {
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
|
||||
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
|
||||
@@ -1169,7 +1230,13 @@ async fn run_updates_shared_state_to_idle_after_completion() {
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Wait for the run to complete
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
@@ -1183,7 +1250,13 @@ async fn run_populates_history() {
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
|
||||
@@ -1201,7 +1274,13 @@ async fn events_are_broadcast() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::run_text("Hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut saw_turn_start = false;
|
||||
let mut saw_text_delta = false;
|
||||
@@ -1236,10 +1315,8 @@ async fn events_are_broadcast() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn double_run_returns_error() {
|
||||
// Keep the first turn in-flight until the test drops the handle. A
|
||||
// finite stream can finish before the second Method reaches the
|
||||
// controller in the full test suite, making this assertion racy.
|
||||
async fn submit_while_running_is_durably_queued() {
|
||||
// Keep the first turn in-flight until the second Submit is accepted.
|
||||
let events = vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
LlmEvent::text_delta(0, "slow..."),
|
||||
@@ -1249,35 +1326,67 @@ async fn double_run_returns_error() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
// Send first run and wait until the controller has entered Running.
|
||||
handle.send(Method::run_text("first")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text("request-first", "first"))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
handle
|
||||
.send(Method::submit_text("request-second", "second"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Now the second run must be rejected by drive_turn's live Method arm.
|
||||
handle.send(Method::run_text("second")).await.unwrap();
|
||||
|
||||
// Look for the error event
|
||||
let mut saw_already_running = false;
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = rx.recv() => {
|
||||
match event {
|
||||
Ok(Event::Error { code, .. }) => {
|
||||
if code == worker::ErrorCode::AlreadyRunning {
|
||||
saw_already_running = true;
|
||||
let mut accepted = None;
|
||||
let mut pending_snapshot = None;
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
match tokio::time::timeout(std::time::Duration::from_millis(100), rx.recv()).await {
|
||||
Ok(Ok(Event::SubmissionAccepted {
|
||||
submission_request_id,
|
||||
disposition,
|
||||
..
|
||||
})) if submission_request_id == "request-second" => accepted = Some(disposition),
|
||||
Ok(Ok(Event::PendingSubmissionsChanged { pending }))
|
||||
if pending.submissions.len() == 1 =>
|
||||
{
|
||||
pending_snapshot = Some(pending)
|
||||
}
|
||||
Ok(Ok(Event::Error { code, message })) if code == worker::ErrorCode::AlreadyRunning => {
|
||||
panic!("Submit was busy-rejected: {message}")
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if accepted.is_some() && pending_snapshot.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => break,
|
||||
}
|
||||
}
|
||||
|
||||
assert!(saw_already_running, "should see already_running error");
|
||||
assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued));
|
||||
let pending_snapshot = pending_snapshot.expect("pending snapshot");
|
||||
assert_eq!(pending_snapshot.submissions.len(), 1);
|
||||
handle.send(Method::Pause).await.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Paused).await;
|
||||
handle
|
||||
.send(Method::ContinuePending {
|
||||
expected_revision: pending_snapshot.revision,
|
||||
expected_head_id: pending_snapshot.head_id.expect("pending head"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let rejection = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
loop {
|
||||
if let Ok(Event::Error { code, message }) = rx.recv().await
|
||||
&& code == worker::ErrorCode::InvalidRequest
|
||||
&& message.contains("requires an idle Worker")
|
||||
{
|
||||
break message;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("paused ContinuePending rejection");
|
||||
assert!(rejection.contains("Resume or Cancel"));
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1365,7 +1474,8 @@ async fn run_with_paste_segment_inlines_content_and_emits_typed_user_message() {
|
||||
protocol::Segment::text(" thanks"),
|
||||
];
|
||||
handle
|
||||
.send(Method::Run {
|
||||
.send(Method::Submit {
|
||||
submission_request_id: protocol::new_submission_request_id(),
|
||||
input: segments.clone(),
|
||||
})
|
||||
.await
|
||||
@@ -1437,7 +1547,13 @@ async fn run_with_resolvable_file_ref_attaches_system_message_after_user() {
|
||||
path: "notes.md".into(),
|
||||
},
|
||||
];
|
||||
handle.send(Method::Run { input: segments }).await.unwrap();
|
||||
handle
|
||||
.send(Method::Submit {
|
||||
submission_request_id: protocol::new_submission_request_id(),
|
||||
input: segments,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Wait for the turn to complete.
|
||||
let mut rx = handle.subscribe();
|
||||
@@ -1485,7 +1601,8 @@ async fn run_with_file_ref_uses_manifest_file_upload_limit() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
handle
|
||||
.send(Method::Run {
|
||||
.send(Method::Submit {
|
||||
submission_request_id: protocol::new_submission_request_id(),
|
||||
input: vec![protocol::Segment::FileRef {
|
||||
path: "long.txt".into(),
|
||||
}],
|
||||
@@ -1538,7 +1655,13 @@ async fn run_with_unresolved_segment_emits_alert_and_placeholder() {
|
||||
path: "src/lib.rs".into(),
|
||||
},
|
||||
];
|
||||
handle.send(Method::Run { input: segments }).await.unwrap();
|
||||
handle
|
||||
.send(Method::Submit {
|
||||
submission_request_id: protocol::new_submission_request_id(),
|
||||
input: segments,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
let mut saw_alert_for_file_ref = false;
|
||||
@@ -1586,6 +1709,7 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() {
|
||||
|
||||
handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message: "turn finished".into(),
|
||||
auto_run: true,
|
||||
})
|
||||
@@ -1626,6 +1750,19 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() {
|
||||
saw_notify_in_mirror,
|
||||
"Method::Notify should commit a SystemItem::Notification entry; mirror = {entries:?}"
|
||||
);
|
||||
let queue_checkpoint_is_atomic = entries.iter().any(|entry| match entry {
|
||||
LogEntry::AnnotatedSystemItem { extensions, .. } => extensions.iter().any(|extension| {
|
||||
extension.domain == "worker.pending_activations.v1"
|
||||
&& extension.payload["pending_notifications"]
|
||||
.as_array()
|
||||
.is_some_and(Vec::is_empty)
|
||||
}),
|
||||
_ => false,
|
||||
});
|
||||
assert!(
|
||||
queue_checkpoint_is_atomic,
|
||||
"notification history and queue claim must share one log entry"
|
||||
);
|
||||
|
||||
// Exactly one request was made; it must contain the formatted
|
||||
// notification as one of the items (committed to history by
|
||||
@@ -1671,14 +1808,18 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
|
||||
let client_for_assert = client.clone();
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
let notification_request_id = protocol::new_submission_request_id();
|
||||
|
||||
for _ in 0..2 {
|
||||
handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: notification_request_id.clone(),
|
||||
message: "progress snapshot".into(),
|
||||
auto_run: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
||||
@@ -1687,7 +1828,13 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
|
||||
"weak Notify must not stage RunForNotification while idle"
|
||||
);
|
||||
|
||||
handle.send(Method::run_text("continue")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"continue",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
loop {
|
||||
if !client_for_assert.captured_requests().is_empty() {
|
||||
@@ -1867,9 +2014,16 @@ async fn notify_while_running_does_not_emit_already_running_error() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::run_text("start")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"start",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message: "ping".into(),
|
||||
auto_run: true,
|
||||
})
|
||||
@@ -1902,6 +2056,66 @@ async fn notify_while_running_does_not_emit_already_running_error() {
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn weak_notify_while_running_is_deduped_and_survives_until_next_submit() {
|
||||
let client = MockClient::sequential(vec![
|
||||
MockResponse::Hang(Vec::new()),
|
||||
MockResponse::Complete(simple_text_events()),
|
||||
]);
|
||||
let client_for_assert = client.clone();
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"first",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
|
||||
let notification_request_id = protocol::new_submission_request_id();
|
||||
for _ in 0..2 {
|
||||
handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: notification_request_id.clone(),
|
||||
message: "durable weak notice".into(),
|
||||
auto_run: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
handle.send(Method::Cancel).await.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
|
||||
let mut rx = handle.subscribe();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"second",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
loop {
|
||||
if matches!(rx.recv().await, Ok(Event::TurnEnd { .. })) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second submit completes");
|
||||
|
||||
let requests = client_for_assert.captured_requests();
|
||||
let notice_count = requests[1]
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|item| item.as_text())
|
||||
.filter(|text| text.contains("durable weak notice"))
|
||||
.count();
|
||||
assert_eq!(notice_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn status_json_reflects_worker_name() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
@@ -1936,7 +2150,13 @@ async fn socket_run_receives_events() {
|
||||
let mut writer = JsonLineWriter::new(writer);
|
||||
|
||||
// Send run method via socket
|
||||
writer.write(&Method::run_text("Hello")).await.unwrap();
|
||||
writer
|
||||
.write(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Collect events
|
||||
let mut saw_turn_start = false;
|
||||
@@ -2243,7 +2463,13 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::run_text("hello")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"hello",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Wait for the partial text_delta to confirm the first stream is
|
||||
// live before we pause.
|
||||
@@ -2332,7 +2558,7 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
assert!(!has_tool_call, "no orphan tool_call in history");
|
||||
}
|
||||
|
||||
/// Paused with an orphan `tool_use` in history + a fresh `Method::Run`
|
||||
/// Paused with an orphan `tool_use` in history + a fresh `Method::Submit`
|
||||
/// must produce a wire-valid next LLM request: the orphan is closed
|
||||
/// with a synthetic `tool_result`, a system note is inserted, and the
|
||||
/// new user input is appended.
|
||||
@@ -2369,7 +2595,13 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::run_text("first")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"first",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Wait for ToolCallDone — the ToolCall is committed to history
|
||||
// right before the Engine enters tool execution and pends.
|
||||
@@ -2400,7 +2632,13 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
|
||||
// `last_run_interrupted` and runs its interrupt-prep step, which
|
||||
// closes the orphan + injects a system note before the fresh user
|
||||
// message.
|
||||
handle.send(Method::run_text("new request")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"new request",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
e,
|
||||
@@ -2531,7 +2769,13 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::run_text("first")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"first",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
e,
|
||||
@@ -2599,7 +2843,10 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
|
||||
);
|
||||
|
||||
handle
|
||||
.send(Method::run_text("fresh request"))
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"fresh request",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
@@ -2688,7 +2935,13 @@ async fn empty_turn_cancel_rolls_back_submit_entries_and_emits_signal() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::run_text("rollback me")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"rollback me",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
handle.send(Method::Cancel).await.unwrap();
|
||||
|
||||
@@ -2721,7 +2974,10 @@ async fn empty_turn_pause_rolls_back_and_snapshot_does_not_restore_input() {
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle
|
||||
.send(Method::run_text("pause rollback"))
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"pause rollback",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
@@ -2755,7 +3011,13 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() {
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::run_text("first kept")).await.unwrap();
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"first kept",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
e,
|
||||
@@ -2769,7 +3031,10 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() {
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
|
||||
handle
|
||||
.send(Method::run_text("second rolled back"))
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"second rolled back",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
@@ -2816,7 +3081,10 @@ async fn pause_after_assistant_token_does_not_rollback() {
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle
|
||||
.send(Method::run_text("keep this turn"))
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"keep this turn",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
|
||||
@@ -538,7 +538,7 @@ fn initial_worker_input(segments: &[Segment]) -> Option<EmbeddedWorkerInput> {
|
||||
Some(EmbeddedWorkerInput {
|
||||
kind: EmbeddedWorkerInputKind::User,
|
||||
content: Segment::flatten_to_text(segments),
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: Some(segments.to_vec()),
|
||||
})
|
||||
}
|
||||
@@ -2706,7 +2706,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
||||
},
|
||||
content: request.content,
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: request.segments,
|
||||
};
|
||||
match self.runtime.send_input(&worker_ref, input) {
|
||||
@@ -3934,7 +3934,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
||||
},
|
||||
content: request.content,
|
||||
submission_id: None,
|
||||
submission_request_id: None,
|
||||
segments: request.segments,
|
||||
};
|
||||
match self.post_json::<_, RuntimeHttpWorkerInputResponse>(
|
||||
@@ -5195,7 +5195,7 @@ mod tests {
|
||||
"missing test context",
|
||||
);
|
||||
};
|
||||
let submission_id = input.submission_id.clone();
|
||||
let submission_request_id = input.submission_request_id.clone();
|
||||
let content = input.content;
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
@@ -5212,11 +5212,13 @@ mod tests {
|
||||
status: protocol::WorkerStatus::Idle,
|
||||
});
|
||||
});
|
||||
if let Some(submission_id) = submission_id {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted_input_committed(
|
||||
if let Some(submission_request_id) = submission_request_id {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted_submission(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
submission_id,
|
||||
submission_request_id,
|
||||
uuid::Uuid::now_v7().to_string(),
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
} else {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||
|
||||
@@ -32,11 +32,13 @@ impl WorkerExecutionBackend for TestExecutionBackend {
|
||||
_handle: &WorkerExecutionHandle,
|
||||
input: worker_runtime::interaction::WorkerInput,
|
||||
) -> WorkerExecutionResult {
|
||||
if let Some(submission_id) = input.submission_id {
|
||||
WorkerExecutionResult::accepted_input_committed(
|
||||
if let Some(submission_request_id) = input.submission_request_id {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
submission_id,
|
||||
submission_request_id,
|
||||
uuid::Uuid::now_v7().to_string(),
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
} else {
|
||||
WorkerExecutionResult::accepted(
|
||||
|
||||
@@ -48,8 +48,7 @@ use workdir::http::{
|
||||
};
|
||||
use workdir::workspace::{
|
||||
MaterializerKind, WorkingDirectoryCleanupTarget, WorkingDirectoryOccupancy,
|
||||
WorkingDirectoryStatusKind, WorkingDirectorySummary, WorkspaceWorkdirSessionFence,
|
||||
WorkspaceWorkdirSessionOperationRequest,
|
||||
WorkingDirectoryStatusKind, WorkingDirectorySummary, WorkspaceWorkdirSessionOperationRequest,
|
||||
};
|
||||
use workdir::{CommandHandle, WorkdirSessionHandle};
|
||||
use worker::feature::builtin::{WorkerObservationSubject, WorkerObservationSubjectRef};
|
||||
@@ -355,7 +354,6 @@ static EMBEDDED_RUNTIME_REQUEST_IDENTITY: std::sync::LazyLock<
|
||||
struct WorkdirCommandSession {
|
||||
source: WorkdirSessionHandle,
|
||||
provider_handle: CommandHandle,
|
||||
delegations: Vec<workdir::WorkdirDelegationRequest>,
|
||||
}
|
||||
|
||||
enum RegisteredWorkdirSession {
|
||||
@@ -398,7 +396,6 @@ impl WorkdirSessionRegistry {
|
||||
worker: RuntimeWorkerRef,
|
||||
source: WorkdirSessionHandle,
|
||||
provider_handle: CommandHandle,
|
||||
delegations: Vec<workdir::WorkdirDelegationRequest>,
|
||||
) -> CommandHandle {
|
||||
let external_handle = loop {
|
||||
let candidate = CommandHandle(Uuid::now_v7().to_string());
|
||||
@@ -414,7 +411,6 @@ impl WorkdirSessionRegistry {
|
||||
WorkdirCommandSession {
|
||||
source,
|
||||
provider_handle,
|
||||
delegations,
|
||||
},
|
||||
);
|
||||
external_handle
|
||||
@@ -2653,10 +2649,6 @@ fn build_inner_router(api: WorkspaceApi) -> Router {
|
||||
post(scoped_attach_current_worker_workdir)
|
||||
.delete(scoped_detach_current_worker_workdir),
|
||||
)
|
||||
.route(
|
||||
"/api/w/{workspace_id}/workers/self/workdir-session/fence",
|
||||
get(scoped_current_worker_workdir_session_fence),
|
||||
)
|
||||
.route(
|
||||
"/api/w/{workspace_id}/workers/self/workdir-session/operations",
|
||||
post(scoped_execute_current_worker_workdir_operation),
|
||||
@@ -7412,46 +7404,11 @@ async fn scoped_detach_current_worker_workdir(
|
||||
}))
|
||||
}
|
||||
|
||||
async fn scoped_current_worker_workdir_session_fence(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedWorkspacePath>,
|
||||
headers: HeaderMap,
|
||||
) -> ApiResult<Json<WorkspaceWorkdirSessionFence>> {
|
||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
||||
let worker = current_worker_identity(&api, &path.workspace_id, &headers)?;
|
||||
let session_lock = current_worker_session_lock(&api, &worker);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
let link = current_worker_active_attachment(&api, &worker)?;
|
||||
Ok(Json(WorkspaceWorkdirSessionFence {
|
||||
value: current_worker_workdir_session_fence(&link),
|
||||
}))
|
||||
}
|
||||
|
||||
fn current_worker_workdir_session_fence(link: &WorkerWorkdirLinkRecord) -> String {
|
||||
format!("v1:{}\0{}", link.workdir_id, link.linked_at)
|
||||
}
|
||||
|
||||
fn validate_current_worker_workdir_session_fence(
|
||||
link: &WorkerWorkdirLinkRecord,
|
||||
expected: Option<&str>,
|
||||
) -> Result<()> {
|
||||
if expected.is_some_and(|expected| expected != current_worker_workdir_session_fence(link)) {
|
||||
Err(Error::WorkdirAttachmentConflict(
|
||||
"delegated Workdir session attachment changed".to_string(),
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn validated_current_worker_attachment(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
expected_session_fence: Option<&str>,
|
||||
) -> ApiResult<WorkerWorkdirLinkRecord> {
|
||||
let link = current_worker_active_attachment(api, worker)?;
|
||||
validate_current_worker_workdir_session_fence(&link, expected_session_fence)?;
|
||||
Ok(link)
|
||||
current_worker_active_attachment(api, worker)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -7506,23 +7463,13 @@ async fn scoped_execute_current_worker_workdir_operation(
|
||||
) -> std::result::Result<Json<WorkdirSessionOperationResult>, WorkdirOperationApiError> {
|
||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
||||
let worker = current_worker_identity(&api, &path.workspace_id, &headers)?;
|
||||
let expected_session_fence = request.expected_session_fence;
|
||||
let delegations = request.delegations;
|
||||
let result = match request.operation {
|
||||
WorkdirSessionOperation::CommandStart(command) => {
|
||||
let session_lock = current_worker_session_lock(&api, &worker);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
let link = validated_current_worker_attachment(
|
||||
&api,
|
||||
&worker,
|
||||
expected_session_fence.as_deref(),
|
||||
)?;
|
||||
let link = validated_current_worker_attachment(&api, &worker)?;
|
||||
let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?;
|
||||
let applied =
|
||||
apply_current_worker_delegations(&worker, source.clone(), delegations.clone())
|
||||
.await?;
|
||||
let provider_handle = applied
|
||||
.scoped_session
|
||||
let provider_handle = source
|
||||
.start_command(command)
|
||||
.await
|
||||
.map_err(|error| current_worker_workdir_operation_error(&worker, error))?;
|
||||
@@ -7541,58 +7488,32 @@ async fn scoped_execute_current_worker_workdir_operation(
|
||||
.workdir_sessions
|
||||
.lock()
|
||||
.expect("Workdir session registry lock poisoned")
|
||||
.register_command(
|
||||
worker.clone(),
|
||||
registered_source,
|
||||
provider_handle,
|
||||
delegations,
|
||||
);
|
||||
.register_command(worker.clone(), registered_source, provider_handle);
|
||||
WorkdirSessionOperationResult::CommandStart(external_handle)
|
||||
}
|
||||
WorkdirSessionOperation::CommandStatus(external_handle) => {
|
||||
let (session, provider_handle) = current_worker_command_session(
|
||||
&api,
|
||||
&worker,
|
||||
&external_handle,
|
||||
&delegations,
|
||||
expected_session_fence.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
let (session, provider_handle) =
|
||||
current_worker_command_session(&api, &worker, &external_handle)?;
|
||||
session
|
||||
.scoped_session
|
||||
.command_status(provider_handle)
|
||||
.await
|
||||
.map(WorkdirSessionOperationResult::CommandStatus)
|
||||
.map_err(|error| current_worker_workdir_operation_error(&worker, error))?
|
||||
}
|
||||
WorkdirSessionOperation::CommandOutput(mut output) => {
|
||||
let (session, provider_handle) = current_worker_command_session(
|
||||
&api,
|
||||
&worker,
|
||||
&output.handle,
|
||||
&delegations,
|
||||
expected_session_fence.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
let (session, provider_handle) =
|
||||
current_worker_command_session(&api, &worker, &output.handle)?;
|
||||
output.handle = provider_handle;
|
||||
session
|
||||
.scoped_session
|
||||
.command_output(output)
|
||||
.await
|
||||
.map(WorkdirSessionOperationResult::CommandOutput)
|
||||
.map_err(|error| current_worker_workdir_operation_error(&worker, error))?
|
||||
}
|
||||
WorkdirSessionOperation::CommandCancel(external_handle) => {
|
||||
let (session, provider_handle) = current_worker_command_session(
|
||||
&api,
|
||||
&worker,
|
||||
&external_handle,
|
||||
&delegations,
|
||||
expected_session_fence.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
let (session, provider_handle) =
|
||||
current_worker_command_session(&api, &worker, &external_handle)?;
|
||||
session
|
||||
.scoped_session
|
||||
.cancel_command(provider_handle)
|
||||
.await
|
||||
.map(|()| WorkdirSessionOperationResult::CommandCancel)
|
||||
@@ -7607,14 +7528,9 @@ async fn scoped_execute_current_worker_workdir_operation(
|
||||
| WorkdirSessionOperation::Grep(_)) => {
|
||||
let session_lock = current_worker_session_lock(&api, &worker);
|
||||
let _session_guard = session_lock.lock().await;
|
||||
let link = validated_current_worker_attachment(
|
||||
&api,
|
||||
&worker,
|
||||
expected_session_fence.as_deref(),
|
||||
)?;
|
||||
let link = validated_current_worker_attachment(&api, &worker)?;
|
||||
let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?;
|
||||
let applied = apply_current_worker_delegations(&worker, source, delegations).await?;
|
||||
execute_workdir_session_operation(&applied.scoped_session, operation)
|
||||
execute_workdir_session_operation(&source, operation)
|
||||
.await
|
||||
.map_err(|error| current_worker_workdir_operation_error(&worker, error))?
|
||||
}
|
||||
@@ -7622,29 +7538,12 @@ async fn scoped_execute_current_worker_workdir_operation(
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
async fn apply_current_worker_delegations(
|
||||
worker: &RuntimeWorkerRef,
|
||||
source: WorkdirSessionHandle,
|
||||
delegations: Vec<workdir::WorkdirDelegationRequest>,
|
||||
) -> Result<workdir::AppliedWorkdirDelegation> {
|
||||
workdir::apply_delegation_chain(source, delegations)
|
||||
.await
|
||||
.map_err(|error| Error::RuntimeOperationFailed {
|
||||
runtime_id: worker.runtime_id.clone(),
|
||||
code: "workdir_session_delegation_failed".to_string(),
|
||||
message: error.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn current_worker_command_session(
|
||||
fn current_worker_command_session(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
external_handle: &CommandHandle,
|
||||
delegations: &[workdir::WorkdirDelegationRequest],
|
||||
expected_session_fence: Option<&str>,
|
||||
) -> std::result::Result<(workdir::AppliedWorkdirDelegation, CommandHandle), WorkdirOperationApiError>
|
||||
{
|
||||
let _link = validated_current_worker_attachment(api, worker, expected_session_fence)?;
|
||||
) -> std::result::Result<(WorkdirSessionHandle, CommandHandle), WorkdirOperationApiError> {
|
||||
let _link = validated_current_worker_attachment(api, worker)?;
|
||||
let command = api
|
||||
.workdir_sessions
|
||||
.lock()
|
||||
@@ -7656,15 +7555,7 @@ async fn current_worker_command_session(
|
||||
workdir::WorkdirError::UnknownCommand(external_handle.0.clone()),
|
||||
))
|
||||
})?;
|
||||
if command.delegations != delegations {
|
||||
return Err(Error::WorkdirAttachmentConflict(
|
||||
"command lifecycle delegation differs from CommandStart".to_string(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let session =
|
||||
apply_current_worker_delegations(worker, command.source, command.delegations).await?;
|
||||
Ok((session, command.provider_handle))
|
||||
Ok((command.source, command.provider_handle))
|
||||
}
|
||||
|
||||
fn current_worker_workdir_operation_error(
|
||||
@@ -8606,13 +8497,15 @@ async fn scoped_list_runtimes(
|
||||
|
||||
async fn scoped_workspace_protocol_ws(
|
||||
State(api): State<WorkspaceApi>,
|
||||
Extension(actor): Extension<RequestActor>,
|
||||
AxumPath(workspace_id): AxumPath<String>,
|
||||
ws: axum::extract::ws::WebSocketUpgrade,
|
||||
) -> std::result::Result<Response, Response> {
|
||||
validate_workspace_scope(&api, &workspace_id).map_err(|error| error.into_response())?;
|
||||
let input_source = authenticated_browser_input_source(&actor);
|
||||
Ok(ws
|
||||
.on_upgrade(move |socket| {
|
||||
crate::workspace_subscription::serve_workspace_subscription(api, socket)
|
||||
crate::workspace_subscription::serve_workspace_subscription(api, socket, input_source)
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
@@ -9318,7 +9211,7 @@ async fn scoped_capture_worker_observation_session(
|
||||
return Err(ApiError::from(Error::UnknownWorker { worker: target }));
|
||||
}
|
||||
|
||||
let mut connection = connect_workspace_worker_protocol(&api, &target).await?;
|
||||
let mut connection = connect_workspace_worker_protocol(&api, &target, None).await?;
|
||||
let event = tokio::time::timeout(std::time::Duration::from_secs(10), connection.events.recv())
|
||||
.await
|
||||
.map_err(|_| {
|
||||
@@ -11741,6 +11634,7 @@ async fn scoped_cancel_runtime_worker(
|
||||
async fn scoped_worker_protocol_ws(
|
||||
ws: WebSocketUpgrade,
|
||||
State(api): State<WorkspaceApi>,
|
||||
Extension(actor): Extension<RequestActor>,
|
||||
AxumPath(path): AxumPath<ScopedRuntimeWorkerPath>,
|
||||
) -> Response {
|
||||
if let Err(err) = validate_workspace_scope(&api, &path.workspace_id) {
|
||||
@@ -11748,6 +11642,7 @@ async fn scoped_worker_protocol_ws(
|
||||
}
|
||||
worker_protocol_ws(
|
||||
State(api),
|
||||
Extension(actor),
|
||||
AxumPath((path.worker.runtime_id, path.worker.worker_id)),
|
||||
ws,
|
||||
)
|
||||
@@ -14140,8 +14035,45 @@ async fn cancel_runtime_worker(
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
fn authenticated_browser_input_source(actor: &RequestActor) -> protocol::AuthenticatedInputSource {
|
||||
protocol::AuthenticatedInputSource::Account {
|
||||
account_id: actor.account_id.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn authorize_browser_worker_method(
|
||||
method: protocol::Method,
|
||||
source: &protocol::AuthenticatedInputSource,
|
||||
) -> std::result::Result<protocol::Method, &'static str> {
|
||||
match method {
|
||||
protocol::Method::Submit {
|
||||
submission_request_id,
|
||||
input,
|
||||
} => Ok(protocol::Method::SubmitTracked {
|
||||
submission_request_id,
|
||||
input,
|
||||
source: source.clone(),
|
||||
}),
|
||||
protocol::Method::Notify {
|
||||
notification_request_id,
|
||||
message,
|
||||
auto_run,
|
||||
} => Ok(protocol::Method::NotifyTracked {
|
||||
notification_request_id,
|
||||
message,
|
||||
auto_run,
|
||||
source: source.clone(),
|
||||
}),
|
||||
protocol::Method::SubmitTracked { .. } | protocol::Method::NotifyTracked { .. } => {
|
||||
Err("authenticated Worker input source is server-owned")
|
||||
}
|
||||
other => Ok(other),
|
||||
}
|
||||
}
|
||||
|
||||
async fn worker_protocol_ws(
|
||||
State(api): State<WorkspaceApi>,
|
||||
Extension(actor): Extension<RequestActor>,
|
||||
AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>,
|
||||
ws: WebSocketUpgrade,
|
||||
) -> impl IntoResponse {
|
||||
@@ -14165,7 +14097,8 @@ async fn worker_protocol_ws(
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
ws.on_upgrade(move |socket| worker_protocol_ws_session(source, socket))
|
||||
let input_source = authenticated_browser_input_source(&actor);
|
||||
ws.on_upgrade(move |socket| worker_protocol_ws_session(source, socket, input_source))
|
||||
}
|
||||
|
||||
pub(crate) struct WorkspaceWorkerProtocolConnection {
|
||||
@@ -14176,6 +14109,7 @@ pub(crate) struct WorkspaceWorkerProtocolConnection {
|
||||
pub(crate) async fn connect_workspace_worker_protocol(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
input_source: Option<&protocol::AuthenticatedInputSource>,
|
||||
) -> Result<WorkspaceWorkerProtocolConnection> {
|
||||
let source = match api.observation_proxy.source(worker) {
|
||||
Ok(source) => source,
|
||||
@@ -14192,15 +14126,39 @@ pub(crate) async fn connect_workspace_worker_protocol(
|
||||
}
|
||||
};
|
||||
match source {
|
||||
RuntimeObservationSource::RemoteWs(config) => connect_remote_worker_protocol(config).await,
|
||||
RuntimeObservationSource::RemoteWs(config) => {
|
||||
connect_remote_worker_protocol(config, input_source).await
|
||||
}
|
||||
RuntimeObservationSource::Embedded(source) => {
|
||||
connect_embedded_worker_protocol(source).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_authenticated_input_source_header(
|
||||
headers: &mut HeaderMap,
|
||||
input_source: Option<&protocol::AuthenticatedInputSource>,
|
||||
) -> Result<()> {
|
||||
let Some(input_source) = input_source else {
|
||||
return Ok(());
|
||||
};
|
||||
let protocol::AuthenticatedInputSource::Account { account_id } = input_source else {
|
||||
return Err(Error::Config(
|
||||
"remote Worker protocol transport supports only Account input source".into(),
|
||||
));
|
||||
};
|
||||
headers.insert(
|
||||
protocol::AUTHENTICATED_ACCOUNT_ID_HEADER,
|
||||
account_id.parse().map_err(|error| {
|
||||
Error::Config(format!("invalid authenticated Account identity: {error}"))
|
||||
})?,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn connect_remote_worker_protocol(
|
||||
config: RuntimeObservationSourceConfig,
|
||||
input_source: Option<&protocol::AuthenticatedInputSource>,
|
||||
) -> Result<WorkspaceWorkerProtocolConnection> {
|
||||
let mut request = config
|
||||
.endpoint
|
||||
@@ -14215,6 +14173,7 @@ async fn connect_remote_worker_protocol(
|
||||
})?,
|
||||
);
|
||||
}
|
||||
insert_authenticated_input_source_header(request.headers_mut(), input_source)?;
|
||||
let (socket, _) =
|
||||
connect_async(request)
|
||||
.await
|
||||
@@ -14292,13 +14251,17 @@ async fn connect_embedded_worker_protocol(
|
||||
Ok(WorkspaceWorkerProtocolConnection { methods, events })
|
||||
}
|
||||
|
||||
async fn worker_protocol_ws_session(source: RuntimeObservationSource, socket: WebSocket) {
|
||||
async fn worker_protocol_ws_session(
|
||||
source: RuntimeObservationSource,
|
||||
socket: WebSocket,
|
||||
input_source: protocol::AuthenticatedInputSource,
|
||||
) {
|
||||
match source {
|
||||
RuntimeObservationSource::RemoteWs(config) => {
|
||||
remote_worker_protocol_ws_session(config, socket).await;
|
||||
remote_worker_protocol_ws_session(config, socket, input_source).await;
|
||||
}
|
||||
RuntimeObservationSource::Embedded(source) => {
|
||||
embedded_worker_protocol_ws_session(source, socket).await;
|
||||
embedded_worker_protocol_ws_session(source, socket, input_source).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14306,6 +14269,7 @@ async fn worker_protocol_ws_session(source: RuntimeObservationSource, socket: We
|
||||
async fn remote_worker_protocol_ws_session(
|
||||
config: RuntimeObservationSourceConfig,
|
||||
socket: WebSocket,
|
||||
input_source: protocol::AuthenticatedInputSource,
|
||||
) {
|
||||
let mut request = match config.endpoint.clone().into_client_request() {
|
||||
Ok(request) => request,
|
||||
@@ -14333,6 +14297,16 @@ async fn remote_worker_protocol_ws_session(
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Err(error) =
|
||||
insert_authenticated_input_source_header(request.headers_mut(), Some(&input_source))
|
||||
{
|
||||
let mut socket = socket;
|
||||
let event = protocol_error_event(format!(
|
||||
"failed to build authenticated Account identity header: {error}"
|
||||
));
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
|
||||
let (upstream, _) = match connect_async(request).await {
|
||||
Ok(connection) => connection,
|
||||
@@ -14354,14 +14328,33 @@ async fn remote_worker_protocol_ws_session(
|
||||
inbound = client_stream.next() => {
|
||||
match inbound {
|
||||
Some(Ok(WsMessage::Text(text))) => {
|
||||
if upstream_sink.send(TungsteniteMessage::Text(text.to_string().into())).await.is_err() {
|
||||
let method = match protocol::stream::decode_method(text.as_ref()) {
|
||||
Ok(method) => match authorize_browser_worker_method(method, &input_source) {
|
||||
Ok(method) => method,
|
||||
Err(message) => {
|
||||
if let Ok(event) = protocol::stream::encode_event(&protocol_error_event(message)) {
|
||||
let _ = client_sink.send(WsMessage::Text(event.into())).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
},
|
||||
Err(error) => {
|
||||
if let Ok(event) = protocol::stream::encode_event(&protocol_error_event(error.to_string())) {
|
||||
let _ = client_sink.send(WsMessage::Text(event.into())).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
};
|
||||
let Ok(method) = protocol::stream::encode_method(&method) else { break };
|
||||
if upstream_sink.send(TungsteniteMessage::Text(method.into())).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Some(Ok(WsMessage::Binary(binary))) => {
|
||||
if upstream_sink.send(TungsteniteMessage::Binary(binary.to_vec().into())).await.is_err() {
|
||||
break;
|
||||
Some(Ok(WsMessage::Binary(_))) => {
|
||||
if let Ok(event) = protocol::stream::encode_event(&protocol_error_event("binary Worker methods are not accepted")) {
|
||||
let _ = client_sink.send(WsMessage::Text(event.into())).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
Some(Ok(WsMessage::Close(_))) | None => {
|
||||
let _ = upstream_sink.send(TungsteniteMessage::Close(None)).await;
|
||||
@@ -14417,6 +14410,7 @@ async fn remote_worker_protocol_ws_session(
|
||||
async fn embedded_worker_protocol_ws_session(
|
||||
source: crate::observation::EmbeddedRuntimeObservationSource,
|
||||
mut socket: WebSocket,
|
||||
input_source: protocol::AuthenticatedInputSource,
|
||||
) {
|
||||
let mut upstream = match RuntimeObservationClient::connect(&RuntimeObservationSource::Embedded(
|
||||
source.clone(),
|
||||
@@ -14436,6 +14430,7 @@ async fn embedded_worker_protocol_ws_session(
|
||||
inbound = socket.next() => {
|
||||
match inbound {
|
||||
Some(Ok(WsMessage::Text(text))) => match decode_method(&text) {
|
||||
Ok(method) => match authorize_browser_worker_method(method, &input_source) {
|
||||
Ok(method) => match source.runtime.send_protocol_method(&source.worker_ref, method) {
|
||||
Ok(events) => {
|
||||
for event in events {
|
||||
@@ -14451,9 +14446,16 @@ async fn embedded_worker_protocol_ws_session(
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(message) => {
|
||||
let event = protocol_error_event(message);
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
Err(error) => {
|
||||
let event =
|
||||
protocol_error_event(format!("malformed protocol method frame: {error}"));
|
||||
let event = protocol_error_event(format!(
|
||||
"malformed protocol method frame: {error}"
|
||||
));
|
||||
if !send_protocol_event(&mut socket, &event).await {
|
||||
return;
|
||||
}
|
||||
@@ -16593,6 +16595,48 @@ mod tests {
|
||||
&tail[..end]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_worker_methods_receive_server_owned_account_source() {
|
||||
let source = protocol::AuthenticatedInputSource::Account {
|
||||
account_id: "account-1".into(),
|
||||
};
|
||||
let method = authorize_browser_worker_method(
|
||||
protocol::Method::Submit {
|
||||
submission_request_id: "request-1".into(),
|
||||
input: vec![protocol::Segment::text("hello")],
|
||||
},
|
||||
&source,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
method,
|
||||
protocol::Method::SubmitTracked {
|
||||
source: protocol::AuthenticatedInputSource::Account { ref account_id },
|
||||
..
|
||||
} if account_id == "account-1"
|
||||
));
|
||||
assert!(authorize_browser_worker_method(method, &source).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_worker_protocol_header_preserves_authenticated_account_source() {
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_authenticated_input_source_header(
|
||||
&mut headers,
|
||||
Some(&protocol::AuthenticatedInputSource::Account {
|
||||
account_id: "account-1".into(),
|
||||
}),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
headers
|
||||
.get(protocol::AUTHENTICATED_ACCOUNT_ID_HEADER)
|
||||
.unwrap(),
|
||||
"account-1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_request_http_paths_observe_refs_through_runtime_provider_authority() {
|
||||
let source = include_str!("server.rs");
|
||||
@@ -16788,6 +16832,7 @@ mod tests {
|
||||
command: "printf ready; sleep 30".to_string(),
|
||||
timeout_secs: 60,
|
||||
output_limit: 4096,
|
||||
cwd: None,
|
||||
spill_dir: None,
|
||||
tool_call_id: Some("tool-call-command-session".to_string()),
|
||||
})
|
||||
@@ -16797,12 +16842,8 @@ mod tests {
|
||||
let mut registry = WorkdirSessionRegistry::default();
|
||||
registry.insert_attachment(worker.clone(), source.clone());
|
||||
let registered_source = registry.remove_attachment(&worker).unwrap();
|
||||
let external_handle = registry.register_command(
|
||||
worker.clone(),
|
||||
registered_source,
|
||||
provider_handle.clone(),
|
||||
Vec::new(),
|
||||
);
|
||||
let external_handle =
|
||||
registry.register_command(worker.clone(), registered_source, provider_handle.clone());
|
||||
assert_ne!(external_handle, provider_handle);
|
||||
|
||||
let refreshed: WorkdirSessionHandle = Arc::new(workdir::LocalWorkdirSession::new(
|
||||
@@ -18843,7 +18884,7 @@ mod tests {
|
||||
.get(handle.worker_ref())
|
||||
.cloned()
|
||||
.expect("execution context");
|
||||
let submission_id = input.submission_id.clone();
|
||||
let submission_request_id = input.submission_request_id.clone();
|
||||
let content = input.content.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(std::time::Duration::from_millis(25));
|
||||
@@ -18851,11 +18892,13 @@ mod tests {
|
||||
text: format!("server companion echoed: {content}"),
|
||||
});
|
||||
});
|
||||
if let Some(submission_id) = submission_id {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted_input_committed(
|
||||
if let Some(submission_request_id) = submission_request_id {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted_submission(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||
worker_runtime::execution::WorkerExecutionRunState::Idle,
|
||||
submission_id,
|
||||
submission_request_id,
|
||||
uuid::Uuid::now_v7().to_string(),
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
} else {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||
@@ -23747,30 +23790,6 @@ mod tests {
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delegated_workdir_session_fence_rejects_reattached_link() {
|
||||
let first = WorkerWorkdirLinkRecord {
|
||||
workspace_id: "workspace-a".to_string(),
|
||||
worker: workdir::workspace::RuntimeWorkerRef::new("runtime-a", "worker-a"),
|
||||
workdir_id: "workdir-a".to_string(),
|
||||
role: "primary".to_string(),
|
||||
linked_at: "2026-01-01T00:00:00Z".to_string(),
|
||||
unlinked_at: None,
|
||||
};
|
||||
let expected = current_worker_workdir_session_fence(&first);
|
||||
assert!(validate_current_worker_workdir_session_fence(&first, None).is_ok());
|
||||
assert!(validate_current_worker_workdir_session_fence(&first, Some(&expected)).is_ok());
|
||||
|
||||
let reattached = WorkerWorkdirLinkRecord {
|
||||
linked_at: "2026-01-01T00:00:01Z".to_string(),
|
||||
..first
|
||||
};
|
||||
assert!(matches!(
|
||||
validate_current_worker_workdir_session_fence(&reattached, Some(&expected)),
|
||||
Err(Error::WorkdirAttachmentConflict(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_workdir_session_proxy_executes_typed_operations() {
|
||||
use manifest::Scope;
|
||||
@@ -27638,6 +27657,16 @@ mod tests {
|
||||
(runtime, worker_ref, endpoint)
|
||||
}
|
||||
|
||||
fn test_browser_request_actor() -> RequestActor {
|
||||
RequestActor {
|
||||
user_id: "test-user".into(),
|
||||
account_id: format!("account-{TEST_WORKSPACE_ID}"),
|
||||
handle: "test".into(),
|
||||
display_name: "Test".into(),
|
||||
auth_method: ActorAuthMethod::BrowserSession,
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn_workspace_proxy(
|
||||
source: RuntimeObservationSourceConfig,
|
||||
) -> (String, tempfile::TempDir) {
|
||||
@@ -27656,11 +27685,8 @@ mod tests {
|
||||
.unwrap();
|
||||
let app_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let app_addr = app_listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(app_listener, build_inner_router(api))
|
||||
.await
|
||||
.unwrap()
|
||||
});
|
||||
let app = build_inner_router(api).layer(Extension(test_browser_request_actor()));
|
||||
tokio::spawn(async move { axum::serve(app_listener, app).await.unwrap() });
|
||||
(
|
||||
format!("ws://{app_addr}/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws"),
|
||||
dir,
|
||||
@@ -27672,7 +27698,8 @@ mod tests {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let app = build_inner_router(test_api(dir.path()).await);
|
||||
let app = build_inner_router(test_api(dir.path()).await)
|
||||
.layer(Extension(test_browser_request_actor()));
|
||||
let server = tokio::spawn(async move {
|
||||
let _ = axum::serve(listener, app).await;
|
||||
});
|
||||
@@ -27720,7 +27747,7 @@ mod tests {
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let app = build_inner_router(api);
|
||||
let app = build_inner_router(api).layer(Extension(test_browser_request_actor()));
|
||||
let server = tokio::spawn(async move {
|
||||
let _ = axum::serve(listener, app).await;
|
||||
});
|
||||
|
||||
@@ -11,7 +11,9 @@ use tokio::sync::mpsc;
|
||||
use worker_runtime::identity::RuntimeWorkerRef;
|
||||
|
||||
use crate::runtime_subscription::{BrokerSubscriptionEvent, RuntimeSubscriptionBroker};
|
||||
use crate::server::{WorkspaceApi, connect_workspace_worker_protocol};
|
||||
use crate::server::{
|
||||
WorkspaceApi, authorize_browser_worker_method, connect_workspace_worker_protocol,
|
||||
};
|
||||
use crate::store::WorkspaceResourceKind;
|
||||
|
||||
const OUTBOUND_CAPACITY: usize = 256;
|
||||
@@ -21,7 +23,11 @@ struct ActiveSubscription {
|
||||
methods: Option<mpsc::Sender<protocol::Method>>,
|
||||
}
|
||||
|
||||
pub(crate) async fn serve_workspace_subscription(api: WorkspaceApi, socket: WebSocket) {
|
||||
pub(crate) async fn serve_workspace_subscription(
|
||||
api: WorkspaceApi,
|
||||
socket: WebSocket,
|
||||
input_source: protocol::AuthenticatedInputSource,
|
||||
) {
|
||||
let broker = api.runtime_subscription_broker().clone();
|
||||
let (mut socket_sender, mut socket_receiver) = socket.split();
|
||||
let (control_outbound, mut control_receiver) = mpsc::channel::<WsMessage>(OUTBOUND_CAPACITY);
|
||||
@@ -85,7 +91,13 @@ pub(crate) async fn serve_workspace_subscription(api: WorkspaceApi, socket: WebS
|
||||
runtime_id: Some(runtime_id),
|
||||
} => {
|
||||
let worker = RuntimeWorkerRef::new(&runtime_id, worker_id.as_str());
|
||||
match connect_workspace_worker_protocol(&api, &worker).await {
|
||||
match connect_workspace_worker_protocol(
|
||||
&api,
|
||||
&worker,
|
||||
Some(&input_source),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(connection) => {
|
||||
let methods = connection.methods.clone();
|
||||
let task = tokio::spawn(run_worker_protocol(
|
||||
@@ -153,7 +165,12 @@ pub(crate) async fn serve_workspace_subscription(api: WorkspaceApi, socket: WebS
|
||||
else {
|
||||
break;
|
||||
};
|
||||
if methods.send(message.method).await.is_err() {
|
||||
let Ok(method) =
|
||||
authorize_browser_worker_method(message.method, &input_source)
|
||||
else {
|
||||
break;
|
||||
};
|
||||
if methods.send(method).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,11 +55,12 @@ Workspace Server schema migration v26 removes the legacy `flow_instances`, `flow
|
||||
|
||||
## Worker boundary
|
||||
|
||||
Flow invocation uses the normal Submit/Run segment vector rather than a Worker-create field:
|
||||
Flow invocation uses the normal Submit segment vector rather than a Worker-create field:
|
||||
|
||||
```json
|
||||
{
|
||||
"method": "run",
|
||||
"method": "submit",
|
||||
"submission_request_id": "018f4f15-5c41-7d3a-8a72-2e755bc71681",
|
||||
"input": [
|
||||
{ "kind": "flow", "selector": "builtin:coder-review" },
|
||||
{ "kind": "text", "content": "Ticket 00001... implementation" }
|
||||
@@ -69,7 +70,7 @@ Flow invocation uses the normal Submit/Run segment vector rather than a Worker-c
|
||||
|
||||
Runtime accepts exactly one Flow segment only when the resolved Profile enables `feature.flow` and a Workspace client is available. The Worker asks Workspace authority only for an immutable source snapshot, creates the instance locally, replaces the Flow segment with the entered state's instructions, and commits that runtime state atomically with the remaining Submit segments before LLM execution. A Worker with an active Flow rejects the duplicate input without changing its local state or events.
|
||||
|
||||
The generic model-facing `WorkerSpawn` accepts `initial_submit: Vec<Segment>` and routes them unchanged through the shared Workspace spawn request into Runtime `CreateWorkerRequest.initial_input`. It does not have a parallel `initial_text` or a role-specific `SpawnCoder` wrapper. Backend derives the flat content projection from the canonical segment vector, validates Flow shape before spawn, and includes the segment vector in lifecycle idempotency fingerprints. Runtime does not commit Worker creation or report spawn success merely because the initial Run method entered the Worker's in-memory channel: Runtime assigns the Submit an opaque id, the Worker commits that id as an extension on the same `UserInput` entry as any initial `FlowRuntimeState`, and the execution backend must return a matching typed input-commit acknowledgement. Restoring the same Worker never replays spawn initial segments.
|
||||
The generic model-facing `WorkerSpawn` accepts `initial_submit: Vec<Segment>` and routes them unchanged through the shared Workspace spawn request into Runtime `CreateWorkerRequest.initial_input`. It does not have a parallel `initial_text` or a role-specific `SpawnCoder` wrapper. Backend derives the flat content projection from the canonical segment vector, validates Flow shape before spawn, and includes the segment vector in lifecycle idempotency fingerprints. Runtime does not commit Worker creation or report spawn success merely because the initial Submit request entered the Worker's in-memory channel: Runtime assigns the Submit an opaque id, the Worker commits that id as an extension on the same `UserInput` entry as any initial `FlowRuntimeState`, and the execution backend must return a matching typed input-commit acknowledgement. Restoring the same Worker never replays spawn initial segments.
|
||||
|
||||
When an Orchestrator supplies `ticket_id` to generic `WorkerSpawn`, the Worker tool derives the assignment operation id from the durable tool-call id rather than accepting lifecycle authority from model input. The shared Workspace worker-create route projects that request into a Coder Ticket-role intent and atomically applies the existing queued-Ticket assignment operation only after Runtime has returned the input-commit acknowledgement. A spawn or pre-commit input failure therefore leaves the Ticket queued and unassigned.
|
||||
|
||||
|
||||
@@ -286,7 +286,7 @@ User triggers a Ticket action in yoi panel
|
||||
-> client Ticket role launcher reads .yoi/workspace.toml [ticket] settings
|
||||
-> launcher selects the role Profile
|
||||
-> launcher spawns the role Worker
|
||||
-> launcher sends Method::Run with Text segments
|
||||
-> launcher sends Method::Submit with Text segments
|
||||
-> launcher waits for run-acceptance evidence
|
||||
-> Dashboard reports success/failure
|
||||
```
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
- いくつかの async/socket tests はまだ fixed sleeps と process-wide environment mutation (`YOI_RUNTIME_DIR`, `YOI_HOME`, `XDG_RUNTIME_DIR`) に依存している。一部ファイルでは env-changing tests を guards で serialize しているが、fixed timing と global env は高負荷 CI や parallel execution 下で flakiness risk のまま。
|
||||
- Real provider wire behavior は意図的に `pod` の外側だが、crate と streaming edge cases の interaction はまだ大部分が mock されている。tests は重要な `Worker` outcomes をカバーしているが、malformed/partial provider streams を real provider adapter 経由では exercise していない。
|
||||
- Prompt tests は数が多く有用だが、一部は behavior-coupled というより prose-coupled である。critical safety wording に対して、その文字列を意図的に stable contract として扱う場合だけこれは許容できる。そうでなければ maintenance noise になる。
|
||||
- startup profile resolution、socket server、`Method::Run`、session persistence、shutdown、restore を跨ぐ full lifecycle integration は slice ごとにしかカバーされておらず、1 つの scenario としてはカバーされていない。
|
||||
- startup profile resolution、socket server、`Method::Submit`、session persistence、shutdown、restore を跨ぐ full lifecycle integration は slice ごとにしかカバーされておらず、1 つの scenario としてはカバーされていない。
|
||||
|
||||
## 追加を提案するもの
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
};
|
||||
|
||||
review = {
|
||||
instructions = "Use the current Ticket Merge Request as review authority. Call `ShowMergeRequest` and confirm its source selector resolves to exact committed implementation HEAD, then spawn one actual direct-child SubWorker with profile builtin:reviewer, write scope for Workdir inspection and command validation, and only the Ticket id in the structured review handoff. The trusted spawn layer records `ReviewRequested` with the exact source ref and injects review capability; do not place commit/ref identity, capability material, or a prewritten verdict in model input. The child must commit `ReviewMergeRequest`; prose output and Worker observation are not approval authority. After the structured result for the exact current source ref exists, request a Flow transition.";
|
||||
instructions = "Use the current Ticket Merge Request as review authority. Call `ShowMergeRequest` and confirm its source selector resolves to exact committed implementation HEAD, then spawn one actual direct-child SubWorker with profile builtin:reviewer, write scope plus an explicit command grant for Workdir inspection and command validation, and only the Ticket id in the structured review handoff. The trusted spawn layer records `ReviewRequested` with the exact source ref and injects review capability; do not place commit/ref identity, capability material, or a prewritten verdict in model input. The child must commit `ReviewMergeRequest`; prose output and Worker observation are not approval authority. After the structured result for the exact current source ref exists, request a Flow transition.";
|
||||
transitions = {
|
||||
approved = {
|
||||
target = "complete";
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
Spawn a parent-owned Internal SubWorker session to split context for a delegated task. The parent Worker's write scope is reduced by the scope passed here; the Internal SubWorker starts running `task` immediately without creating a Runtime Worker record, OS process, PID, or Unix socket. It remains available for follow-up turns until explicitly stopped or its parent exits.
|
||||
|
||||
Optional `cwd`: when provided, the spawned SubWorker's tool default working directory only. It must be an absolute existing directory covered by the child's delegated readable scope, and it does not change workspace/Profile/memory/Ticket roots or grant authority. `name` must be unique among this Worker's direct children.
|
||||
Optional `cwd`: when provided, the spawned SubWorker's tool default working directory only. It must be a Workdir-relative existing directory covered by the child's readable scope, and it does not change workspace/Profile/memory/Ticket roots or grant authority. `name` must be unique among this Worker's direct children.
|
||||
|
||||
Profile selection: `profile` may be omitted or set to `default` to use the effective child default profile, set to `inherit` to derive reusable child configuration from this Worker, or set to one of the registry selectors below. Raw/path profile selectors are not accepted by SubWorkerSpawn. `scope` is always the only delegated filesystem capability; profile scope is replaced by the explicit SubWorkerSpawn scope.
|
||||
Profile selection: `profile` may be omitted or set to `default` to use the effective child default profile, set to `inherit` to derive reusable child configuration from this Worker, or set to one of the registry selectors below. Raw/path profile selectors are not accepted by SubWorkerSpawn. `scope` is the child's only filesystem capability and replaces profile scope. `command` is a separate explicit grant, defaults to false, and is accepted only with a writable scope; writable scope alone does not grant command execution.
|
||||
|
||||
Default profile: {{ default_profile }}
|
||||
Special selector: inherit — derive reusable model/worker/tool policy from the spawner while replacing worker.name and scope.
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"dev": "deno run -A npm:vite@7.2.7 dev",
|
||||
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
||||
"check": "deno run -A npm:@sveltejs/kit@2.49.4 sync && deno run -A npm:svelte-check@4.3.4 --tsconfig ./tsconfig.json",
|
||||
"test": "deno test --allow-read=src,test,tests --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts src/lib/workspace/auth/model.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/markdown.test.ts test/console/ansi.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-launch.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/ui.test.ts",
|
||||
"test": "deno test --allow-read=src,test,tests --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts src/lib/workspace/auth/model.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/markdown.test.ts test/console/ansi.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-launch.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/ui.test.ts",
|
||||
"build": "deno run -A npm:vite@7.2.7 build",
|
||||
"preview": "deno run -A npm:vite@7.2.7 preview"
|
||||
},
|
||||
|
||||
@@ -103,7 +103,13 @@ entry_id: string,
|
||||
*/
|
||||
timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array<string>, } & ({ "kind": "user_input", segments: Array<Segment>, } | { "kind": "message", role: SessionMessageRole, content: Array<SessionContentPart>, } | { "kind": "tool_call", call_id: string, name: string, arguments: string, } | { "kind": "tool_result", call_id: string, summary: string, content?: string | null, is_error: boolean, attachments?: Array<SessionToolAttachment>, } | { "kind": "system_item", item_kind: string, content: string, data?: unknown, } | { "kind": "run_error", message: string, });
|
||||
|
||||
export type SessionSnapshot = { entries: Array<SessionSnapshotEntry>, };
|
||||
export type PendingSubmissionSummary = { submission_id: string, accepted_at_ms: number, segment_count: number, byte_len: number, };
|
||||
|
||||
export type PendingSubmissionsSnapshot = { revision: number, notification_count: number, head_id: string | null, submissions: Array<PendingSubmissionSummary>, };
|
||||
|
||||
export type SubmissionDisposition = "started" | "queued";
|
||||
|
||||
export type SessionSnapshot = { pending_submissions: PendingSubmissionsSnapshot, entries: Array<SessionSnapshotEntry>, };
|
||||
|
||||
export type InternalWorkerKind = "sub_worker" | { "service": { kind: string, } };
|
||||
|
||||
@@ -225,9 +231,9 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip
|
||||
|
||||
export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod });
|
||||
|
||||
export type Method = { "method": "run", "params": { input: Array<Segment>, } } | { "method": "notify", "params": { message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
|
||||
export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
|
||||
|
||||
export type Event = { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number,
|
||||
export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number,
|
||||
/**
|
||||
* The attempt that just failed. 1 origin.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
declare const Deno: {
|
||||
test(name: string, fn: () => void): void;
|
||||
};
|
||||
|
||||
import {
|
||||
canDeliverComposerDraft,
|
||||
sendComposerDelivery,
|
||||
} from "./composer-delivery.ts";
|
||||
|
||||
function assertEquals(actual: unknown, expected: unknown): void {
|
||||
if (actual !== expected) {
|
||||
throw new Error(`Expected ${String(expected)}, got ${String(actual)}`);
|
||||
}
|
||||
}
|
||||
|
||||
const base = {
|
||||
protocolOpen: true,
|
||||
sending: false,
|
||||
hasText: true,
|
||||
hasAttachments: false,
|
||||
};
|
||||
|
||||
Deno.test("running Composer enables Queue Submit and Notify but not immediate Submit", () => {
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "queue",
|
||||
workerState: "running",
|
||||
}),
|
||||
true,
|
||||
);
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "notify",
|
||||
workerState: "running",
|
||||
}),
|
||||
true,
|
||||
);
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "submit",
|
||||
workerState: "running",
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("running Queue Submit and Notify dispatch their protocol methods", () => {
|
||||
const sent: string[] = [];
|
||||
assertEquals(
|
||||
sendComposerDelivery(
|
||||
{ ...base, delivery: "queue", workerState: "running" },
|
||||
"submit",
|
||||
(method) => sent.push(method),
|
||||
),
|
||||
true,
|
||||
);
|
||||
assertEquals(
|
||||
sendComposerDelivery(
|
||||
{ ...base, delivery: "notify", workerState: "running" },
|
||||
"notify",
|
||||
(method) => sent.push(method),
|
||||
),
|
||||
true,
|
||||
);
|
||||
assertEquals(sent.join(","), "submit,notify");
|
||||
});
|
||||
|
||||
Deno.test("idle Composer enables only immediate Submit", () => {
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "submit",
|
||||
workerState: "idle",
|
||||
}),
|
||||
true,
|
||||
);
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "queue",
|
||||
workerState: "idle",
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "notify",
|
||||
workerState: "idle",
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("running delivery remains fenced by protocol, send state, and payload kind", () => {
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "queue",
|
||||
workerState: "running",
|
||||
protocolOpen: false,
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "notify",
|
||||
workerState: "running",
|
||||
sending: true,
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "notify",
|
||||
workerState: "running",
|
||||
hasAttachments: true,
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assertEquals(
|
||||
canDeliverComposerDraft({
|
||||
...base,
|
||||
delivery: "queue",
|
||||
workerState: "running",
|
||||
hasText: false,
|
||||
hasAttachments: true,
|
||||
}),
|
||||
true,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
export type ComposerDelivery = "submit" | "queue" | "notify";
|
||||
|
||||
export type ComposerDeliveryState = {
|
||||
delivery: ComposerDelivery;
|
||||
workerState: string;
|
||||
protocolOpen: boolean;
|
||||
sending: boolean;
|
||||
hasText: boolean;
|
||||
hasAttachments: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve whether the current Composer draft can use one delivery action.
|
||||
* Immediate Submit is idle-only; Queue and Notify are running-only.
|
||||
*/
|
||||
export function canDeliverComposerDraft(state: ComposerDeliveryState): boolean {
|
||||
if (!state.protocolOpen || state.sending) return false;
|
||||
|
||||
const hasInput = state.hasText || state.hasAttachments;
|
||||
switch (state.delivery) {
|
||||
case "submit":
|
||||
return state.workerState === "idle" && hasInput;
|
||||
case "queue":
|
||||
return state.workerState === "running" && hasInput;
|
||||
case "notify":
|
||||
return state.workerState === "running" && state.hasText &&
|
||||
!state.hasAttachments;
|
||||
}
|
||||
}
|
||||
|
||||
export function sendComposerDelivery<T>(
|
||||
state: ComposerDeliveryState,
|
||||
method: T,
|
||||
send: (method: T) => void,
|
||||
): boolean {
|
||||
if (!canDeliverComposerDraft(state)) return false;
|
||||
send(method);
|
||||
return true;
|
||||
}
|
||||
@@ -2150,6 +2150,12 @@ Deno.test("snapshot restores TaskStore state from system history", () => {
|
||||
const event = snapshotEvent("/repo");
|
||||
if (event.event !== "snapshot") throw new Error("snapshot fixture expected");
|
||||
event.data.session = {
|
||||
pending_submissions: {
|
||||
revision: 0,
|
||||
notification_count: 0,
|
||||
head_id: null,
|
||||
submissions: [],
|
||||
},
|
||||
entries: [{
|
||||
entry_id: "task-reminder-1",
|
||||
timestamp: 1,
|
||||
|
||||
@@ -1064,3 +1064,46 @@ Deno.test("Web Console switches main and direct SubWorker views from the Tasks r
|
||||
"Worker view selection should expose only direct SubWorker session identities with main fallback",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("Web Console uses Notify while running and exposes durable pending controls", async () => {
|
||||
const consolePage = await Deno.readTextFile(
|
||||
new URL(
|
||||
"./../../../routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte",
|
||||
import.meta.url,
|
||||
),
|
||||
);
|
||||
|
||||
for (
|
||||
const token of [
|
||||
'method: "submit"',
|
||||
'method: "notify"',
|
||||
"notification_request_id: crypto.randomUUID()",
|
||||
"submission_request_id: crypto.randomUUID()",
|
||||
'payload.event === "pending_submissions_changed"',
|
||||
'method: "cancel_pending_submission"',
|
||||
'method: "clear_pending_submissions"',
|
||||
'method: "continue_pending"',
|
||||
"handleQueueSubmit",
|
||||
"handleNotifySubmit",
|
||||
'submitDraft(composerInputElement?.snapshot() ?? draft, "queue")',
|
||||
"disabled={!canQueueDraft}",
|
||||
"disabled={!canNotifyDraft}",
|
||||
">Queue Submit</button>",
|
||||
">Notify</button>",
|
||||
]
|
||||
) {
|
||||
assert(
|
||||
consolePage.includes(token),
|
||||
`missing durable pending control token: ${token}`,
|
||||
);
|
||||
}
|
||||
|
||||
const userCase = consolePage.slice(
|
||||
consolePage.indexOf('case "user":'),
|
||||
consolePage.indexOf('case "compact":'),
|
||||
);
|
||||
assert(
|
||||
!userCase.includes("workerRunning"),
|
||||
"ordinary text must remain Submit instead of being implicitly converted to Notify",
|
||||
);
|
||||
});
|
||||
|
||||
+199
-12
@@ -5,6 +5,11 @@
|
||||
import ConsoleTimeline from "$lib/workspace/console/ConsoleTimeline.svelte";
|
||||
import ComposerInput from "$lib/workspace/console/ComposerInput.svelte";
|
||||
import type { ComposerDraftSnapshot } from "$lib/workspace/console/composer-draft";
|
||||
import {
|
||||
canDeliverComposerDraft,
|
||||
sendComposerDelivery,
|
||||
type ComposerDelivery,
|
||||
} from "$lib/workspace/console/composer-delivery";
|
||||
import {
|
||||
buildComposerSegmentsRequest,
|
||||
type WorkerConsoleInputRequest,
|
||||
@@ -31,7 +36,13 @@
|
||||
type ConsoleViewMode,
|
||||
type ConsoleViewScroll,
|
||||
} from "$lib/workspace/console/model";
|
||||
import type { Event as ProtocolEvent, Method as ProtocolMethod, RewindTarget, Segment } from "$lib/generated/protocol";
|
||||
import type {
|
||||
Event as ProtocolEvent,
|
||||
Method as ProtocolMethod,
|
||||
PendingSubmissionsSnapshot,
|
||||
RewindTarget,
|
||||
Segment,
|
||||
} from "$lib/generated/protocol";
|
||||
import {
|
||||
MAX_FILES_PER_SUBMISSION,
|
||||
uploadAttachment,
|
||||
@@ -152,6 +163,13 @@
|
||||
"connecting",
|
||||
);
|
||||
let protocolSubscription: WorkspaceMultiplexerSubscription | null = null;
|
||||
let pendingSubmissions = $state<PendingSubmissionsSnapshot>({
|
||||
revision: 0,
|
||||
notification_count: 0,
|
||||
head_id: null,
|
||||
submissions: [],
|
||||
});
|
||||
let pendingSubmissionItems = $derived(pendingSubmissions.submissions ?? []);
|
||||
let pendingCompletionRequest: {
|
||||
resolve: (entries: ComposerCompletionEntry[]) => void;
|
||||
reject: (error: Error) => void;
|
||||
@@ -234,13 +252,42 @@
|
||||
const workerState = $derived(liveWorkerState ?? worker?.state ?? "loading");
|
||||
const workerRunning = $derived(workerState === "running");
|
||||
const workerPaused = $derived(workerState === "paused");
|
||||
const inputReady = $derived(workerState === "idle");
|
||||
const composerEditable = $derived(protocolState === "open" && !sending);
|
||||
const canSubmitDraft = $derived(inputReady && composerEditable);
|
||||
const canSend = $derived(canSubmitDraft && draft.content.trim().length > 0);
|
||||
const draftHasText = $derived(draft.content.trim().length > 0);
|
||||
const draftHasAttachments = $derived(attachments.length > 0);
|
||||
const canSubmitDraft = $derived(
|
||||
canDeliverComposerDraft({
|
||||
delivery: "submit",
|
||||
workerState,
|
||||
protocolOpen: protocolState === "open",
|
||||
sending,
|
||||
hasText: draftHasText,
|
||||
hasAttachments: draftHasAttachments,
|
||||
}),
|
||||
);
|
||||
const canQueueDraft = $derived(
|
||||
canDeliverComposerDraft({
|
||||
delivery: "queue",
|
||||
workerState,
|
||||
protocolOpen: protocolState === "open",
|
||||
sending,
|
||||
hasText: draftHasText,
|
||||
hasAttachments: draftHasAttachments,
|
||||
}),
|
||||
);
|
||||
const canNotifyDraft = $derived(
|
||||
canDeliverComposerDraft({
|
||||
delivery: "notify",
|
||||
workerState,
|
||||
protocolOpen: protocolState === "open",
|
||||
sending,
|
||||
hasText: draftHasText,
|
||||
hasAttachments: draftHasAttachments,
|
||||
}),
|
||||
);
|
||||
const canStopFromComposer = $derived(workerRunning && composerEditable);
|
||||
const composerSubmitDisabled = $derived(
|
||||
workerRunning ? !canStopFromComposer : !canSend,
|
||||
workerRunning ? !canStopFromComposer : !canSubmitDraft,
|
||||
);
|
||||
|
||||
async function getJson<T>(path: string): Promise<T> {
|
||||
@@ -334,6 +381,13 @@
|
||||
|
||||
function handleIncomingProtocolEvent(payload: ProtocolEvent) {
|
||||
handleProtocolCommandEvent(payload);
|
||||
if (payload.event === "snapshot") {
|
||||
pendingSubmissions = payload.data.session.pending_submissions;
|
||||
} else if (payload.event === "segment_rotated") {
|
||||
pendingSubmissions = payload.data.session.pending_submissions;
|
||||
} else if (payload.event === "pending_submissions_changed") {
|
||||
pendingSubmissions = payload.data.pending;
|
||||
}
|
||||
if (payload.event === "error") {
|
||||
queueObservationDiagnostic({
|
||||
code: payload.data.code,
|
||||
@@ -556,8 +610,9 @@
|
||||
switch (request.kind) {
|
||||
case "user":
|
||||
return {
|
||||
method: "run",
|
||||
method: "submit",
|
||||
params: {
|
||||
submission_request_id: crypto.randomUUID(),
|
||||
input: request.segments ?? [
|
||||
{ kind: "text", content: request.content },
|
||||
],
|
||||
@@ -566,7 +621,11 @@
|
||||
case "notify":
|
||||
return {
|
||||
method: "notify",
|
||||
params: { message: request.content, auto_run: true },
|
||||
params: {
|
||||
notification_request_id: crypto.randomUUID(),
|
||||
message: request.content,
|
||||
auto_run: true,
|
||||
},
|
||||
};
|
||||
case "compact":
|
||||
return { method: "compact" };
|
||||
@@ -638,6 +697,14 @@
|
||||
void submitDraft(composerInputElement?.snapshot() ?? draft);
|
||||
}
|
||||
|
||||
function handleQueueSubmit() {
|
||||
void submitDraft(composerInputElement?.snapshot() ?? draft, "queue");
|
||||
}
|
||||
|
||||
function handleNotifySubmit() {
|
||||
void submitDraft(composerInputElement?.snapshot() ?? draft, "notify");
|
||||
}
|
||||
|
||||
function attachmentPath(): string {
|
||||
return `/api/w/${encodeURIComponent(workspaceId)}/runtimes/${encodeURIComponent(runtimeId)}/workers/${encodeURIComponent(workerId)}`;
|
||||
}
|
||||
@@ -737,7 +804,15 @@
|
||||
if (event.dataTransfer?.files) addAttachmentFiles(event.dataTransfer.files);
|
||||
}
|
||||
|
||||
async function submitDraft(value: ComposerDraftSnapshot) {
|
||||
async function submitDraft(
|
||||
value: ComposerDraftSnapshot,
|
||||
delivery: ComposerDelivery = "submit",
|
||||
) {
|
||||
if (delivery === "notify" && attachments.length > 0) {
|
||||
composerNotice = null;
|
||||
sendError = "Notify accepts text only; remove attachments or queue a Submit.";
|
||||
return;
|
||||
}
|
||||
const incompleteAttachment = attachments.find((attachment) =>
|
||||
attachment.state !== "uploaded" || !attachment.reference
|
||||
);
|
||||
@@ -767,19 +842,38 @@
|
||||
composerInputElement?.clear();
|
||||
return;
|
||||
}
|
||||
if (sending || !inputReady) {
|
||||
const deliveryState = {
|
||||
delivery,
|
||||
workerState,
|
||||
protocolOpen: protocolState === "open",
|
||||
sending,
|
||||
hasText: value.content.trim().length > 0,
|
||||
hasAttachments: attachments.length > 0,
|
||||
};
|
||||
if (!canDeliverComposerDraft(deliveryState)) {
|
||||
return;
|
||||
}
|
||||
|
||||
let request: WorkerConsoleInputRequest = command.request;
|
||||
if (delivery === "notify") {
|
||||
if (request.kind !== "user") {
|
||||
composerNotice = null;
|
||||
sendError = "Notify accepts ordinary text, not a Composer command.";
|
||||
return;
|
||||
}
|
||||
request = { kind: "notify", content: request.content };
|
||||
}
|
||||
sending = true;
|
||||
sendError = null;
|
||||
try {
|
||||
const method = composerRequestToProtocolMethod(command.request);
|
||||
sendProtocolMethod(method);
|
||||
const method = composerRequestToProtocolMethod(request);
|
||||
if (!sendComposerDelivery(deliveryState, method, sendProtocolMethod)) {
|
||||
return;
|
||||
}
|
||||
composerInputElement?.recordHistory(value);
|
||||
composerInputElement?.clear();
|
||||
attachments = [];
|
||||
if (method.method === "run" || method.method === "notify") {
|
||||
if (method.method === "submit" || method.method === "notify") {
|
||||
liveWorkerState = "running";
|
||||
}
|
||||
composerNotice = "Sent through Worker protocol.";
|
||||
@@ -1722,6 +1816,62 @@
|
||||
</aside>
|
||||
{/if}
|
||||
|
||||
{#if pendingSubmissionItems.length > 0 || pendingSubmissions.notification_count > 0}
|
||||
<details class="pending-submissions">
|
||||
<summary>
|
||||
Pending activations ({pendingSubmissionItems.length} submissions · {pendingSubmissions.notification_count} notifications)
|
||||
</summary>
|
||||
<ol>
|
||||
{#each pendingSubmissionItems as submission (submission.submission_id)}
|
||||
<li>
|
||||
<code>{submission.submission_id}</code>
|
||||
<span>{submission.segment_count} segments · {submission.byte_len} bytes</span>
|
||||
<button
|
||||
type="button"
|
||||
onclick={() =>
|
||||
sendControl(
|
||||
{
|
||||
method: "cancel_pending_submission",
|
||||
params: {
|
||||
submission_id: submission.submission_id,
|
||||
expected_revision: pendingSubmissions.revision,
|
||||
},
|
||||
},
|
||||
"Pending submission cancellation",
|
||||
)}
|
||||
>Cancel</button>
|
||||
</li>
|
||||
{/each}
|
||||
</ol>
|
||||
<button
|
||||
type="button"
|
||||
disabled={workerRunning || pendingSubmissions.head_id === null}
|
||||
onclick={() =>
|
||||
sendControl(
|
||||
{
|
||||
method: "continue_pending",
|
||||
params: {
|
||||
expected_revision: pendingSubmissions.revision,
|
||||
expected_head_id: pendingSubmissions.head_id ?? "",
|
||||
},
|
||||
},
|
||||
"Pending activation continue",
|
||||
)}
|
||||
>Continue next</button>
|
||||
<button
|
||||
type="button"
|
||||
onclick={() =>
|
||||
sendControl(
|
||||
{
|
||||
method: "clear_pending_submissions",
|
||||
params: { expected_revision: pendingSubmissions.revision },
|
||||
},
|
||||
"Pending submissions clear",
|
||||
)}
|
||||
>Clear all</button>
|
||||
</details>
|
||||
{/if}
|
||||
|
||||
{#if workerRunning}
|
||||
<WorkerRunStatus
|
||||
startedAtMs={consoleProjection.runActivity.startedAtMs}
|
||||
@@ -1854,6 +2004,18 @@
|
||||
</div>
|
||||
</div>
|
||||
<div class="composer-actions">
|
||||
{#if workerRunning}
|
||||
<button
|
||||
type="button"
|
||||
disabled={!canQueueDraft}
|
||||
onclick={handleQueueSubmit}
|
||||
>Queue Submit</button>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!canNotifyDraft}
|
||||
onclick={handleNotifySubmit}
|
||||
>Notify</button>
|
||||
{/if}
|
||||
{#if composerNotice}
|
||||
<span class="composer-notice">{composerNotice}</span>
|
||||
{/if}
|
||||
@@ -2035,6 +2197,31 @@
|
||||
display: none;
|
||||
}
|
||||
|
||||
.pending-submissions {
|
||||
margin: 0 var(--space-3);
|
||||
color: var(--muted);
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.pending-submissions ol {
|
||||
display: grid;
|
||||
gap: var(--space-1);
|
||||
margin: var(--space-2) 0;
|
||||
padding-left: var(--space-5);
|
||||
}
|
||||
|
||||
.pending-submissions li {
|
||||
display: flex;
|
||||
gap: var(--space-2);
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.pending-submissions code {
|
||||
max-width: 16rem;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.console-log {
|
||||
display: grid;
|
||||
align-content: start;
|
||||
|
||||
Reference in New Issue
Block a user