From 26434bb8676b582bafa9d9617c7de3c8719f518c Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 12 Oct 2026 05:13:53 +0900 Subject: [PATCH 1/2] feat: route Orchestrator delivery and Worker starts from Ticket Assign --- crates/server-api/src/lib.rs | 14 + .../src/feature/builtin/manage_worker.rs | 14 + .../src/feature/builtin/orchestration.rs | 24 +- crates/worker/src/prompt/catalog.rs | 56 +- .../src/assignment_delivery.rs | 380 ++++ .../src/assignment_delivery_tests.rs | 794 ++++++++ .../src/assignment_tracking_tests.rs | 9 +- crates/workspace-server/src/authority.rs | 96 +- crates/workspace-server/src/hosts.rs | 5 + crates/workspace-server/src/server.rs | 1763 ++++++----------- .../server/tests/ticket_worker_claim_tests.rs | 1 + .../tests/ticket_worker_launch_tests.rs | 56 +- .../src/server_assignment_attention_tests.rs | 1073 ++++++++++ .../src/server_assignment_delivery_tests.rs | 962 +++++++++ .../src/server_assignment_start_tests.rs | 752 +++++++ .../src/server_assignment_tracking_tests.rs | 27 +- crates/workspace-server/src/store.rs | 455 +++-- .../src/ticket_tag_upgrade_tests.rs | 4 +- .../src/workspace_deletion.rs | 10 + .../development/ticket-assignment-delivery.md | 60 + openapi/server-api.json | 23 +- .../workspace_orchestrator_queue_attention.md | 15 +- .../src/lib/generated/companion-api.ts | 2 +- .../src/lib/generated/repository-api.ts | 2 +- .../src/lib/generated/runtime-api.ts | 4 +- web/workspace/src/lib/generated/ticket-api.ts | 4 +- .../src/lib/generated/worker-launch-api.ts | 4 +- 27 files changed, 5226 insertions(+), 1383 deletions(-) create mode 100644 crates/workspace-server/src/assignment_delivery.rs create mode 100644 crates/workspace-server/src/assignment_delivery_tests.rs create mode 100644 crates/workspace-server/src/server_assignment_attention_tests.rs create mode 100644 crates/workspace-server/src/server_assignment_delivery_tests.rs create mode 100644 crates/workspace-server/src/server_assignment_start_tests.rs create mode 100644 docs/development/ticket-assignment-delivery.md diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index 3acc5956a..87eaa3a01 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -1977,6 +1977,7 @@ pub trait ServerApi { )] async fn runtime_worker_create_alias( &self, + #[extension] context: ServerRequestContext, #[path] runtime_id: String, #[body] request: RuntimeWorkerSpawnRequest, ) -> Result; @@ -2006,6 +2007,7 @@ pub trait ServerApi { )] async fn runtime_worker_create( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] runtime_id: String, #[body] request: RuntimeWorkerSpawnRequest, @@ -7940,6 +7942,10 @@ pub struct TicketWorkerAttachmentBinding { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(deny_unknown_fields)] pub struct SetTicketRoleAssignmentRequest { + /// Observed delegation required for Orchestrator Ticket starts; manual starts omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub orchestrator_assignment_id: Option, + pub operation_id: String, pub principal: TicketAssignmentPrincipal, pub expected_assignment_id: Option, @@ -8580,6 +8586,10 @@ pub enum RuntimeWorkerSpawnAcceptanceRequirement { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RuntimeWorkerTicketAssignmentRequest { + /// Observed delegation required for Orchestrator Ticket starts; manual starts omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub orchestrator_assignment_id: Option, + pub ticket_id: String, pub operation_id: String, } @@ -9909,6 +9919,10 @@ pub struct BrowserWorkerWorkingDirectorySelection { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(deny_unknown_fields)] pub struct CreateWorkspaceWorkerTicketAssignmentRequest { + /// Observed delegation required for Orchestrator Ticket starts; manual starts omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub orchestrator_assignment_id: Option, + pub ticket_id: String, pub operation_id: String, } diff --git a/crates/worker/src/feature/builtin/manage_worker.rs b/crates/worker/src/feature/builtin/manage_worker.rs index 96dab0b7e..b320afec6 100644 --- a/crates/worker/src/feature/builtin/manage_worker.rs +++ b/crates/worker/src/feature/builtin/manage_worker.rs @@ -473,6 +473,9 @@ pub struct WorkerLifecycleWorkdirAttachment { #[derive(Debug, Clone)] pub struct WorkerLifecycleSpawnRequest { + /// Observed delegation required for Orchestrator Ticket starts. + pub orchestrator_assignment_id: Option, + pub runtime_id: String, /// Ordered alias-keyed selections forwarded as canonical Workdir attachments. pub workdir_attachments: Vec, @@ -494,6 +497,7 @@ fn workspace_worker_create_request( let ticket_assignment = match (request.ticket_id, request.operation_id) { (Some(ticket_id), Some(operation_id)) => { Some(CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: request.orchestrator_assignment_id, ticket_id, operation_id, }) @@ -729,6 +733,9 @@ struct WorkerSpawnInput { /// neither the profile nor an initial Flow segment grants review authority. #[serde(default)] ticket_id: Option, + /// Observed current Orchestrator delegation, required for a Ticket start by the Orchestrator. + #[serde(default)] + orchestrator_assignment_id: Option, #[serde(default)] display_name: Option, /// Normal typed initial user submission delivered after spawn. An empty @@ -913,6 +920,10 @@ impl Tool for WorkspaceWorkerTool { .transpose()?; self.control .spawn_worker(WorkerLifecycleSpawnRequest { + orchestrator_assignment_id: input + .orchestrator_assignment_id + .map(|id| authority_id(&id, "orchestrator_assignment_id")) + .transpose()?, runtime_id: authority_id(&input.runtime_id, "runtime_id")?, workdir_attachments: vec![WorkerLifecycleWorkdirAttachment { alias: INITIAL_WORKDIR_ALIAS.to_string(), @@ -1895,6 +1906,7 @@ mod tests { #[test] fn worker_lifecycle_spawn_request_preserves_canonical_attachment_collection() { let request = workspace_worker_create_request(WorkerLifecycleSpawnRequest { + orchestrator_assignment_id: None, runtime_id: "runtime-1".to_string(), workdir_attachments: vec![ WorkerLifecycleWorkdirAttachment { @@ -1954,6 +1966,7 @@ mod tests { #[test] fn worker_lifecycle_spawn_serializes_ticket_assignment_without_workdirs_or_flow() { let request = workspace_worker_create_request(WorkerLifecycleSpawnRequest { + orchestrator_assignment_id: Some("delegation-742".into()), runtime_id: "runtime-1".to_string(), workdir_attachments: Vec::new(), profile: "project:ticket-worker".to_string(), @@ -1971,6 +1984,7 @@ mod tests { value["ticket_assignment"], serde_json::json!({ "ticket_id": "T-719", + "orchestrator_assignment_id": "delegation-742", "operation_id": "spawn-ticket-worker:T-719:call-1", }) ); diff --git a/crates/worker/src/feature/builtin/orchestration.rs b/crates/worker/src/feature/builtin/orchestration.rs index abbdca4e2..495c382cf 100644 --- a/crates/worker/src/feature/builtin/orchestration.rs +++ b/crates/worker/src/feature/builtin/orchestration.rs @@ -21,7 +21,7 @@ use crate::feature::{ const FEATURE_ID: &str = "orchestration"; const TOOL_NAME: &str = "SpawnTicketWorker"; -const TOOL_DESCRIPTION: &str = "Spawn and atomically assign a generic Ticket Worker for the natural-language initial request. Profile is optional (defaults to builtin:ticket-worker); Flow and repository/Workdir attachments are optional. Results and return choices follow user intent, not a universal approval or MR gate. Backend validates claims and derives attachment access from Ticket targets; acceptance follows spawn, initial input, assignment, and resource finalization. Profile/Flow selection grants neither trusted review nor merge authority."; +const TOOL_DESCRIPTION: &str = "Spawn and atomically assign a generic Ticket Worker for the natural-language initial request. Profile is optional (defaults to builtin:ticket-worker); Flow and repository/Workdir attachments are optional. Results and return choices follow user intent, not a universal approval or MR gate. Orchestrator starts must supply the observed orchestrator_assignment_id; stale delegations cannot start new work. Assign is enough (no Queue/ready); unresolved dependencies are diagnostics for partial work and never start implicitly. Backend validates claims and derives attachment access from Ticket targets; acceptance follows spawn, initial input, assignment, and resource finalization. Profile/Flow selection grants neither trusted review nor merge authority."; #[derive(Debug, Default)] pub struct OrchestrationFeature; @@ -78,6 +78,9 @@ struct SpawnTicketWorkerWorkdirInput { #[serde(deny_unknown_fields)] struct SpawnTicketWorkerInput { ticket_id: String, + /// Observed current delegation identity. Required by Backend for an Orchestrator start; manual callers omit it. + #[serde(default)] + orchestrator_assignment_id: Option, runtime_id: String, /// Optional registered profile selector; omitted/null selects builtin:ticket-worker. /// Backend resolves the profile and validates claims, not natural-language satisfaction. @@ -132,6 +135,10 @@ impl Tool for SpawnTicketWorkerTool { let response = self .worker_service .spawn(WorkerLifecycleSpawnRequest { + orchestrator_assignment_id: input + .orchestrator_assignment_id + .map(|id| authority_id(id, "orchestrator_assignment_id")) + .transpose()?, runtime_id, workdir_attachments, profile, @@ -307,6 +314,7 @@ mod tests { let service = Arc::new(RecordingService::default()); let mut input = input(); input["flow"] = serde_json::json!("project:investigation"); + input["orchestrator_assignment_id"] = serde_json::json!("delegation-742"); input["workdir_attachments"] = serde_json::json!([ { "alias": "checkout", @@ -327,6 +335,10 @@ mod tests { assert_eq!(requests.len(), 1); let request = &requests[0]; assert_eq!(request.runtime_id, "runtime-1"); + assert_eq!( + request.orchestrator_assignment_id.as_deref(), + Some("delegation-742") + ); assert_eq!(request.profile, "project:ticket-worker"); assert_eq!(request.ticket_id.as_deref(), Some("00001KZXN51C7")); assert_eq!( @@ -475,6 +487,16 @@ mod tests { "initial_request must not be empty", ), ("flow", serde_json::json!(" "), "flow must not be empty"), + ( + "orchestrator_assignment_id", + serde_json::json!("http://invalid/id"), + "orchestrator_assignment_id must be an authority id", + ), + ( + "orchestrator_assignment_id", + serde_json::json!(" "), + "orchestrator_assignment_id must not be empty", + ), ( "runtime_id", serde_json::json!("runtime/1"), diff --git a/crates/worker/src/prompt/catalog.rs b/crates/worker/src/prompt/catalog.rs index cdb30cfae..4d2497661 100644 --- a/crates/worker/src/prompt/catalog.rs +++ b/crates/worker/src/prompt/catalog.rs @@ -156,11 +156,15 @@ impl WorkerPrompt { ]; } -/// Model-visible queued Ticket projection shared by Server and TUI backlog attention paths. +/// Model-visible Ticket projection shared by Server and TUI attention paths. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct OrchestratorQueueAttentionTicket { resource_key: String, title: String, + assignment_id: Option, + input_event_id: Option, + waiting_reason: Option, + next_action: Option, } impl OrchestratorQueueAttentionTicket { @@ -175,8 +179,28 @@ impl OrchestratorQueueAttentionTicket { Ok(Self { resource_key, title: bounded_queue_attention_text(&title.into(), 240), + assignment_id: None, + input_event_id: None, + waiting_reason: None, + next_action: None, }) } + + /// Committed delegation/input identity, not a handling-status transition. + /// The receiving Worker must reread current authority before a new action. + pub fn with_assignment_input( + mut self, + assignment_id: &str, + input_event_id: &str, + waiting_reason: Option<&str>, + next_action: Option<&str>, + ) -> Self { + self.assignment_id = Some(bounded_queue_attention_text(assignment_id, 1024)); + self.input_event_id = Some(bounded_queue_attention_text(input_event_id, 1024)); + self.waiting_reason = waiting_reason.map(|text| bounded_queue_attention_text(text, 1024)); + self.next_action = next_action.map(|text| bounded_queue_attention_text(text, 1024)); + self + } } /// Shared model-visible context for every Orchestrator backlog attention renderer. @@ -200,7 +224,7 @@ impl OrchestratorQueueAttentionContext { } } -/// Prompt-catalog entries that must share the same backlog-attention body contract. +/// Prompt-catalog entries sharing a bounded Ticket resource-key projection. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OrchestratorQueueAttentionPrompt { Server, @@ -841,8 +865,10 @@ mod tests { .orchestrator_queue_attention(OrchestratorQueueAttentionPrompt::Tui, &context) .unwrap(); - assert_eq!(server, tui); - assert!(server.starts_with("Queued Tickets require attention:")); + assert!(server.starts_with("Assigned Tickets have unreceived input:")); + // TUI's final operation model is a separate downstream scope; both + // renderers still consume the same bounded resource-key projection. + assert!(tui.starts_with("Queued Tickets require attention:")); assert!(server.contains("- T-1 — Ticket 1 with control ")); assert!(!server.contains("T-21")); assert!(server.contains("were omitted from this notice: 1")); @@ -859,6 +885,28 @@ mod tests { } } + #[test] + fn assignment_attention_carries_input_identity_and_bounded_waiting_context() { + let ticket = OrchestratorQueueAttentionTicket::new("T-742", "Partial work") + .unwrap() + .with_assignment_input( + "assignment-a", + "answer-event", + Some("human\nanswer"), + Some(&"x".repeat(2048)), + ); + let context = OrchestratorQueueAttentionContext::new(vec![ticket]); + let output = PromptCatalog::builtins_only() + .unwrap() + .orchestrator_queue_attention(OrchestratorQueueAttentionPrompt::Server, &context) + .unwrap(); + assert!(output.contains("Delegation: assignment-a; input: answer-event.")); + assert!(output.contains("Waiting reason: human answer")); + assert!(output.contains(&format!("Next action: {}…", "x".repeat(1024)))); + assert!(!output.contains(&"x".repeat(1025))); + assert!(output.contains("no ready/Queue step is required")); + } + #[test] fn queue_attention_prompt_omits_truncation_text_for_complete_list() { let catalog = PromptCatalog::builtins_only().unwrap(); diff --git a/crates/workspace-server/src/assignment_delivery.rs b/crates/workspace-server/src/assignment_delivery.rs new file mode 100644 index 000000000..db59d0f51 --- /dev/null +++ b/crates/workspace-server/src/assignment_delivery.rs @@ -0,0 +1,380 @@ +//! Durable operational delivery and startup fencing, independent of handling. +use super::*; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TicketAssignmentDelivery { + pub ticket_id: String, + pub assignment_id: String, + pub input_event_id: String, + pub delivery_id: String, + pub request_id: String, + pub waiting_reason: Option, + pub next_action: Option, +} + +pub(super) const SCHEMA_SQL: &str = r#" +CREATE TABLE ticket_assignment_delivery_origins ( + workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, assignment_id TEXT NOT NULL, + input_event_id TEXT NOT NULL, after_ticket_event_index INTEGER NOT NULL, + PRIMARY KEY(workspace_id,assignment_id), + FOREIGN KEY(workspace_id,ticket_id,assignment_id) REFERENCES ticket_worker_assignments(workspace_id,ticket_id,assignment_id) +); +CREATE TABLE ticket_assignment_deliveries ( + workspace_id TEXT NOT NULL, assignment_id TEXT NOT NULL, input_event_id TEXT NOT NULL, + delivery_id TEXT NOT NULL, payload_json TEXT NOT NULL CHECK(json_valid(payload_json)), + reserved_source_json TEXT NOT NULL CHECK(json_valid(reserved_source_json)), + PRIMARY KEY(workspace_id,assignment_id,input_event_id), UNIQUE(workspace_id,delivery_id), + FOREIGN KEY(workspace_id,assignment_id) REFERENCES ticket_worker_assignments(workspace_id,assignment_id) +); +CREATE TABLE ticket_assignment_delivery_requests ( + workspace_id TEXT NOT NULL, delivery_id TEXT NOT NULL, content TEXT NOT NULL, + PRIMARY KEY(workspace_id,delivery_id), + FOREIGN KEY(workspace_id,delivery_id) REFERENCES ticket_assignment_deliveries(workspace_id,delivery_id) +); +CREATE TRIGGER ticket_assignment_delivery_request_no_update BEFORE UPDATE ON ticket_assignment_delivery_requests +BEGIN SELECT RAISE(ABORT,'Assignment delivery request is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_request_no_delete BEFORE DELETE ON ticket_assignment_delivery_requests +BEGIN SELECT RAISE(ABORT,'Assignment delivery request is immutable'); END; +CREATE TABLE ticket_assignment_delivery_acknowledgements ( + workspace_id TEXT NOT NULL, delivery_id TEXT NOT NULL, + source_json TEXT NOT NULL CHECK(json_valid(source_json)), PRIMARY KEY(workspace_id,delivery_id), + FOREIGN KEY(workspace_id,delivery_id) REFERENCES ticket_assignment_deliveries(workspace_id,delivery_id) +); +CREATE TABLE ticket_assignment_operation_fences ( + workspace_id TEXT NOT NULL, operation_id TEXT NOT NULL, orchestrator_assignment_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, agent_key TEXT NOT NULL, + authentication_binding_id TEXT NOT NULL, source_json TEXT NOT NULL CHECK(json_valid(source_json)), + PRIMARY KEY(workspace_id,operation_id), + FOREIGN KEY(workspace_id,operation_id) REFERENCES ticket_assignment_operations(workspace_id,operation_id), + FOREIGN KEY(workspace_id,orchestrator_assignment_id) REFERENCES ticket_worker_assignments(workspace_id,assignment_id) +); +CREATE TRIGGER ticket_assignment_delivery_origin_insert +AFTER INSERT ON typed_ticket_assignment_tracking_history +WHEN json_extract(NEW.source_context_json,'$.kind')='assignment' +BEGIN + INSERT INTO ticket_assignment_delivery_origins + VALUES(NEW.workspace_id,NEW.ticket_id,NEW.assignment_id,'assignment-event:' || NEW.event_id, + COALESCE((SELECT MAX(event_index) FROM typed_ticket_events + WHERE workspace_id=NEW.workspace_id AND ticket_id=NEW.ticket_id),-1)); +END; +CREATE TRIGGER ticket_assignment_delivery_payload_no_update BEFORE UPDATE ON ticket_assignment_deliveries +BEGIN SELECT RAISE(ABORT,'Assignment delivery payload is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_payload_no_delete BEFORE DELETE ON ticket_assignment_deliveries +BEGIN SELECT RAISE(ABORT,'Assignment delivery payload is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_ack_no_update BEFORE UPDATE ON ticket_assignment_delivery_acknowledgements +BEGIN SELECT RAISE(ABORT,'Assignment delivery acknowledgement is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_ack_no_delete BEFORE DELETE ON ticket_assignment_delivery_acknowledgements +BEGIN SELECT RAISE(ABORT,'Assignment delivery acknowledgement is immutable'); END; +CREATE TRIGGER ticket_assignment_operation_fence_no_update BEFORE UPDATE ON ticket_assignment_operation_fences +BEGIN SELECT RAISE(ABORT,'Assignment startup fence is immutable'); END; +CREATE TRIGGER ticket_assignment_operation_fence_no_delete BEFORE DELETE ON ticket_assignment_operation_fences +BEGIN SELECT RAISE(ABORT,'Assignment startup fence is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_origin_no_update BEFORE UPDATE ON ticket_assignment_delivery_origins +BEGIN SELECT RAISE(ABORT,'Assignment input origin is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_origin_no_delete BEFORE DELETE ON ticket_assignment_delivery_origins +BEGIN SELECT RAISE(ABORT,'Assignment input origin is immutable'); END; +"#; + +/// Additive Workspace schema only. Native pre-upgrade assignments use their +/// committed initial audit time to exclude earlier Ticket inputs; ambiguous +/// legacy relationships are NOT treated as fresh delegations. Migration inputs +/// retain the exact Queue fence already reconciled by Ticket schema 11. +pub(super) fn install(conn: &Connection) -> Result<()> { + conn.execute_batch(SCHEMA_SQL)?; + conn.execute_batch(r#" +INSERT INTO ticket_assignment_delivery_origins +SELECT h.workspace_id,h.ticket_id,h.assignment_id,'assignment-event:' || h.event_id, + COALESCE((SELECT MAX(e.event_index) FROM typed_ticket_events e + WHERE e.workspace_id=h.workspace_id AND e.ticket_id=h.ticket_id AND e.ata.after_ticket_event_index + -- External content/answer inputs, not case/routing/history or own self-report. + -- Ticket supports typed extension events; an answer need not be named comment. + AND e.kind NOT IN ('create','close','state_changed','status_changed','history') + AND e.kind NOT LIKE 'assignment_tracking%' + AND e.kind NOT LIKE 'assignment_delivery%' + AND COALESCE(e.state_field,'') NOT IN ('case','routing','assignment_tracking') +) +SELECT i.ticket_id,i.assignment_id,i.input_event_id, + json_extract(i.tracking_json,'$.waiting_reason'),json_extract(i.tracking_json,'$.next_action'),d.payload_json +FROM inputs i +LEFT JOIN ticket_assignment_deliveries d ON d.workspace_id=?1 AND d.assignment_id=i.assignment_id AND d.input_event_id=i.input_event_id +WHERE NOT EXISTS(SELECT 1 FROM ticket_assignment_delivery_acknowledgements ack + WHERE ack.workspace_id=?1 AND ack.delivery_id=d.delivery_id) +AND (?3 IS NULL OR i.assignment_id=?3) AND (?4 IS NULL OR i.input_event_id=?4) +"#; + +fn decode_payload(payload: &str) -> Result { + serde_json::from_str(payload).map_err(|e| Error::Store(e.to_string())) +} + +pub(super) fn list_in_tx( + conn: &Connection, + workspace_id: &str, + limit: usize, +) -> Result> { + inputs_in_tx(conn, workspace_id, limit, None, None) +} +fn inputs_in_tx( + conn: &Connection, + workspace_id: &str, + limit: usize, + assignment_id: Option<&str>, + input_event_id: Option<&str>, +) -> Result> { + let sql = format!("{INPUTS_SQL} ORDER BY i.ticket_id,i.assignment_id,i.input_order LIMIT ?2"); + let mut statement = conn.prepare(&sql)?; + let rows = statement.query_map( + params![workspace_id, limit as i64, assignment_id, input_event_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + )) + }, + )?; + rows.map(|row| { + let (ticket_id, assignment_id, input_event_id, waiting_reason, next_action, payload) = row?; + if let Some(payload) = payload { + return decode_payload(&payload); + } + // Hash the identities, not mutable handling. Scope is Workspace-qualified. + let identities = serde_json::to_string(&(workspace_id, &assignment_id, &input_event_id)) + .map_err(|e| Error::Store(e.to_string()))?; + let digest = Sha256::digest(identities.as_bytes()); + let hex = digest + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let delivery_id = format!("assignment-delivery:{hex}"); + Ok(TicketAssignmentDelivery { + ticket_id, + assignment_id, + input_event_id, + request_id: delivery_id.clone(), + delivery_id, + waiting_reason, + next_action, + }) + }) + .collect() +} + +fn source_guard( + conn: &Connection, + workspace_id: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result<()> { + assignment_tracking::require_runtime_source( + conn, + workspace_id, + &source.runtime_id, + &source.worker_id, + &source.agent_key, + &source.authentication_binding_id, + ) +} + +pub(super) fn require_current( + conn: &Connection, + workspace_id: &str, + ticket_id: &str, + assignment_id: &str, +) -> Result<()> { + let current:bool=conn.query_row(r#"SELECT EXISTS(SELECT 1 FROM ticket_current_worker_assignments c + JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.assignment_id=c.assignment_id + JOIN typed_tickets t ON t.workspace_id=a.workspace_id AND t.ticket_id=a.ticket_id + WHERE c.workspace_id=?1 AND c.ticket_id=?2 AND c.assignment_id=?3 AND c.role='orchestrator' + AND a.principal_kind='workspace_agent' AND a.principal_id='workspace-orchestrator' + AND t.workflow_state='open' AND json_extract(a.tracking_json,'$.status') IN ('waiting','inprogress') + AND EXISTS(SELECT 1 FROM ticket_assignment_delivery_origins o WHERE o.workspace_id=a.workspace_id AND o.assignment_id=a.assignment_id) + AND NOT EXISTS(SELECT 1 FROM ticket_assignment_work_releases r WHERE r.workspace_id=a.workspace_id AND r.assignment_id=a.assignment_id))"#,params![workspace_id,ticket_id,assignment_id],|r|r.get(0))?; + if !current { + return Err(Error::TicketAssignmentConflict( + "Assignment is no longer a current actionable delegation".into(), + )); + } + Ok(()) +} + +pub(super) fn reserve( + store: &SqliteWorkspaceStore, + workspace_id: &str, + assignment_id: &str, + input_event_id: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result { + store.with_conn_mut(|conn| { + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + source_guard(&tx,workspace_id,source)?; + let ticket_id:String=tx.query_row("SELECT ticket_id FROM ticket_worker_assignments WHERE workspace_id=?1 AND assignment_id=?2",params![workspace_id,assignment_id],|r|r.get(0))?; + require_current(&tx,workspace_id,&ticket_id,assignment_id)?; + let saved:Option=tx.query_row("SELECT payload_json FROM ticket_assignment_deliveries WHERE workspace_id=?1 AND assignment_id=?2 AND input_event_id=?3",params![workspace_id,assignment_id,input_event_id],|r|r.get(0)).optional()?; + if let Some(saved)=saved {let result=decode_payload(&saved)?;tx.commit()?;return Ok(result);} + // A caller cannot fabricate an input. Restrict the same common query by + // exact assignment/input identity rather than trusting a scan snapshot. + let candidates=inputs_in_tx(&tx,workspace_id,1,Some(assignment_id),Some(input_event_id))?; + let delivery=candidates.into_iter().find(|d|d.assignment_id==assignment_id && d.input_event_id==input_event_id) + .ok_or_else(||Error::TicketAssignmentConflict("Assignment input is not pending".into()))?; + tx.execute("INSERT INTO ticket_assignment_deliveries VALUES(?1,?2,?3,?4,?5,?6)",params![workspace_id,assignment_id,input_event_id,delivery.delivery_id,serde_json::to_string(&delivery).map_err(|e|Error::Store(e.to_string()))?,serde_json::to_string(source).map_err(|e|Error::Store(e.to_string()))?])?; + tx.commit()?; + Ok(delivery) + }) +} + +/// Persist the exact rendered Runtime request before I/O. A losing renderer or +/// retry receives the original bytes, not a new payload under the same ID. +pub(super) fn prepare_content( + store: &SqliteWorkspaceStore, + workspace_id: &str, + delivery_id: &str, + content: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result { + if content.is_empty() || content.len() > 262144 || content.contains('\0') { + return Err(Error::InvalidInput( + "Assignment delivery content must be nonempty, NUL-free and at most 262144 bytes" + .into(), + )); + } + store.with_conn_mut(|conn| { + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + source_guard(&tx,workspace_id,source)?; + let payload:String=tx.query_row("SELECT payload_json FROM ticket_assignment_deliveries WHERE workspace_id=?1 AND delivery_id=?2",params![workspace_id,delivery_id],|r|r.get(0))?; + let delivery=decode_payload(&payload)?; + require_current(&tx,workspace_id,&delivery.ticket_id,&delivery.assignment_id)?; + let saved:Option=tx.query_row("SELECT content FROM ticket_assignment_delivery_requests WHERE workspace_id=?1 AND delivery_id=?2",params![workspace_id,delivery_id],|r|r.get(0)).optional()?; + let result=if let Some(saved)=saved {saved} else { + tx.execute("INSERT INTO ticket_assignment_delivery_requests VALUES(?1,?2,?3)",params![workspace_id,delivery_id,content])?; + content.to_owned() + }; + tx.commit()?; + Ok(result) + }) +} + +pub(super) fn ack( + store: &SqliteWorkspaceStore, + workspace_id: &str, + delivery_id: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result<()> { + store.with_conn_mut(|conn| { + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + source_guard(&tx,workspace_id,source)?; + let payload:String=tx.query_row("SELECT payload_json FROM ticket_assignment_deliveries WHERE workspace_id=?1 AND delivery_id=?2",params![workspace_id,delivery_id],|r|r.get(0))?; + let delivery=decode_payload(&payload)?; + require_current(&tx,workspace_id,&delivery.ticket_id,&delivery.assignment_id)?; + let source_json=serde_json::to_string(source).map_err(|e|Error::Store(e.to_string()))?; + let saved:Option=tx.query_row("SELECT source_json FROM ticket_assignment_delivery_acknowledgements WHERE workspace_id=?1 AND delivery_id=?2",params![workspace_id,delivery_id],|r|r.get(0)).optional()?; + if let Some(saved)=saved { + if saved!=source_json {return Err(Error::TicketAssignmentConflict("Delivery acknowledgement source differs from saved receipt".into()));} + } else {tx.execute("INSERT INTO ticket_assignment_delivery_acknowledgements VALUES(?1,?2,?3)",params![workspace_id,delivery_id,source_json])?;} + tx.commit()?; + Ok(()) + }) +} + +/// Must run before receipt replay as well as every bind/finalize write. Absence +/// is the existing manual reservation contract, not inferred delegation. +pub(super) fn operation_guard( + conn: &Connection, + workspace_id: &str, + operation_id: &str, +) -> Result<()> { + let fence=conn.query_row("SELECT o.ticket_id,f.orchestrator_assignment_id,f.runtime_id,f.worker_id,f.agent_key,f.authentication_binding_id FROM ticket_assignment_operation_fences f JOIN ticket_assignment_operations o ON o.workspace_id=f.workspace_id AND o.operation_id=f.operation_id WHERE f.workspace_id=?1 AND f.operation_id=?2",params![workspace_id,operation_id],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?,r.get::<_,String>(2)?,r.get::<_,String>(3)?,r.get::<_,String>(4)?,r.get::<_,String>(5)?))).optional()?; + if let Some((ticket_id, assignment_id, runtime_id, worker_id, agent_key, binding_id)) = fence { + assignment_tracking::require_runtime_source( + conn, + workspace_id, + &runtime_id, + &worker_id, + &agent_key, + &binding_id, + )?; + require_current(conn, workspace_id, &ticket_id, &assignment_id)?; + } + Ok(()) +} + +pub(super) fn save_operation_fence( + conn: &Connection, + workspace_id: &str, + operation_id: &str, + ticket_id: &str, + assignment_id: &str, + source: &TicketAssignmentTrackingSourceContext, + existing: bool, +) -> Result<()> { + source_guard(conn, workspace_id, source)?; + require_current(conn, workspace_id, ticket_id, assignment_id)?; + let source_json = serde_json::to_string(source).map_err(|e| Error::Store(e.to_string()))?; + let saved:Option<(String,String)>=conn.query_row("SELECT orchestrator_assignment_id,source_json FROM ticket_assignment_operation_fences WHERE workspace_id=?1 AND operation_id=?2",params![workspace_id,operation_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()?; + if let Some(saved) = saved { + if saved != (assignment_id.to_owned(), source_json) { + return Err(Error::TicketAssignmentConflict( + "Startup operation assignment/source differs from saved fence".into(), + )); + } + } else { + if existing { + return Err(Error::TicketAssignmentConflict( + "Cannot add a delegation fence to an existing manual operation".into(), + )); + } + conn.execute( + "INSERT INTO ticket_assignment_operation_fences VALUES(?1,?2,?3,?4,?5,?6,?7,?8)", + params![ + workspace_id, + operation_id, + assignment_id, + source.runtime_id, + source.worker_id, + source.agent_key, + source.authentication_binding_id, + source_json + ], + )?; + } + Ok(()) +} diff --git a/crates/workspace-server/src/assignment_delivery_tests.rs b/crates/workspace-server/src/assignment_delivery_tests.rs new file mode 100644 index 000000000..7a93d3c12 --- /dev/null +++ b/crates/workspace-server/src/assignment_delivery_tests.rs @@ -0,0 +1,794 @@ +// Included in store::tests: exercise persisted transaction boundaries, not Runtime I/O. +fn delivery_store_external_event( + store: &SqliteWorkspaceStore, + index: i64, + kind: &str, + state_field: Option<&str>, +) { + store.with_conn(|conn| { + conn.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,author,at,state_field,body) VALUES('w','tracking',?1,?2,'user','2026-10-12T00:00:00Z',?3,'Answer arrived')",params![index,kind,state_field])?; + Ok(()) + }).unwrap(); +} + +fn delivery_store_reserve_initial( + store: &SqliteWorkspaceStore, + source: &TicketAssignmentTrackingSourceContext, +) -> TicketAssignmentDelivery { + let inputs = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!(inputs.len(), 1); + store + .reserve_ticket_assignment_delivery( + "w", + &inputs[0].assignment_id, + &inputs[0].input_event_id, + source, + ) + .unwrap() +} + +#[tokio::test] +async fn assignment_delivery_waiting_self_reports_do_not_busy_loop_and_answers_are_pending() { + let (_directory, store) = tracking_store_fixture().await; + // Existing history before Assign must not become a new consideration input. + delivery_store_external_event(&store, 0, "comment", None); + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let initial = delivery_store_reserve_initial(&store, &source); + store + .ack_ticket_assignment_delivery("w", &initial.delivery_id, &source) + .unwrap(); + let mut update = tracking_store_input("a", "event-assign-a", "waiting-a"); + update.tracking.status = TicketAssignmentTrackingStatus::Waiting; + update.tracking.waiting_reason = Some("Waiting for owner answer".into()); + update.tracking.next_action = Some("Read answer and decide".into()); + store + .update_ticket_assignment_tracking("w", "tracking", &update, &source) + .unwrap(); + assert!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .is_empty() + ); + delivery_store_external_event( + &store, + 1, + "assignment_tracking", + Some("assignment_tracking"), + ); + delivery_store_external_event(&store, 2, "comment", None); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].input_event_id, "ticket-event:2"); + assert_eq!( + pending[0].waiting_reason.as_deref(), + Some("Waiting for owner answer") + ); + assert_eq!( + pending[0].next_action.as_deref(), + Some("Read answer and decide") + ); + let answer = store + .reserve_ticket_assignment_delivery("w", "a", "ticket-event:2", &source) + .unwrap(); + store + .ack_ticket_assignment_delivery("w", &answer.delivery_id, &source) + .unwrap(); + assert!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .is_empty() + ); + // Typed extension inputs must not disappear merely because they are not + // comments; own tracking/history events remain excluded. + delivery_store_external_event(&store, 3, "assignment_tracking_updated", None); + delivery_store_external_event(&store, 4, "history", None); + delivery_store_external_event(&store, 5, "owner_answer", None); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].input_event_id, "ticket-event:5"); + let saved = store + .get_ticket_assignment_tracking("w", "tracking", "a") + .unwrap() + .unwrap(); + assert_eq!( + saved.tracking.unwrap().status, + TicketAssignmentTrackingStatus::Waiting + ); +} + +#[tokio::test] +async fn assignment_delivery_response_loss_returns_immutable_payload_after_progress_changes_and_reopen() + { + let (directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let initial = delivery_store_reserve_initial(&store, &source); + let mut update = tracking_store_input("a", "event-assign-a", "waiting-a"); + update.tracking.status = TicketAssignmentTrackingStatus::Waiting; + update.tracking.waiting_reason = Some("New reason not part of original delivery".into()); + store + .update_ticket_assignment_tracking("w", "tracking", &update, &source) + .unwrap(); + drop(store); + let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + assert_eq!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(), + vec![initial.clone()] + ); + assert_eq!( + store + .reserve_ticket_assignment_delivery("w", "a", &initial.input_event_id, &source) + .unwrap(), + initial + ); + store + .ack_ticket_assignment_delivery("w", &initial.delivery_id, &source) + .unwrap(); + store + .ack_ticket_assignment_delivery("w", &initial.delivery_id, &source) + .unwrap(); + assert!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .is_empty() + ); + store + .with_conn(|conn| { + assert!( + conn.execute( + "UPDATE ticket_assignment_deliveries SET payload_json='{}'", + [] + ) + .is_err() + ); + assert!( + conn.execute( + "DELETE FROM ticket_assignment_delivery_acknowledgements", + [] + ) + .is_err() + ); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_delivery_simultaneous_reservations_converge_to_one_receipt() { + let (directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .remove(0); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let threads = (0..2) + .map(|_| { + let barrier = barrier.clone(); + let path = directory.path().join("workspace.db"); + let input = pending.input_event_id.clone(); + std::thread::spawn(move || { + let store = SqliteWorkspaceStore::open(path).unwrap(); + barrier.wait(); + store + .reserve_ticket_assignment_delivery("w", "a", &input, &tracking_store_source()) + .unwrap() + }) + }) + .collect::>(); + let receipts = threads + .into_iter() + .map(|t| t.join().unwrap()) + .collect::>(); + assert_eq!(receipts[0], receipts[1]); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM ticket_assignment_deliveries", + [], + |r| r.get(0), + )?; + assert_eq!(count, 1); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_delivery_done_close_reopen_and_unknown_history_do_not_revive_delegation() { + for change in ["done", "close-reopen", "unknown"] { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let initial = delivery_store_reserve_initial(&store, &source); + match change { + "done" => { + store + .update_ticket_assignment_tracking( + "w", + "tracking", + &tracking_store_input("a", "event-assign-a", "done-a"), + &source, + ) + .unwrap(); + } + "close-reopen" => { + store.with_conn(|conn| {conn.execute("UPDATE typed_tickets SET workflow_state='closed' WHERE ticket_id='tracking'",[])?;conn.execute("UPDATE typed_tickets SET workflow_state='open' WHERE ticket_id='tracking'",[])?;Ok(())}).unwrap(); + } + _ => { + store.with_conn(|conn| {conn.execute("UPDATE ticket_worker_assignments SET tracking_json=NULL,last_tracking_event_id=NULL WHERE assignment_id='a'",[])?;Ok(())}).unwrap(); + } + } + delivery_store_external_event(&store, 1, "comment", None); + assert!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .is_empty(), + "{change}" + ); + assert!( + store + .reserve_ticket_assignment_delivery("w", "a", &initial.input_event_id, &source) + .is_err(), + "{change}" + ); + } +} + +#[tokio::test] +async fn assignment_delivery_clear_reassign_fences_scanned_and_reserved_inputs() { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let old = delivery_store_reserve_initial(&store, &source); + store + .clear_current_ticket_role_assignment( + "w", + "tracking", + TicketAssignmentRole::Orchestrator, + "a", + "clear-a", + "clear-a", + "owner", + "2", + None, + ) + .unwrap(); + let new = tracking_store_assign(&store, "b", "assign-b"); + assert_eq!(new.assignment_id, "b"); + assert!( + store + .reserve_ticket_assignment_delivery("w", "a", &old.input_event_id, &source) + .is_err() + ); + assert!( + store + .ack_ticket_assignment_delivery("w", &old.delivery_id, &source) + .is_err() + ); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].assignment_id, "b"); + assert_ne!(pending[0].request_id, old.request_id); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM ticket_worker_assignments WHERE role='orchestrator'", + [], + |r| r.get(0), + )?; + assert_eq!(count, 2); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_delivery_old_worker_or_authenticated_binding_cannot_reserve_or_ack() { + for change in ["replacement", "revoke", "binding-change"] { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let old = delivery_store_reserve_initial(&store, &source); + store.with_conn(|conn| { + match change { + "replacement" => {conn.execute("UPDATE worker_singleton_owners SET worker_id='replacement' WHERE workspace_id='w'",[])?;}, + "revoke" => {conn.execute("UPDATE workspace_runtime_bindings SET revoked_at='2' WHERE workspace_id='w'",[])?;}, + _ => {conn.execute("UPDATE workspace_runtime_bindings SET binding_id='new-binding' WHERE workspace_id='w'",[])?;}, + } Ok(()) + }).unwrap(); + assert!( + matches!( + store.reserve_ticket_assignment_delivery("w", "a", &old.input_event_id, &source), + Err(Error::WorkspacePermissionDenied(_)) + ), + "{change}" + ); + assert!( + matches!( + store.ack_ticket_assignment_delivery("w", &old.delivery_id, &source), + Err(Error::WorkspacePermissionDenied(_)) + ), + "{change}" + ); + if change == "replacement" { + let mut replacement = source.clone(); + replacement.worker_id = "replacement".into(); + assert_eq!( + store + .reserve_ticket_assignment_delivery("w", "a", &old.input_event_id, &replacement) + .unwrap(), + old + ); + store + .ack_ticket_assignment_delivery("w", &old.delivery_id, &replacement) + .unwrap(); + assert_eq!( + store + .list_current_ticket_assignment_tracking("w", "tracking") + .unwrap() + .len(), + 1 + ); + } + } +} + +fn delivery_store_start_reserve( + store: &SqliteWorkspaceStore, + key: &str, + assignment: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result { + store.reserve_ticket_assignment_operation_for_orchestrator( + "w", + key, + "tracking", + "r", + Some("replacement"), + "start-input", + "3", + assignment, + source, + ) +} + +#[tokio::test] +async fn assignment_startup_clear_reassign_fences_reservation_replay_bind_and_finalize() { + for change in ["clear", "reassign", "close-reopen"] { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + assert!(delivery_store_start_reserve(&store, "start-a", "a", &source).unwrap()); + assert!(!delivery_store_start_reserve(&store, "start-a", "a", &source).unwrap()); + if change == "close-reopen" { + store.with_conn(|conn| {conn.execute("UPDATE typed_tickets SET workflow_state='closed' WHERE ticket_id='tracking'",[])?;conn.execute("UPDATE typed_tickets SET workflow_state='open' WHERE ticket_id='tracking'",[])?;Ok(())}).unwrap(); + } else { + // Clear/reassign must not be blocked by a pending startup claim: + // assignment authority can invalidate it without Runtime side effects. + store + .clear_current_ticket_role_assignment( + "w", + "tracking", + TicketAssignmentRole::Orchestrator, + "a", + "clear-a", + "clear-a", + "owner", + "4", + None, + ) + .unwrap(); + if change == "reassign" { + tracking_store_assign(&store, "b", "assign-b"); + } + } + assert!( + delivery_store_start_reserve(&store, "start-a", "a", &source).is_err(), + "{change}" + ); + assert!( + delivery_store_start_reserve(&store, "start-new", "a", &source).is_err(), + "{change}" + ); + assert!( + store + .validate_ticket_assignment_operation_fence("w", "start-a") + .is_err(), + "{change}" + ); + assert!( + store + .bind_ticket_assignment_operation_worker("w", "start-a", "replacement") + .is_err(), + "{change}" + ); + let record = TicketWorkerAssignmentRecord { + workspace_id: "w".into(), + ticket_id: "tracking".into(), + assignment_id: "target-a".into(), + worker: RuntimeWorkerRef::new("r", "replacement"), + assigned_by: "owner".into(), + assigned_at: "4".into(), + }; + assert!( + store + .set_current_ticket_worker_assignment( + &record, + None, + "worker-event", + "start-a", + false + ) + .is_err(), + "{change}" + ); + assert!( + store + .get_active_ticket_role_assignment("w", "tracking", TicketAssignmentRole::Worker) + .unwrap() + .is_none() + ); + } +} + +#[tokio::test] +async fn assignment_startup_replaced_worker_cannot_start_or_replay_saved_operations() { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + delivery_store_start_reserve(&store, "start-a", "a", &source).unwrap(); + store + .with_conn(|conn| { + conn.execute( + "UPDATE worker_singleton_owners SET worker_id='replacement' WHERE workspace_id='w'", + [], + )?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + delivery_store_start_reserve(&store, "start-a", "a", &source), + Err(Error::WorkspacePermissionDenied(_)) + )); + assert!(matches!( + delivery_store_start_reserve(&store, "start-new", "a", &source), + Err(Error::WorkspacePermissionDenied(_)) + )); + assert!(matches!( + store.validate_ticket_assignment_operation_fence("w", "start-a"), + Err(Error::WorkspacePermissionDenied(_)) + )); + assert!(matches!( + store.bind_ticket_assignment_operation_worker("w", "start-a", "replacement"), + Err(Error::WorkspacePermissionDenied(_)) + )); + // Even the new Worker cannot adopt the old sender's startup operation key. + let mut replacement = source.clone(); + replacement.worker_id = "replacement".into(); + assert!(delivery_store_start_reserve(&store, "start-a", "a", &replacement).is_err()); +} + +#[tokio::test] +async fn assignment_startup_saved_fence_cannot_be_bypassed_or_attached_to_manual_receipt() { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + delivery_store_start_reserve(&store, "start-a", "a", &source).unwrap(); + assert!( + store + .reserve_ticket_assignment_operation( + "w", + "start-a", + "tracking", + "r", + Some("replacement"), + "start-input", + "3" + ) + .is_err() + ); + store.with_conn(|conn| {assert!(conn.execute("UPDATE ticket_assignment_operation_fences SET orchestrator_assignment_id='different'",[]).is_err());Ok(())}).unwrap(); + store + .fail_ticket_assignment_operation("w", "start-a", "explicit cancellation") + .unwrap(); + store + .reserve_ticket_assignment_operation( + "w", + "manual", + "tracking", + "r", + Some("replacement"), + "start-input", + "3", + ) + .unwrap(); + assert!(delivery_store_start_reserve(&store, "manual", "a", &source).is_err()); +} + +#[tokio::test] +async fn assignment_delivery_rendered_content_is_immutable_across_restart_and_concurrent_renderers() +{ + let (directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let initial = delivery_store_reserve_initial(&store, &source); + assert_eq!( + store + .prepare_ticket_assignment_delivery_content( + "w", + &initial.delivery_id, + "original rendered prompt", + &source + ) + .unwrap(), + "original rendered prompt" + ); + drop(store); + let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + assert_eq!( + store + .prepare_ticket_assignment_delivery_content( + "w", + &initial.delivery_id, + "different title/handling/config prompt", + &source + ) + .unwrap(), + "original rendered prompt" + ); + store + .with_conn(|conn| { + assert!( + conn.execute( + "UPDATE ticket_assignment_delivery_requests SET content='changed'", + [] + ) + .is_err() + ); + assert!( + conn.execute("DELETE FROM ticket_assignment_delivery_requests", []) + .is_err() + ); + Ok(()) + }) + .unwrap(); + store + .clear_current_ticket_role_assignment( + "w", + "tracking", + TicketAssignmentRole::Orchestrator, + "a", + "clear-a", + "clear-a", + "owner", + "4", + None, + ) + .unwrap(); + assert!( + store + .prepare_ticket_assignment_delivery_content( + "w", + &initial.delivery_id, + "original rendered prompt", + &source + ) + .is_err() + ); +} + +#[test] +fn assignment_delivery_schema92_upgrade_uses_reconciled_migration_evidence_not_legacy_history() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("schema92.db"); + // A genuine retained Ticket-10/Workspace-91 fixture goes through its actual + // applied tracking migration; no current DB markers or columns are lowered. + let conn = tracking_schema91_fixture(&path); + migrate_ticket_tracking_v91_to_v92(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 92); + let assignments = stored_fixture_rows(&conn, "ticket_worker_assignments"); + let operations = stored_fixture_rows(&conn, "ticket_assignment_operations"); + let tracking = stored_fixture_rows(&conn, "typed_ticket_assignment_tracking_history"); + drop(conn); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!( + pending + .iter() + .map(|d| d.ticket_id.as_str()) + .collect::>(), + vec!["pending", "progress"] + ); + assert_eq!( + pending[0].input_event_id, + "assignment-event:migration-tracking:a-pending" + ); + store + .with_conn(|conn| { + assert_eq!(current_schema_version(conn)?, 93); + assert_eq!( + stored_fixture_rows(conn, "ticket_worker_assignments"), + assignments + ); + assert_eq!( + stored_fixture_rows(conn, "ticket_assignment_operations"), + operations + ); + assert_eq!( + stored_fixture_rows(conn, "typed_ticket_assignment_tracking_history"), + tracking + ); + let violations: i64 = + conn.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |r| { + r.get(0) + })?; + assert_eq!(violations, 0); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_startup_simultaneous_reservations_admit_runtime_effects_once() { + let (directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let threads = (0..2) + .map(|_| { + let barrier = barrier.clone(); + let path = directory.path().join("workspace.db"); + std::thread::spawn(move || { + let store = SqliteWorkspaceStore::open(path).unwrap(); + barrier.wait(); + delivery_store_start_reserve(&store, "start-a", "a", &tracking_store_source()) + .unwrap() + }) + }) + .collect::>(); + let admitted = threads + .into_iter() + .map(|t| t.join().unwrap()) + .collect::>(); + assert_eq!(admitted.iter().filter(|result| **result).count(), 1); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM ticket_assignment_operation_fences", + [], + |r| r.get(0), + )?; + assert_eq!(count, 1); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn assignment_delivery_fresh_and_upgraded_operational_schemas_are_identical() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("retained.db"); + let conn = tracking_schema91_fixture(&path); + migrate_ticket_tracking_v91_to_v92(&conn).unwrap(); + drop(conn); + let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory().unwrap(); + let objects = |store: &SqliteWorkspaceStore| { + store.with_conn(|conn| { + let mut statement=conn.prepare("SELECT type,name,sql FROM sqlite_schema WHERE name LIKE 'ticket_assignment_delivery_%' OR name LIKE 'ticket_assignment_operation_fence%' ORDER BY name")?; + Ok(statement.query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?,r.get::<_,String>(2)?)))?.collect::>>()?) + }).unwrap() + }; + assert_eq!(objects(&fresh), objects(&upgraded)); +} + +#[tokio::test] +async fn assignment_delivery_missing_immutable_request_guard_refuses_startup() { + let (directory, store) = tracking_store_fixture().await; + store + .with_conn(|conn| { + conn.execute( + "DROP TRIGGER ticket_assignment_delivery_request_no_update", + [], + )?; + Ok(()) + }) + .unwrap(); + drop(store); + assert!( + matches!(SqliteWorkspaceStore::open(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update")) + ); +} + +#[tokio::test] +async fn assignment_delivery_fabricated_inputs_and_foreign_workspace_cannot_create_receipts() { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + assert!( + store + .reserve_ticket_assignment_delivery("w", "a", "ticket-event:9999", &source) + .is_err() + ); + assert!( + store + .reserve_ticket_assignment_delivery( + "other", + "a", + "assignment-event:event-assign-a", + &source + ) + .is_err() + ); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM ticket_assignment_deliveries", + [], + |r| r.get(0), + )?; + assert_eq!(count, 0); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_startup_unfinished_dependencies_remain_diagnostics_not_store_admission_gates() { + let (_directory, store) = manual_worker_start_store().await; + seed_manual_start_ticket(&store, "partial", "open"); + seed_manual_start_ticket(&store, "dependency", "open"); + store.with_conn(|conn| { + conn.execute("INSERT INTO typed_ticket_relations(workspace_id,ticket_id,kind,target,author,at) VALUES('w','partial','depends_on','dependency','owner','1')",[])?; + Ok(()) + }).unwrap(); + let assignment = manual_worker_assignment("partial", "partial-assignment"); + store + .start_ticket_with_worker_assignment( + &assignment, + "partial-event", + "partial-operation", + None, + ) + .unwrap(); + assert!( + store + .get_active_ticket_role_assignment("w", "partial", TicketAssignmentRole::Worker) + .unwrap() + .is_some() + ); + assert!( + store + .get_active_ticket_role_assignment("w", "dependency", TicketAssignmentRole::Worker) + .unwrap() + .is_none() + ); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM typed_ticket_relations WHERE kind='depends_on'", + [], + |r| r.get(0), + )?; + assert_eq!(count, 1); + Ok(()) + }) + .unwrap(); +} diff --git a/crates/workspace-server/src/assignment_tracking_tests.rs b/crates/workspace-server/src/assignment_tracking_tests.rs index 3fd7f8b6b..7c85c4812 100644 --- a/crates/workspace-server/src/assignment_tracking_tests.rs +++ b/crates/workspace-server/src/assignment_tracking_tests.rs @@ -439,12 +439,12 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip .iter() .map(|m| m.version) .collect::>(), - vec![92] + vec![92, 93] ); let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { - assert_eq!(current_schema_version(conn)?, 92); + assert_eq!(current_schema_version(conn)?, 93); assert_eq!( conn.query_row( "SELECT applied_at FROM ticket_schema_migrations WHERE version=10", @@ -457,7 +457,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip workspace_schema_migration_history(conn)?, vec![ (91, "workspace schema baseline".into()), - (92, "Ticket-owned assignment tracking".into()) + (92, "Ticket-owned assignment tracking".into()), + (93, "Durable assignment deliveries and source-bound startup fences".into()) ] ); for (table, rows) in retained.iter().zip(original.iter()) { @@ -595,7 +596,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first .is_empty() ); store.with_conn(|conn| { - assert_eq!(current_schema_version(conn)?, 92); + assert_eq!(current_schema_version(conn)?, 93); assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations", [], |row| row.get::<_,i64>(0))?, 11); assert_eq!(conn.query_row("SELECT workflow_state FROM typed_tickets WHERE ticket_id='historical-done'", [], |row| row.get::<_,String>(0))?, "closed"); let stored: (Option, Option) = conn.query_row("SELECT tracking_json,last_tracking_event_id FROM ticket_worker_assignments WHERE assignment_id='a-historical-done'", [], |row| Ok((row.get(0)?,row.get(1)?)))?; diff --git a/crates/workspace-server/src/authority.rs b/crates/workspace-server/src/authority.rs index 686be3115..abda3e7b2 100644 --- a/crates/workspace-server/src/authority.rs +++ b/crates/workspace-server/src/authority.rs @@ -783,12 +783,12 @@ impl SqliteWorkspaceAuthority { }) }) .transpose()?; - // Responsibility remains visible after work ends; action eligibility must - // use only active unfinished work, including after a Ticket is reopened. + // Work start eligibility uses unfinished work. Delegation authoring and + // clearing use the retained current pointer, including after close. let active_role_assignments = self .store .list_active_ticket_role_assignments(&self.workspace_id, id)?; - let has_orchestrator = active_role_assignments + let has_orchestrator = role_assignments .iter() .any(|assignment| assignment.role == TicketAssignmentRole::Orchestrator); let has_worker = active_role_assignments @@ -811,7 +811,6 @@ impl SqliteWorkspaceAuthority { }) .collect::>>()?; let write_repository_keys = ticket_write_target_repository_keys(&targets); - let has_blockers = !ticket.relations.blockers.is_empty(); let mut assignment_diagnostics = Vec::new(); if let Some(legacy_assignee) = ticket .meta @@ -827,12 +826,18 @@ impl SqliteWorkspaceAuthority { let action_eligibility = TicketActionEligibility { // Case openness is not readiness. These are explicit operations; // occupancy remains an independent resource constraint. - can_assign_orchestrator: !has_orchestrator && !has_worker, + // Delegation neither occupies nor transfers existing Worker work. + // Start eligibility below separately retains unfinished-work guards. + // A fresh operation explicitly delegates/reissues even to the same + // principal. Expected-assignment CAS (when supplied) and source + // authority are checked by the mutation, not by Ticket phase. + can_assign_orchestrator: true, can_unassign_orchestrator: has_orchestrator, can_queue: ticket.meta.workflow_state == TicketState::Open, - can_start_manual_worker: ticket.meta.workflow_state == TicketState::Open - && !has_worker - && !has_blockers, + // Unfinished dependencies remain relation diagnostics, not a blanket + // start refusal. Backend start/assign still validates authorization, + // resource grants, reservations and actual unfinished work. + can_start_manual_worker: ticket.meta.workflow_state == TicketState::Open && !has_worker, queue_tickets: dependency_check.queue_tickets.clone(), blockers: action_blockers, }; @@ -3232,6 +3237,81 @@ mod tests { ); } + #[tokio::test] + async fn unfinished_dependencies_remain_diagnostic_without_disabling_manual_start() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("workspace.db"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: "w".into(), + owner_account_id: "owner-account".into(), + display_name: "W".into(), + state: "active".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .await + .unwrap(); + let backend = SqliteTicketBackend::open_verified(&path, "w").unwrap(); + let dependent = backend + .create(ticket::NewTicket::new( + "Partial investigation without resources", + )) + .unwrap(); + let prerequisite = backend + .create(ticket::NewTicket::new("Unfinished prerequisite")) + .unwrap(); + backend + .add_ticket_relation( + dependent.id.clone().into(), + ticket::NewTicketRelation { + kind: ticket::TicketRelationKind::DependsOn, + target: prerequisite.id.clone(), + note: Some("Partial work can proceed".into()), + author: Some("owner-account".into()), + }, + ) + .unwrap(); + let authority = SqliteWorkspaceAuthority::new(&path, "w").unwrap(); + let detail = authority.ticket(&dependent.id).unwrap(); + assert!(detail.targets.is_empty()); + assert!(detail.action_eligibility.can_start_manual_worker); + assert!(detail.action_eligibility.blockers.is_empty()); + assert_eq!(detail.relations.blockers.len(), 1); + assert_eq!( + detail.relations.blockers[0].blocking_ticket, + prerequisite.id + ); + assert_eq!(detail.relations.outgoing[0].kind, "depends_on"); + let blocked = authority + .query_tickets(TicketQueryRequest { + attention: vec![server_api::TicketAttention::Blocked], + ..TicketQueryRequest::default() + }) + .unwrap(); + assert_eq!(blocked.items.len(), 1); + assert_eq!(blocked.items[0].id, dependent.id); + assert_eq!(blocked.items[0].unresolved_blocker_count, 1); + let prerequisite_detail = authority.ticket(&prerequisite.id).unwrap(); + assert!(prerequisite_detail.assignments.is_empty()); + assert_eq!(prerequisite_detail.state.as_str(), "open"); + rusqlite::Connection::open(&path) + .unwrap() + .execute( + "UPDATE typed_tickets SET workflow_state='closed' WHERE ticket_id=?1", + [&dependent.id], + ) + .unwrap(); + assert!( + !authority + .ticket(&dependent.id) + .unwrap() + .action_eligibility + .can_start_manual_worker + ); + } + #[tokio::test] async fn manual_worker_eligibility_accepts_explicit_states_without_reviving_released_work() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index cb8dfa169..c0fe34c99 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -450,6 +450,10 @@ pub struct WorkerSpawnWorkingDirectoryRequest { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct WorkerTicketAssignmentRequest { + /// Observed delegation required for Orchestrator Ticket starts; manual starts omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub orchestrator_assignment_id: Option, + pub ticket_id: String, pub operation_id: String, } @@ -7546,6 +7550,7 @@ mod tests { fn worker_spawn_idempotency_fingerprint_covers_canonical_initial_submit() { let mut request = embedded_spawn_request(); request.ticket_assignment = Some(WorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: "00001KVZSGT0Q".to_string(), operation_id: "operation-1".to_string(), }); diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 6740bc570..f536b55aa 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1538,6 +1538,7 @@ struct OrchestratorAttentionDelivery { fingerprint: String, request_id: String, content: String, + input: crate::store::TicketAssignmentDelivery, } struct OrchestratorAttentionReservation(Option>>); @@ -3687,6 +3688,9 @@ impl WorkspaceApi { } } api.recover_backend_jobs()?; + if let Some(orchestrator) = find_online_workspace_orchestrator(&api) { + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } Ok(api) } @@ -6864,6 +6868,7 @@ fn worker_ticket_assignment_from_api( assignment: server_api::RuntimeWorkerTicketAssignmentRequest, ) -> WorkerTicketAssignmentRequest { WorkerTicketAssignmentRequest { + orchestrator_assignment_id: assignment.orchestrator_assignment_id, ticket_id: assignment.ticket_id, operation_id: assignment.operation_id, } @@ -8483,10 +8488,12 @@ impl server_api::ServerApi for ServerApiContractService { request: server_api::CreateWorkspaceWorkerRequest, ) -> std::result::Result { - create_workspace_worker( - State(self.workspace_api()?.clone()), + create_workspace_worker_authenticated( + self.workspace_api()?.clone(), contract_request_headers(&context)?, - Json(request), + request, + None, + Some(&context), ) .await .map(|Json(response)| response) @@ -8548,11 +8555,12 @@ impl server_api::ServerApi for ServerApiContractService { request: server_api::CreateWorkspaceWorkerRequest, ) -> std::result::Result { - spawn_known_worker( + spawn_known_worker_authenticated( State(self.workspace_api()?.clone()), AxumPath(ScopedWorkspacePath { workspace_id }), current_worker_contract_headers(&context)?, Json(request), + Some(&context), ) .await .map(|Json(response)| response) @@ -8726,11 +8734,14 @@ impl server_api::ServerApi for ServerApiContractService { request: server_api::CreateWorkspaceWorkerRequest, ) -> std::result::Result { - scoped_create_workspace_worker( - State(self.workspace_api()?.clone()), - AxumPath(ScopedWorkspacePath { workspace_id }), + validate_workspace_scope(self.workspace_api()?, &workspace_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + create_workspace_worker_authenticated( + self.workspace_api()?.clone(), contract_request_headers(&context)?, - Json(request), + request, + None, + Some(&context), ) .await .map(|Json(response)| response) @@ -9192,14 +9203,16 @@ impl server_api::ServerApi for ServerApiContractService { async fn runtime_worker_create_alias( &self, + context: server_api::ServerRequestContext, runtime_id: String, request: server_api::RuntimeWorkerSpawnRequest, ) -> std::result::Result { - let Json(response) = create_runtime_worker( + let Json(response) = create_runtime_worker_authenticated( State(self.workspace_api()?.clone()), AxumPath(runtime_id), Json(request), + Some(&context), ) .await .map_err(ApiError::into_repository_api_error)?; @@ -9228,18 +9241,19 @@ impl server_api::ServerApi for ServerApiContractService { async fn runtime_worker_create( &self, + context: server_api::ServerRequestContext, workspace_id: String, runtime_id: String, request: server_api::RuntimeWorkerSpawnRequest, ) -> std::result::Result { - let Json(response) = scoped_create_runtime_worker( + validate_workspace_scope(self.workspace_api()?, &workspace_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + let Json(response) = create_runtime_worker_authenticated( State(self.workspace_api()?.clone()), - AxumPath(ScopedRuntimePath { - workspace_id, - runtime_id, - }), + AxumPath(runtime_id), Json(request), + Some(&context), ) .await .map_err(ApiError::into_repository_api_error)?; @@ -13373,6 +13387,7 @@ async fn start_manual_ticket_worker_assignment( record: &TicketRoleAssignmentRecord, operation_id: &str, selected: &[server_api::TicketWorkerAttachmentBinding], + delegation: Option<&TicketStartDelegation>, ) -> ApiResult { let TicketAssignmentPrincipal::Worker { runtime_id, @@ -13405,6 +13420,20 @@ async fn start_manual_ticket_worker_assignment( .store .get_ticket_assignment_operation(&api.config.workspace_id, operation_id)? { + if delegation.is_some() { + reserve_ticket_start( + api, + operation_id, + &record.ticket_id, + &worker.runtime_id, + Some(&worker.worker_id), + &fingerprint, + &record.assigned_at, + delegation, + )?; + } + api.store + .validate_ticket_assignment_operation_fence(&api.config.workspace_id, operation_id)?; if operation.request_fingerprint.as_deref() != Some(fingerprint.as_str()) { return Err(Error::TicketAssignmentConflict( "manual Worker claim operation was reused with different input".into(), @@ -13500,14 +13529,15 @@ async fn start_manual_ticket_worker_assignment( } let binding = validate_manual_ticket_worker_workdir_binding(api, &worker, &record.ticket_id, &selected)?; - let admitted = api.store.reserve_ticket_assignment_operation( - &api.config.workspace_id, + let admitted = reserve_ticket_start( + api, operation_id, &record.ticket_id, &worker.runtime_id, Some(&worker.worker_id), &fingerprint, &record.assigned_at, + delegation, )?; if !admitted { return Err(Error::TicketAssignmentConflict("manual Worker claim is already admitted; reread its durable result instead of redispatching".into()).into()); @@ -13531,6 +13561,8 @@ async fn start_manual_ticket_worker_assignment( )?; return Err(error.into()); } + api.store + .validate_ticket_assignment_operation_fence(&api.config.workspace_id, operation_id)?; let stopped = api .runtime .stop_worker( @@ -13884,11 +13916,16 @@ async fn apply_set_ticket_assignment( ) .into()); } + let delegation = ticket_start_delegation( + source.as_ref(), + request.orchestrator_assignment_id.as_deref(), + )?; start_manual_ticket_worker_assignment( &api, &record, &operation_id, &request.workdir_bindings, + delegation.as_ref(), ) .await? } @@ -13900,6 +13937,13 @@ async fn apply_set_ticket_assignment( .into()); } }; + if role == TicketAssignmentRole::Orchestrator + && let Some(orchestrator) = find_workspace_orchestrator(&api) + { + // The Assign row/event/receipt has committed. Delivery failure cannot + // roll back delegation; startup/restore/idle use this same durable scan. + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } Ok(Json(server_api::TicketRoleAssignmentMutationResponse { workspace_id, ticket_id: ticket.id, @@ -14090,6 +14134,118 @@ async fn cancel_ticket_worker( Ok(()) } +/// Observation-bound delegation for a new Ticket execution operation. This is +/// distinct from self-reported handling and from the Worker work assignment. +type TicketStartDelegation = (String, ticket::TicketAssignmentTrackingSourceContext); + +fn ticket_start_delegation( + source: Option<&crate::store::TicketOrchestratorAssignmentSourceContext>, + observed_assignment_id: Option<&str>, +) -> Result> { + use crate::store::TicketOrchestratorAssignmentSourceContext; + match source { + Some(TicketOrchestratorAssignmentSourceContext::Runtime { + runtime_id, + worker_id, + agent_key, + authentication_binding_id, + }) => { + let assignment_id = observed_assignment_id.filter(|id| !id.trim().is_empty()) + .ok_or_else(|| Error::TicketAssignmentConflict("Orchestrator Ticket start requires the observed orchestrator_assignment_id".into()))?; + if assignment_id.len() > 1024 || assignment_id.chars().any(char::is_control) { + return Err(Error::InvalidInput( + "orchestrator_assignment_id must be control-free and at most 1024 bytes".into(), + )); + } + Ok(Some(( + assignment_id.to_owned(), + ticket::TicketAssignmentTrackingSourceContext { + actor: format!("worker:{runtime_id}/{worker_id}"), + runtime_id: runtime_id.clone(), + worker_id: worker_id.clone(), + agent_key: agent_key.clone(), + authentication_binding_id: authentication_binding_id.clone(), + }, + ))) + } + _ if observed_assignment_id.is_some() => Err(Error::WorkspacePermissionDenied( + "Only the authenticated current Orchestrator may start under a delegation".into(), + )), + _ => Ok(None), + } +} + +fn ticket_spawn_delegation( + api: &WorkspaceApi, + headers: &HeaderMap, + context: Option<&server_api::ServerRequestContext>, + observed_assignment_id: Option<&str>, +) -> Result> { + if let Some(context) = context { + // Generic authenticated Worker/manual starts are not implicit delegation. + // Only the registered Orchestrator starts under its observed assignment. + let worker = authenticated_ticket_worker_source(headers, context)?; + if let Some(worker) = worker.as_ref() { + api.store + .require_current_worker_singleton_owner(&api.config.workspace_id, worker)?; + if find_workspace_orchestrator(api).is_some_and(|current| current.worker == *worker) { + let (_, source) = authenticated_assignment_source(api, headers, context)?; + return ticket_start_delegation(Some(&source), observed_assignment_id); + } + } + ticket_start_delegation(None, observed_assignment_id) + } else { + // Only direct test adapters omit middleware context. Production routes + // must preserve the binding that actually authenticated the request. + #[cfg(test)] + { + ticket_start_delegation(None, observed_assignment_id) + } + #[cfg(not(test))] + { + Err(Error::WorkspacePermissionDenied( + "Ticket start requires Backend-authenticated context".into(), + )) + } + } +} + +fn reserve_ticket_start( + api: &WorkspaceApi, + operation_id: &str, + ticket_id: &str, + runtime_id: &str, + worker_id: Option<&str>, + fingerprint: &str, + created_at: &str, + delegation: Option<&TicketStartDelegation>, +) -> Result { + if let Some((assignment_id, source)) = delegation { + api.store + .reserve_ticket_assignment_operation_for_orchestrator( + &api.config.workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + fingerprint, + created_at, + assignment_id, + source, + ) + } else { + api.store.reserve_ticket_assignment_operation( + &api.config.workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + fingerprint, + created_at, + ) + } +} + fn validate_ticket_assignment_state( api: &WorkspaceApi, assignment: &WorkerTicketAssignmentRequest, @@ -14102,13 +14258,8 @@ fn validate_ticket_assignment_state( ticket.state.as_str() ))); } - let dependencies = - browser_ticket_backend(api)?.dependency_check(TicketIdOrSlug::Id(ticket.id))?; - if !dependencies.blockers.is_empty() { - return Err(Error::TicketAssignmentConflict( - "Ticket work is blocked by unresolved relations".into(), - )); - } + // Dependencies remain diagnostics for the caller's partial-work decision; + // they are not authority, resource occupancy, or execution reservations. Ok(()) } @@ -17233,55 +17384,6 @@ fn active_orchestrator_assignment( } } -fn orchestrator_interested( - api: &WorkspaceApi, - workspace_id: &str, - ticket_id: &str, - _state: &str, -) -> Result> { - let backend = browser_ticket_backend(api)?; - let ticket = backend.show(ticket_id.into()).map_err(Error::from)?; - if ticket.meta.workflow_state != TicketState::Open { - return Ok(None); - } - let has_unfinished_work = api - .store - .get_active_ticket_worker_assignment(workspace_id, ticket_id)? - .is_some(); - // Transitional routing continues to require an explicit queue request or - // unfinished work. Openness alone is not a routing request. A later case - // decision fences older queue receipts; T-741/742 will replace this path. - let explicitly_queued = ticket - .events - .iter() - .rev() - .find_map(|event| { - if event.attributes.contains_key("queue_root_ticket") { - Some(true) - } else if event.attributes.contains_key("case_action") { - Some(false) - } else { - None - } - }) - .unwrap_or_else(|| { - // Old queued/inprogress cases may predate queue receipts. This frozen - // migration handoff preserves their intent, but any new case decision - // fences it so reopening never revives an old routing request. - ticket.case_history.is_empty() - && ticket - .legacy_workflow_snapshot - .as_ref() - .is_some_and(|snapshot| { - matches!(snapshot.workflow_state.as_str(), "queued" | "inprogress") - }) - }); - if !has_unfinished_work && !explicitly_queued { - return Ok(None); - } - active_orchestrator_assignment(api, workspace_id, ticket_id) -} - fn worker_ticket_source_context( api: &WorkspaceApi, workspace_id: &str, @@ -17440,13 +17542,8 @@ fn notify_ticket_recipients( { recipients.push(assignment.worker.clone()); } - if orchestrator_interested(api, workspace_id, ticket_id, current_state) - .ok() - .flatten() - .is_some() - && let Some(orchestrator) = find_workspace_orchestrator(api) - { - recipients.push(orchestrator.worker.clone()); + if let Some(orchestrator) = find_workspace_orchestrator(api) { + dispatch_orchestrator_assignment_attention(api, &orchestrator.worker); } recipients.sort(); recipients.dedup(); @@ -18768,24 +18865,19 @@ fn maybe_dispatch_orchestrator_turn_end( if orchestrator.worker.runtime_id == EMBEDDED_WORKER_RUNTIME_ID && orchestrator.worker.worker_id == worker_id { - dispatch_orchestrator_queue_attention(api, &orchestrator.worker, false); + dispatch_orchestrator_assignment_attention(api, &orchestrator.worker); } } -fn dispatch_orchestrator_queue_attention( - api: &WorkspaceApi, - worker: &RuntimeWorkerRef, - retry_only: bool, -) { - with_orchestrator_attention_reservation(api, worker, retry_only, |state, pending| { - check_orchestrator_queue_attention(api, worker, state, pending); +fn dispatch_orchestrator_assignment_attention(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { + with_orchestrator_attention_reservation(api, worker, |state, pending| { + check_orchestrator_assignment_attention(api, worker, state, pending); }); } fn with_orchestrator_attention_reservation( api: &WorkspaceApi, worker: &RuntimeWorkerRef, - mut retry_only: bool, mut check: impl FnMut( &Arc>, Option, @@ -18795,7 +18887,7 @@ fn with_orchestrator_attention_reservation( loop { let pending = { let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); - if state.sending || (retry_only && state.pending.is_none()) { + if state.sending { return; } state.sending = true; @@ -18806,74 +18898,69 @@ fn with_orchestrator_attention_reservation( if !reservation.complete() { return; } - // Accepted Restore is an explicit recheck even when the current check - // was a no-op or an already-online retry. Never infer it from Idle. - retry_only = false; + // Accepted Restore is an explicit durable-input recheck even when the + // current check was a no-op. A local cache is never backlog authority. } } -fn check_orchestrator_queue_attention( +fn orchestrator_delivery_target( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> Result { + if !find_online_workspace_orchestrator(api).is_some_and(|current| current.worker == *worker) { + return Err(Error::WorkspacePermissionDenied( + "Delivery target is not the current registered Orchestrator".into(), + )); + } + let binding = api + .config_store + .get_workspace_runtime_binding(&api.config.workspace_id, &worker.runtime_id)? + .filter(|binding| binding.revoked_at.is_none()) + .ok_or_else(|| { + Error::WorkspacePermissionDenied("Delivery target has no active Runtime binding".into()) + })?; + Ok(ticket::TicketAssignmentTrackingSourceContext { + actor: "backend:assignment-delivery".into(), + runtime_id: worker.runtime_id.clone(), + worker_id: worker.worker_id.clone(), + agent_key: "workspace-orchestrator".into(), + authentication_binding_id: binding.binding_id, + }) +} + +fn check_orchestrator_assignment_attention( api: &WorkspaceApi, worker: &RuntimeWorkerRef, state: &Arc>, pending: Option, ) { - // Unknown outcomes keep the original immutable payload and ID, even if the - // backlog or prompt projection changed before this retry. + let Ok(target) = orchestrator_delivery_target(api, worker) else { + return; + }; if let Some(delivery) = pending { - send_orchestrator_attention(api, worker, &state, delivery); - return; - } - let Ok(backend) = browser_ticket_backend(api) else { - return; - }; - let Ok(mut queued) = backend.list(ticket::TicketListQuery::states([ - ticket::TicketListState::Open, - ])) else { - return; - }; - queued.retain(|ticket| { - orchestrator_interested(api, &api.config.workspace_id, &ticket.id, "open") - .ok() - .flatten() - .is_some() - }); - queued.sort_by(|left, right| left.id.cmp(&right.id)); - if queued.is_empty() { - state.lock().unwrap_or_else(|p| p.into_inner()).accepted = None; - return; - } - let fingerprint = queued - .iter() - .map(|ticket| ticket.id.as_str()) - .collect::>() - .join("|"); - if state - .lock() - .unwrap_or_else(|p| p.into_inner()) - .accepted - .as_ref() - .map(|delivery| delivery.fingerprint.as_str()) - == Some(fingerprint.as_str()) - { - return; - } - - let attention_context = match orchestrator_queue_attention_context( - &api.config.workspace_id, - &api.config.workspace_id, - &queued, - ) { - Ok(context) => context, - Err(error) => { - tracing::warn!( - workspace_id = %api.config.workspace_id, - candidate_count = queued.len(), - diagnostic = error, - "orchestrator backlog attention projection rejected" - ); - return; + // Validate current identity before retrying an immutable request. Clear, + // reissue, close and Worker replacement cannot revive an old delivery. + if api + .store + .reserve_ticket_assignment_delivery( + &api.config.workspace_id, + &delivery.input.assignment_id, + &delivery.input.input_event_id, + &target, + ) + .is_ok() + { + send_orchestrator_attention(api, worker, state, delivery); + } else { + state.lock().unwrap_or_else(|p| p.into_inner()).pending = None; } + return; + } + let Ok(inputs) = api + .store + .list_pending_ticket_assignment_deliveries(&api.config.workspace_id, 20) + else { + return; }; let Ok(Some(config_state)) = api .config_store @@ -18890,27 +18977,66 @@ fn check_orchestrator_queue_attention( let Ok(catalog) = worker::PromptCatalog::from_projection(projection.catalog().clone()) else { return; }; - let content = match catalog.orchestrator_queue_attention( - worker::OrchestratorQueueAttentionPrompt::Server, - &attention_context, - ) { - Ok(content) => content, - Err(error) => { - tracing::warn!( - workspace_id = %api.config.workspace_id, - diagnostic = %error, - "orchestrator backlog attention rendering failed" - ); - return; + for input in inputs { + // Reserve from the same pending-input contract, rechecking current + // assignment and target binding under the write lock before any I/O. + let Ok(input) = api.store.reserve_ticket_assignment_delivery( + &api.config.workspace_id, + &input.assignment_id, + &input.input_event_id, + &target, + ) else { + continue; + }; + let Ok(ticket) = api.authority.ticket(&input.ticket_id) else { + continue; + }; + let Ok(projected) = + worker::OrchestratorQueueAttentionTicket::new(ticket.resource_key, ticket.title) + else { + continue; + }; + let context = + worker::OrchestratorQueueAttentionContext::new(vec![projected.with_assignment_input( + &input.assignment_id, + &input.input_event_id, + input.waiting_reason.as_deref(), + input.next_action.as_deref(), + )]); + let Ok(content) = catalog.orchestrator_queue_attention( + worker::OrchestratorQueueAttentionPrompt::Server, + &context, + ) else { + continue; + }; + // The complete rendered request, including title/catalog projection, is + // pinned before dispatch; restart retries never render a new payload + // under an already accepted Runtime request id. + let Ok(content) = api.store.prepare_ticket_assignment_delivery_content( + &api.config.workspace_id, + &input.delivery_id, + &content, + &target, + ) else { + continue; + }; + let delivery = OrchestratorAttentionDelivery { + fingerprint: input.delivery_id.clone(), + request_id: input.request_id.clone(), + content, + input, + }; + state.lock().unwrap_or_else(|p| p.into_inner()).pending = Some(delivery.clone()); + send_orchestrator_attention(api, worker, state, delivery); + if state + .lock() + .unwrap_or_else(|p| p.into_inner()) + .pending + .is_some() + { + break; } - }; - let delivery = OrchestratorAttentionDelivery { - fingerprint, - request_id: uuid::Uuid::now_v7().to_string(), - content, - }; - state.lock().unwrap_or_else(|p| p.into_inner()).pending = Some(delivery.clone()); - send_orchestrator_attention(api, worker, &state, delivery); + } } fn send_orchestrator_attention( @@ -18919,30 +19045,76 @@ fn send_orchestrator_attention( state: &Mutex, delivery: OrchestratorAttentionDelivery, ) { - // Bounded delivery-side retry, not an Idle-snapshot retry. Rejection and - // transport ambiguity leave pending available for the next attempt. + tracing::debug!(delivery_id = %delivery.fingerprint, assignment_id = %delivery.input.assignment_id, "dispatch committed assignment input"); for _ in 0..2 { - let accepted = api - .runtime - .send_input( - worker, - WorkerInputRequest { - kind: WorkerInputKind::Notify, - content: delivery.content.clone(), - submission_request_id: Some(delivery.request_id.clone()), - segments: None, - }, - ) - .is_ok_and(|result| result.state == InternalWorkerOperationState::Accepted); - if accepted { - let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); - state.accepted = Some(delivery); - state.pending = None; + let Ok(target) = orchestrator_delivery_target(api, worker) else { return; + }; + if api + .store + .reserve_ticket_assignment_delivery( + &api.config.workspace_id, + &delivery.input.assignment_id, + &delivery.input.input_event_id, + &target, + ) + .is_err() + { + state.lock().unwrap_or_else(|p| p.into_inner()).pending = None; + return; + } + let result = api.runtime.send_input( + worker, + WorkerInputRequest { + kind: WorkerInputKind::Notify, + content: delivery.content.clone(), + submission_request_id: Some(delivery.request_id.clone()), + segments: None, + }, + ); + if result + .as_ref() + .is_ok_and(|result| result.state == InternalWorkerOperationState::Accepted) + { + if api + .store + .ack_ticket_assignment_delivery( + &api.config.workspace_id, + &delivery.input.delivery_id, + &target, + ) + .is_ok() + { + let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); + state.accepted = Some(delivery); + state.pending = None; + } + return; + } + if let Some(error_category) = ticket_notification_delivery_error_category(&result) { + tracing::warn!( + event = "ticket_assignment_delivery_pending", + workspace_id = %api.config.workspace_id, + assignment_id = %delivery.input.assignment_id, + input_event_id = %delivery.input.input_event_id, + delivery_id = %delivery.input.delivery_id, + recipient_runtime_id = %worker.runtime_id, + recipient_worker_id = %worker.worker_id, + error_category, + "committed delegation input remains pending; retry the saved request" + ); + emit_ticket_notification_delivery_warning(TicketNotificationDeliveryWarning::new( + &api.config.workspace_id, + &delivery.input.ticket_id, + "open", + worker, + error_category, + )); } } } +#[cfg(test)] fn orchestrator_queue_attention_context( expected_workspace_id: &str, candidate_workspace_id: &str, @@ -22403,11 +22575,22 @@ fn scoped_worker_control_operation_id(controller: &RuntimeWorkerRef, operation_i ) } +#[cfg(test)] async fn spawn_known_worker( + api: State, + path: AxumPath, + headers: HeaderMap, + request: Json, +) -> ApiResult> { + spawn_known_worker_authenticated(api, path, headers, request, None).await +} + +async fn spawn_known_worker_authenticated( State(api): State, AxumPath(path): AxumPath, headers: HeaderMap, Json(mut request): Json, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; @@ -22445,9 +22628,14 @@ async fn spawn_known_worker( operation_id: scoped_worker_control_operation_id(&controller, &operation_id), input_fingerprint, }); - let response = - create_workspace_worker_inner(api.clone(), headers, request, resolved_control_operation) - .await?; + let response = create_workspace_worker_authenticated( + api.clone(), + headers, + request, + resolved_control_operation, + context, + ) + .await?; if let Err(error) = api .store .create_worker_control_grant(&WorkerControlGrantRecord { @@ -22739,6 +22927,8 @@ async fn scoped_start_workspace_orchestrator( State(api): State, AxumPath(path): AxumPath, ) -> ApiResult> { + #[cfg(test)] + tests::seed_test_delivery_binding(&api); validate_workspace_scope(&api, &path.workspace_id)?; let _guard = api.orchestrator_spawn_lock.lock().map_err(|_| { ApiError::with_diagnostics( @@ -22755,8 +22945,9 @@ async fn scoped_start_workspace_orchestrator( if let Some(existing) = find_workspace_orchestrator(&api) { if workspace_orchestrator_is_online(&existing) { drop(_guard); - // Already-online is not a new startup. Only retry outstanding delivery. - dispatch_orchestrator_queue_attention(&api, &existing.worker, true); + // Already-online creation uses the same committed-input scan; neither + // local cache loss nor handled status alone creates new work. + dispatch_orchestrator_assignment_attention(&api, &existing.worker); return Ok(Json(workspace_orchestrator_response(&api, "existing"))); } let attention = orchestrator_attention_state(&api, &existing.worker); @@ -22785,7 +22976,7 @@ async fn scoped_start_workspace_orchestrator( } } drop(_guard); - dispatch_orchestrator_queue_attention(&api, &existing.worker, false); + dispatch_orchestrator_assignment_attention(&api, &existing.worker); return Ok(Json(workspace_orchestrator_response(&api, "restored"))); } if !diagnostics_indicate_unrecoverable_pending_workspace_restore(&restored.diagnostics) { @@ -22865,7 +23056,7 @@ async fn scoped_start_workspace_orchestrator( WorkerRegistryDisplayNamePolicy::UseProvided, )?; drop(_guard); - dispatch_orchestrator_queue_attention(&api, &worker.worker, false); + dispatch_orchestrator_assignment_attention(&api, &worker.worker); Ok(Json(workspace_orchestrator_response(&api, disposition))) } @@ -22942,6 +23133,7 @@ fn workspace_orchestrator_is_online(worker: &InternalWorkerSummary) -> bool { matches!(worker.state.as_str(), "idle" | "running" | "paused") } +#[cfg(test)] async fn scoped_create_workspace_worker( State(api): State, AxumPath(path): AxumPath, @@ -25885,6 +26077,7 @@ async fn scoped_list_runtime_workers( list_runtime_workers(State(api), AxumPath(path.runtime_id), Query(query)).await } +#[cfg(test)] async fn scoped_create_runtime_worker( State(api): State, AxumPath(path): AxumPath, @@ -27707,6 +27900,7 @@ fn browser_worker_spawn_policy( }, WorkerSpawnAcceptanceRequirement::RunAccepted { expected_segments }, Some(WorkerTicketAssignmentRequest { + orchestrator_assignment_id: assignment.orchestrator_assignment_id, ticket_id, operation_id, }), @@ -27754,6 +27948,7 @@ fn validate_caller_worker_singleton_key(singleton_key: Option<&str>) -> Result<( Ok(()) } +#[cfg(test)] async fn create_workspace_worker( State(api): State, headers: HeaderMap, @@ -27762,11 +27957,23 @@ async fn create_workspace_worker( create_workspace_worker_inner(api, headers, request, None).await } +#[cfg(test)] async fn create_workspace_worker_inner( api: WorkspaceApi, headers: HeaderMap, request: CreateWorkspaceWorkerRequest, resolved_control_operation: Option, +) -> ApiResult> { + create_workspace_worker_authenticated(api, headers, request, resolved_control_operation, None) + .await +} + +async fn create_workspace_worker_authenticated( + api: WorkspaceApi, + headers: HeaderMap, + request: CreateWorkspaceWorkerRequest, + resolved_control_operation: Option, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let claim_lock = request .ticket_assignment @@ -27787,7 +27994,23 @@ async fn create_workspace_worker_inner( feature_connections, control_operation_id: _, } = request; - let source = optional_worker_mutation_source(&api, &api.config.workspace_id, &headers)?; + let delegation = ticket_assignment + .as_ref() + .map(|assignment| { + ticket_spawn_delegation( + &api, + &headers, + context, + assignment.orchestrator_assignment_id.as_deref(), + ) + }) + .transpose()? + .flatten(); + let source = if let Some(context) = context { + authenticated_ticket_worker_source(&headers, context)? + } else { + optional_worker_mutation_source(&api, &api.config.workspace_id, &headers)? + }; let config_state = api .config_store .load_workspace_config(&api.config.workspace_id)? @@ -27920,14 +28143,15 @@ async fn create_workspace_worker_inner( if let (Some(assignment), Some(fingerprint)) = (assignment.as_ref(), assignment_fingerprint.as_deref()) { - fresh_ticket_claim = api.store.reserve_ticket_assignment_operation( - &api.config.workspace_id, + fresh_ticket_claim = reserve_ticket_start( + &api, &assignment.operation_id, &assignment.ticket_id, &runtime_id, None, fingerprint, &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + delegation.as_ref(), )?; let operation = api .store @@ -27955,6 +28179,12 @@ async fn create_workspace_worker_inner( .into()); } } + if let Some(assignment) = assignment.as_ref() { + api.store.validate_ticket_assignment_operation_fence( + &api.config.workspace_id, + &assignment.operation_id, + )?; + } let result = match api.spawn_workspace_worker_with_claim_admission( &runtime_id, request, @@ -28491,6 +28721,13 @@ fn resume_internal_worker_restore_intent( request: server_api::WorkerRestoreRequest, ) -> ApiResult { let request_id = request.request_id.clone(); + if request_id.starts_with("manual-worker-restore:") { + let operation_id = request_id + .strip_prefix("manual-worker-restore:") + .expect("checked prefix"); + api.store + .validate_ticket_assignment_operation_fence(&api.config.workspace_id, operation_id)?; + } let result = match WorkspaceWorker::resolve(api, &worker.runtime_id, &worker.worker_id)? .restore_internal(request) { @@ -29039,12 +29276,40 @@ fn runtime_diagnostics_message(diagnostics: &[RuntimeDiagnostic]) -> String { } } +#[cfg(test)] async fn create_runtime_worker( + api: State, + runtime_id: AxumPath, + request: Json, +) -> ApiResult> { + create_runtime_worker_authenticated(api, runtime_id, request, None).await +} + +async fn create_runtime_worker_authenticated( State(api): State, AxumPath(runtime_id): AxumPath, Json(request): Json, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let mut request = worker_spawn_request_from_api(request)?; + let headers = context + .map(contract_request_headers) + .transpose() + .map_err(|error| Error::InvalidInput(error.message))? + .unwrap_or_default(); + let delegation = request + .ticket_assignment + .as_ref() + .map(|assignment| { + ticket_spawn_delegation( + &api, + &headers, + context, + assignment.orchestrator_assignment_id.as_deref(), + ) + }) + .transpose()? + .flatten(); let claim_lock = request .ticket_assignment .as_ref() @@ -29111,14 +29376,15 @@ async fn create_runtime_worker( if let (Some(assignment), Some((_, fingerprint))) = (lifecycle_assignment.as_ref(), spawn_idempotency.as_ref()) { - fresh_ticket_claim = match api.store.reserve_ticket_assignment_operation( - &api.config.workspace_id, + fresh_ticket_claim = match reserve_ticket_start( + &api, &assignment.operation_id, &assignment.ticket_id, &runtime_id, None, fingerprint, &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + delegation.as_ref(), ) { Ok(fresh) => fresh, Err(error) => { @@ -29138,6 +29404,12 @@ async fn create_runtime_worker( diagnostics: Vec::new(), })?)); } + if let Some(assignment) = lifecycle_assignment.as_ref() { + api.store.validate_ticket_assignment_operation_fence( + &api.config.workspace_id, + &assignment.operation_id, + )?; + } let result = api.spawn_workspace_worker_with_claim_admission(&runtime_id, request, fresh_ticket_claim)?; if let Some(worker) = result.worker.as_ref() { @@ -32729,6 +33001,13 @@ impl IntoResponse for ApiError { #[cfg(test)] mod tests { + mod assignment_delivery_tests { + include!("server_assignment_delivery_tests.rs"); + } + + mod assignment_start_tests { + include!("server_assignment_start_tests.rs"); + } mod assignment_tracking_tests { include!("server_assignment_tracking_tests.rs"); } @@ -37337,6 +37616,7 @@ mod tests { Segment::text("Implement Ticket 00001KZ9E0DBS"), ]; let assignment_request = || CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: "00001KZ9E0DBS".to_string(), operation_id: "worker-spawn:00001KZ9E0DBS:call-1".to_string(), }; @@ -37359,6 +37639,7 @@ mod tests { assert_eq!( assignment, Some(WorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: "00001KZ9E0DBS".to_string(), operation_id: "worker-spawn:00001KZ9E0DBS:call-1".to_string(), }) @@ -37366,6 +37647,7 @@ mod tests { assert!( browser_worker_spawn_policy( Some(CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: " ".to_string(), operation_id: "operation".to_string(), }), @@ -37432,6 +37714,7 @@ mod tests { }, profile: ProfileSelector::Builtin("builtin:coder".to_string()), ticket_assignment: Some(WorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket.id, operation_id: "planning-spawn".to_string(), }), @@ -37532,6 +37815,7 @@ mod tests { singleton_key: None, profile: Some("builtin:coder".to_string()), ticket_assignment: Some(CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket.id.clone(), operation_id: "failed-queued-assignment".to_string(), }), @@ -48017,6 +48301,7 @@ mod tests { accept_queued_ticket_after_worker_spawn( &api, &crate::hosts::WorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket.id.clone(), operation_id: "notification-source-assignment-operation".to_string(), }, @@ -48047,27 +48332,37 @@ mod tests { } let inputs = execution.take_inputs(); - let expected_states = [ - ("open", "open"), - ("open", "open"), - ("open", "open"), - ("open", "open"), - ]; - assert_eq!(inputs.len(), expected_states.len()); - for ((recipient, content), (previous_state, current_state)) in - inputs.iter().zip(expected_states) - { + assert_eq!(inputs.len(), 4, "Assign input plus three external events"); + let unique = inputs + .iter() + .map(|(_, content)| content.clone()) + .collect::>(); + assert_eq!( + unique.len(), + inputs.len(), + "each committed input has its own delivery identity" + ); + for (recipient, content) in &inputs { assert_eq!(recipient.worker_id.to_string(), orchestrator.worker_id); - assert!(!content.contains(&ticket.id)); - assert_eq!( - content, - &ticket_notification_content( - ticket.resource_key.as_deref().unwrap(), - previous_state, - current_state, - ) - ); + assert!(content.starts_with("Assigned Tickets have unreceived input:")); + assert!(content.contains(ticket.resource_key.as_deref().unwrap())); + assert!(content.contains("Delegation:")); + assert!(content.contains("input:")); } + let tracking = api + .store + .list_current_ticket_assignment_tracking(TEST_WORKSPACE_ID, &ticket.id) + .unwrap(); + assert_eq!( + tracking[0].tracking.as_ref().unwrap().status, + ticket::TicketAssignmentTrackingStatus::Waiting + ); + assert!( + api.store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) + .unwrap() + .is_empty() + ); } fn case_test_request( @@ -48435,6 +48730,7 @@ mod tests { "orchestrator".to_string(), )), Json(server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, workdir_bindings: Vec::new(), operation_id: "fresh-orchestrator-cycle".to_string(), principal: server_api::TicketAssignmentPrincipal::WorkspaceAgent { @@ -48481,10 +48777,12 @@ mod tests { let inputs = execution.take_inputs(); assert_eq!(inputs.len(), 1); assert_eq!(inputs[0].0.worker_id.to_string(), orchestrator.worker_id); - assert_eq!( - inputs[0].1, - ticket_notification_content(ticket.resource_key.as_deref().unwrap(), "open", "open") + assert!( + inputs[0] + .1 + .starts_with("Assigned Tickets have unreceived input:") ); + assert!(inputs[0].1.contains(&fresh_id)); assert!( api.store .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &ticket.id) @@ -48606,6 +48904,7 @@ mod tests { operation_id: &str, ) -> server_api::SetTicketRoleAssignmentRequest { server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, workdir_bindings: fixture .api .store @@ -49014,6 +49313,7 @@ mod tests { .create(ticket::NewTicket::new("Idempotent assignment")) .unwrap(); let request = || server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, workdir_bindings: Vec::new(), operation_id: "same-role-operation".to_string(), principal: server_api::TicketAssignmentPrincipal::WorkspaceAgent { @@ -49070,6 +49370,7 @@ mod tests { "worker".to_string(), )), Json(server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, workdir_bindings: Vec::new(), operation_id: "invalid-workspace-agent-coder".to_string(), principal: server_api::TicketAssignmentPrincipal::WorkspaceAgent { @@ -49126,7 +49427,7 @@ mod tests { .iter() .any(|blocker| blocker.contains("Orchestrator and manual Worker")) ); - assert!(!detail.action_eligibility.can_assign_orchestrator); + assert!(detail.action_eligibility.can_assign_orchestrator); assert!(!detail.action_eligibility.can_start_manual_worker); let stale = scoped_clear_ticket_assignment( @@ -50038,19 +50339,21 @@ mod tests { assert!(!serialized.contains("Ticket notification:")); let notifications = execution.take_inputs(); - assert_eq!(notifications.len(), 1); + assert_eq!( + notifications.len(), + 2, + "initial delegation plus external comment" + ); assert_eq!( notifications[0].0.worker_id.to_string(), orchestrator.worker_id ); - assert_eq!( - notifications[0].1, - ticket_notification_content( - ticket_ref.resource_key.as_deref().unwrap(), - TicketState::Open.as_str(), - TicketState::Open.as_str() - ) + assert!( + notifications + .iter() + .all(|(_, content)| content.starts_with("Assigned Tickets have unreceived input:")) ); + assert_ne!(notifications[0].1, notifications[1].1); execution.reject_inputs("sensitive fake Runtime transport detail"); TICKET_NOTIFICATION_DELIVERY_WARNING_CAPTURE @@ -50091,7 +50394,11 @@ mod tests { .filter(|warning| warning.ticket_id == ticket_ref.id) .cloned() .collect::>(); - assert_eq!(warnings.len(), 2); + assert_eq!( + warnings.len(), + 3, + "one missing Worker and two bounded Orchestrator delivery attempts" + ); assert!(warnings.iter().any(|warning| { warning.recipient_worker_id == missing_recipient.worker_id && warning.error_category == "unknown_worker" @@ -50105,969 +50412,15 @@ mod tests { assert!(!serialized.contains("Ticket notification:")); assert!(!serialized.contains("sensitive fake Runtime transport detail")); let attempts = execution.take_inputs(); - assert_eq!(attempts.len(), 1); + assert_eq!(attempts.len(), 2); + assert_eq!( + attempts[0].1, attempts[1].1, + "bounded retry retains immutable request payload" + ); assert_eq!(attempts[0].0.worker_id.to_string(), orchestrator.worker_id); } - async fn attention_test_api() -> ( - tempfile::TempDir, - WorkspaceApi, - Arc, - ) { - let dir = tempfile::tempdir().unwrap(); - init_clean_git_workspace(dir.path()); - let (api, execution) = test_api_with_recording_backend(dir.path()).await; - let mut input = ticket::NewTicket::new("Attention backlog"); - input.workflow_state = Some(TicketState::Open); - set_test_ticket_target(&mut input, "test-repository", "HEAD"); - let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); - assign_test_orchestrator(&api, &ticket.id); - browser_ticket_backend(&api) - .unwrap() - .queue_ready(ticket.id.into(), "test-request") - .unwrap(); - (dir, api, execution) - } - - async fn start_attention_test_orchestrator( - api: &WorkspaceApi, - ) -> BrowserWorkspaceOrchestratorResponse { - scoped_start_workspace_orchestrator( - State(api.clone()), - AxumPath(ScopedWorkspacePath { - workspace_id: TEST_WORKSPACE_ID.to_string(), - }), - ) - .await - .unwrap() - .0 - } - - #[tokio::test] - async fn orchestrator_attention_parallel_start_and_idle_observation_send_once() { - let (_dir, api, execution) = attention_test_api().await; - let reentrant_api = api.clone(); - execution.before_input_returns(move |worker| { - use protocol::subscription::SubscriptionWorkerState as S; - // The callback is inside Runtime send_input. Neither initial Idle nor - // a real turn end may deadlock or issue another in-flight delivery. - maybe_dispatch_orchestrator_turn_end( - &reentrant_api, - &worker.worker_id.to_string(), - None, - S::Idle, - ); - maybe_dispatch_orchestrator_turn_end( - &reentrant_api, - &worker.worker_id.to_string(), - Some(S::Running), - S::Idle, - ); - }); - let barrier = Arc::new(std::sync::Barrier::new(8)); - let results = std::thread::scope(|scope| { - let joins = (0..8) - .map(|_| { - let api = api.clone(); - let barrier = barrier.clone(); - scope.spawn(move || { - barrier.wait(); - futures::executor::block_on(start_attention_test_orchestrator(&api)) - }) - }) - .collect::>(); - joins - .into_iter() - .map(|j| j.join().unwrap()) - .collect::>() - }); - assert_eq!( - results - .iter() - .filter(|r| r.disposition == "created") - .count(), - 1 - ); - assert_eq!( - results - .iter() - .filter(|r| r.disposition == "existing") - .count(), - 7 - ); - assert_eq!(execution.take_inputs().len(), 1); - assert_eq!(execution.take_input_request_ids().len(), 1); - assert_eq!( - start_attention_test_orchestrator(&api).await.disposition, - "existing" - ); - assert!(execution.take_inputs().is_empty()); - } - - #[tokio::test] - async fn orchestrator_attention_barrier_callers_share_unsent_reservation() { - let (_dir, api, execution) = attention_test_api().await; - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let state = orchestrator_attention_state(&api, &worker); - state.lock().unwrap().accepted = None; - execution.take_inputs(); - execution.take_input_request_ids(); - let barrier = Arc::new(std::sync::Barrier::new(8)); - let (entered_tx, entered_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel(); - execution.before_input_returns(move |_| { - entered_tx.send(()).unwrap(); - release_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - }); - std::thread::scope(|scope| { - let joins = (0..8) - .map(|_| { - let api = api.clone(); - let worker = worker.clone(); - let barrier = barrier.clone(); - scope.spawn(move || { - barrier.wait(); - dispatch_orchestrator_queue_attention(&api, &worker, false); - }) - }) - .collect::>(); - entered_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - assert!(state.lock().unwrap().sending); - assert!(state.lock().unwrap().accepted.is_none()); - // The success record does not exist yet. A ninth caller must still - // return immediately rather than compare/send twice or wait on Runtime. - dispatch_orchestrator_queue_attention(&api, &worker, false); - release_tx.send(()).unwrap(); - for join in joins { - join.join().unwrap(); - } - }); - assert_eq!(execution.take_inputs().len(), 1); - assert_eq!(execution.take_input_request_ids().len(), 1); - assert!(!state.lock().unwrap().sending); - assert!(state.lock().unwrap().accepted.is_some()); - } - - #[tokio::test] - async fn orchestrator_attention_restore_rechecks_reserved_noop_before_and_after_comparison() { - for pause_before_comparison in [true, false] { - let (_dir, api, execution) = attention_test_api().await; - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let state = orchestrator_attention_state(&api, &worker); - let old_id = state - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .request_id - .clone(); - execution.take_inputs(); - execution.take_input_request_ids(); - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - execution.accept_restores(); - let (entered_tx, entered_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel(); - std::thread::scope(|scope| { - let api_for_caller = api.clone(); - let worker_for_caller = worker.clone(); - let caller = scope.spawn(move || { - let mut first = true; - with_orchestrator_attention_reservation( - &api_for_caller, - &worker_for_caller, - false, - |state, pending| { - if first && pause_before_comparison { - entered_tx.send(()).unwrap(); - release_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - } - check_orchestrator_queue_attention( - &api_for_caller, - &worker_for_caller, - state, - pending, - ); - if first && !pause_before_comparison { - // Old fingerprint matched: check returned without sending, - // but the exact same production reservation still exists. - assert_eq!( - state.lock().unwrap().accepted.as_ref().unwrap().request_id, - old_id - ); - entered_tx.send(()).unwrap(); - release_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - } - first = false; - }, - ); - }); - entered_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - assert!(state.lock().unwrap().sending); - let restored = futures::executor::block_on(start_attention_test_orchestrator(&api)); - assert_eq!(restored.disposition, "restored"); - let deferred = state.lock().unwrap().restore_recheck; - let old_retired = state.lock().unwrap().accepted.is_none(); - let inputs_before_release = execution.take_inputs(); - release_tx.send(()).unwrap(); - caller.join().unwrap(); - assert!(deferred); - assert!(old_retired); - assert!(inputs_before_release.is_empty()); - }); - assert_eq!(execution.take_inputs().len(), 1); - assert_eq!(execution.take_input_request_ids().len(), 1); - let state = state.lock().unwrap(); - assert!(state.accepted.is_some()); - assert!(!state.sending); - assert!(!state.restore_recheck); - } - } - - #[tokio::test] - async fn orchestrator_attention_restore_keeps_pending_unknown_delivery_and_id() { - let (_dir, api, execution) = attention_test_api().await; - execution.reject_inputs("unknown acceptance response"); - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let state = orchestrator_attention_state(&api, &worker); - let pending = state.lock().unwrap().pending.clone().unwrap(); - execution.take_inputs(); - execution.take_input_request_ids(); - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - execution.accept_restores(); - *execution.input_failure.lock().unwrap() = None; - let restored = start_attention_test_orchestrator(&api).await; - assert_eq!(restored.disposition, "restored"); - let attempts = execution.take_inputs(); - assert_eq!(attempts.len(), 1); - assert_eq!(attempts[0].1, pending.content); - assert_eq!( - execution.take_input_request_ids(), - vec![Some(pending.request_id.clone())] - ); - assert!(state.lock().unwrap().pending.is_none()); - assert_eq!( - state.lock().unwrap().accepted.as_ref().unwrap().request_id, - pending.request_id - ); - } - - #[tokio::test] - async fn orchestrator_attention_unknown_response_retry_keeps_id_and_payload() { - let (_dir, api, execution) = attention_test_api().await; - execution.reject_inputs("accepted response lost (unknown transport outcome)"); - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let state = orchestrator_attention_state(&api, &worker); - assert!(state.lock().unwrap().accepted.is_none()); - let pending = state.lock().unwrap().pending.clone().unwrap(); - let attempts = execution.take_inputs(); - let ids = execution.take_input_request_ids(); - assert_eq!(attempts.len(), 2); - assert_eq!(ids, vec![Some(pending.request_id.clone()); 2]); - // Change the backlog before retry: uncertain acceptance must never reuse - // the ID for a newly rendered payload. - let mut next = ticket::NewTicket::new("Changed backlog"); - next.workflow_state = Some(TicketState::Open); - set_test_ticket_target(&mut next, "test-repository", "HEAD"); - let next = browser_ticket_backend(&api).unwrap().create(next).unwrap(); - assign_test_orchestrator(&api, &next.id); - browser_ticket_backend(&api) - .unwrap() - .queue_ready(next.id.clone().into(), "test-request") - .unwrap(); - *execution.input_failure.lock().unwrap() = None; - assert_eq!( - start_attention_test_orchestrator(&api).await.disposition, - "existing" - ); - let retry = execution.take_inputs(); - assert_eq!(retry.len(), 1); - assert_eq!(retry[0].1, pending.content); - assert_eq!( - execution.take_input_request_ids(), - vec![Some(pending.request_id)] - ); - assert!(state.lock().unwrap().pending.is_none()); - assert!(state.lock().unwrap().accepted.is_some()); - use protocol::subscription::SubscriptionWorkerState as S; - maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, Some(S::Running), S::Idle); - let changed = execution.take_inputs(); - assert_eq!(changed.len(), 1); - assert!(changed[0].1.contains("Changed backlog")); - } - - #[tokio::test] - async fn orchestrator_attention_subscription_snapshots_only_establish_baselines() { - use crate::runtime_subscription::BrokerSubscriptionEvent as B; - use protocol::subscription::{ - SubscriptionEventPayload, SubscriptionSnapshot, SubscriptionWorker, - SubscriptionWorkerState as S, - }; - let (_dir, api, execution) = attention_test_api().await; - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let attention = orchestrator_attention_state(&api, &worker); - attention.lock().unwrap().accepted = None; - execution.take_inputs(); - let summary = |state: S| -> SubscriptionWorker { - serde_json::from_value(serde_json::json!({ - "worker_id": worker.worker_id, "state": state, - })) - .unwrap() - }; - let snapshot = |state| B::Snapshot { - snapshot: SubscriptionSnapshot::Workers { - workers: vec![summary(state)], - }, - }; - let event = |state| B::Event { - payload: SubscriptionEventPayload::WorkerUpserted { - worker: summary(state), - }, - }; - let mut states = HashMap::new(); - assert!(observe_orchestrator_turn_end( - &api, - &mut states, - snapshot(S::Idle) - )); - assert!(observe_orchestrator_turn_end( - &api, - &mut states, - event(S::Idle) - )); - observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); - // Neither snapshot itself nor an Idle event after reconnect is turn-end evidence. - observe_orchestrator_turn_end( - &api, - &mut states, - B::Disconnected { - message: "fixture reconnect".into(), - }, - ); - observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); - observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); - observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Running)); - observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); - assert!(attention.lock().unwrap().accepted.is_none()); - assert!(execution.take_inputs().is_empty()); - observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); - observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); - assert_eq!(execution.take_inputs().len(), 1); - } - - // Real Controller and canonical Session storage with isolated local config. - // A closed loopback model endpoint prevents any production-provider access; - // notifications commit at the model boundary before that request can fail. - struct AttentionWorkerFactory { - root: PathBuf, - } - - impl AttentionWorkerFactory { - async fn controller( - &self, - worker_id: String, - restore: bool, - ) -> std::result::Result - { - let name = format!("attention-{worker_id}"); - let root = self.root.join("workers").join(&worker_id); - let store = session_store::CombinedStore::new( - session_store::WorkerSessionStore::new(root.join("session")) - .map_err(|e| e.to_string())?, - session_store::WorkerAggregateStore::new(&root, &name) - .map_err(|e| e.to_string())?, - ); - let manifest = manifest::WorkerManifest::from_toml(&format!( - r#" - [worker] - name = "{name}" - pwd = "./" - [model] - scheme = "anthropic" - model_id = "attention-test" - base_url = "http://127.0.0.1:1" - auth = {{ kind = "none" }} - [engine] - max_tokens = 10 - [scope] - allow = [] - "# - )) - .map_err(|e| e.to_string())?; - let context = worker::WorkerWorkspaceContext::local_filesystem(None); - let workspace_client = context.client_handle(); - let loader = worker::PromptCatalogSource::builtins_only(); - let worker = if restore { - worker::Worker::restore_from_worker_metadata_with_context( - &name, - manifest, - store, - loader, - context, - worker::WorkerFilesystemAuthority::None, - ) - .await - } else { - worker::Worker::from_manifest_with_context( - manifest, - store, - loader, - context, - worker::WorkerFilesystemAuthority::None, - ) - .await - } - .map_err(|e| e.to_string())?; - let started = worker::PreparedWorker::new( - worker.with_notification_coalesce_delay(std::time::Duration::from_millis(10)), - worker::WorkerBootstrapLayout::RuntimeManagedRun { - run_dir: root.join(uuid::Uuid::now_v7().to_string()), - bash_output_dir: root.join("bash-output"), - }, - worker::WorkerControllerTransport::InProcess, - ) - .start() - .await - .map_err(|e| e.to_string())?; - Ok(worker_runtime::worker_backend::RuntimeWorkerController { - handle: started.handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), - controller_task: started.controller_task, - workspace_client, - }) - } - } - - #[async_trait::async_trait] - impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFactory { - async fn spawn_controller( - &self, - request: worker_runtime::execution::WorkerExecutionSpawnRequest, - ) -> std::result::Result - { - self.controller(request.worker_ref.worker_id.to_string(), false) - .await - } - async fn restore_controller( - &self, - request: worker_runtime::execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result - { - self.controller(request.worker_ref.worker_id.to_string(), true) - .await - } - } - - use worker_runtime::execution as attention_execution; - - // Fault injection occurs AFTER the real Controller durably accepted input. - struct AttentionResponseLossBackend { - inner: Arc, - lose_next: std::sync::atomic::AtomicBool, - attempts: Mutex>, - } - - impl attention_execution::WorkerExecutionBackend for AttentionResponseLossBackend { - fn backend_id(&self) -> &str { - self.inner.backend_id() - } - fn spawn_worker( - &self, - r: attention_execution::WorkerExecutionSpawnRequest, - ) -> attention_execution::WorkerExecutionSpawnResult { - self.inner.spawn_worker(r) - } - fn preflight_restore( - &self, - r: &attention_execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result<(), attention_execution::WorkerExecutionResult> { - self.inner.preflight_restore(r) - } - fn reconcile_restore( - &self, - r: attention_execution::WorkerExecutionRestoreRequest, - ) -> attention_execution::WorkerExecutionSpawnResult { - self.inner.reconcile_restore(r) - } - fn restore_worker( - &self, - r: attention_execution::WorkerExecutionRestoreRequest, - ) -> attention_execution::WorkerExecutionSpawnResult { - self.inner.restore_worker(r) - } - fn activate_restored_worker( - &self, - id: attention_execution::WorkerLifecycleOperationId, - worker_ref: &worker_runtime::identity::WorkerRef, - ) -> std::result::Result<(), String> { - self.inner.activate_restored_worker(id, worker_ref) - } - fn dispatch_input( - &self, - worker_ref: &worker_runtime::identity::WorkerRef, - input: worker_runtime::interaction::WorkerInput, - ) -> attention_execution::WorkerExecutionResult { - self.attempts - .lock() - .unwrap() - .push(input.submission_request_id.clone().unwrap()); - let result = self.inner.dispatch_input(worker_ref, input); - assert_eq!( - result.outcome, - attention_execution::WorkerExecutionOutcome::Accepted - ); - if self - .lose_next - .swap(false, std::sync::atomic::Ordering::SeqCst) - { - attention_execution::WorkerExecutionResult::errored( - attention_execution::WorkerExecutionOperation::Input, - "injected loss of accepted response", - ) - } else { - result - } - } - fn dispatch_method( - &self, - worker_ref: &worker_runtime::identity::WorkerRef, - method: protocol::Method, - ) -> attention_execution::WorkerExecutionResult { - self.inner.dispatch_method(worker_ref, method) - } - fn stop_worker_operation( - &self, - r: attention_execution::WorkerExecutionStopRequest, - ) -> attention_execution::WorkerExecutionResult { - self.inner.stop_worker_operation(r) - } - fn worker_snapshot( - &self, - worker_ref: &worker_runtime::identity::WorkerRef, - ) -> Option { - self.inner.worker_snapshot(worker_ref) - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn orchestrator_attention_real_worker_start_restore_accept_and_commit_once_each() { - real_attention_worker_start_restore(false).await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() { - real_attention_worker_start_restore(true).await; - } - - async fn real_attention_worker_start_restore(lose_response: bool) { - use session_store::{LogEntry, Store, WorkerSessionStore}; - let dir = tempfile::tempdir().unwrap(); - init_clean_git_workspace(dir.path()); - let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); - seed_test_registered_workspace(store.as_ref(), &config) - .await - .unwrap(); - // Real Worker Runtime adapter/controller/storage, isolated from profile - // workspace callbacks and real LLM credentials. - let execution = Arc::new( - worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new( - AttentionWorkerFactory { - root: config.embedded_runtime_store_root.clone(), - }, - ) - .unwrap(), - ); - let execution = Arc::new(AttentionResponseLossBackend { - inner: execution, - lose_next: std::sync::atomic::AtomicBool::new(lose_response), - attempts: Mutex::new(Vec::new()), - }); - let api = WorkspaceApi::new_with_execution_backend(config, store, execution.clone()) - .await - .unwrap(); - let mut input = ticket::NewTicket::new("Real notification backlog"); - input.workflow_state = Some(TicketState::Open); - set_test_ticket_target(&mut input, "test-repository", "HEAD"); - let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); - assign_test_orchestrator(&api, &ticket.id); - browser_ticket_backend(&api) - .unwrap() - .queue_ready(ticket.id.clone().into(), "test-request") - .unwrap(); - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let session = WorkerSessionStore::open_read_only( - api.config - .embedded_runtime_store_root - .join("workers") - .join(&worker.worker_id) - .join("session"), - ) - .unwrap(); - let entries = || { - let id = session.session_id().unwrap().unwrap(); - session - .list_segments(id) - .unwrap() - .into_iter() - .flat_map(|segment| session.read_all_read_only(id, segment).unwrap()) - .collect::>() - }; - let receipts = || { - entries() - .iter() - .rev() - .find_map(|entry| match entry { - LogEntry::Extension { - domain, payload, .. - } if domain == "worker.pending_activations.v1" => { - Some(payload["notification_receipts"].as_array().unwrap().clone()) - } - LogEntry::AnnotatedSystemItem { extensions, .. } => extensions - .iter() - .find(|e| e.domain == "worker.pending_activations.v1") - .map(|e| { - e.payload["notification_receipts"] - .as_array() - .unwrap() - .clone() - }), - _ => None, - }) - .unwrap() - }; - let commits = || { - entries().iter().filter(|entry| matches!(entry, - LogEntry::AnnotatedSystemItem { entry, .. } - if matches!(&entry.item, session_store::SystemItem::Notification { message, .. } - if message.starts_with("Queued Tickets require attention:")) - )).count() - }; - async fn wait_for_commits usize>(count: F, expected: usize) { - tokio::time::timeout(std::time::Duration::from_secs(5), async { - loop { - let actual = count(); - assert!(actual <= expected, "duplicate canonical notifications"); - if actual == expected { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - }) - .await - .expect("controlled Worker notification commit"); - } - assert_eq!(receipts().len(), 1); - wait_for_commits(&commits, 1).await; - assert_eq!( - execution.attempts.lock().unwrap().len(), - if lose_response { 2 } else { 1 } - ); - let first_id = orchestrator_attention_state(&api, &worker) - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .request_id - .clone(); - use protocol::subscription::SubscriptionWorkerState as S; - for previous in [ - None, - Some(S::Idle), - Some(S::Stopped), - Some(S::Paused), - Some(S::Running), - ] { - maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, previous, S::Idle); - } - assert_eq!( - start_attention_test_orchestrator(&api).await.disposition, - "existing" - ); - assert_eq!(receipts().len(), 1); - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - let restored = start_attention_test_orchestrator(&api).await; - assert_eq!(restored.disposition, "restored"); - let second_id = orchestrator_attention_state(&api, &worker) - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .request_id - .clone(); - assert_ne!(first_id, second_id); - wait_for_commits(&commits, 2).await; - let attempts = execution.attempts.lock().unwrap(); - assert_eq!(attempts.len(), if lose_response { 3 } else { 2 }); - assert_eq!( - attempts.iter().filter(|id| **id == first_id).count(), - if lose_response { 2 } else { 1 } - ); - assert_eq!(attempts.iter().filter(|id| **id == second_id).count(), 1); - drop(attempts); - let receipts = receipts(); - assert_eq!(receipts.len(), 2); - assert_eq!( - receipts - .iter() - .filter(|r| r["notification_request_id"] == first_id) - .count(), - 1 - ); - assert_eq!( - receipts - .iter() - .filter(|r| r["notification_request_id"] == second_id) - .count(), - 1 - ); - // Stop only this isolated test fixture; no live dogfooding Worker is touched. - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - } - - #[tokio::test] - async fn orchestrator_attention_restore_and_replacement_have_first_delivery() { - let (_dir, api, execution) = attention_test_api().await; - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - assert_eq!(execution.take_inputs().len(), 1); - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - execution.accept_restores(); - let restored = start_attention_test_orchestrator(&api).await; - assert_eq!(restored.disposition, "restored"); - assert_eq!(restored.worker.unwrap().worker_id, worker.worker_id); - assert_eq!(execution.take_inputs().len(), 1); - use protocol::subscription::SubscriptionWorkerState as S; - maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, None, S::Idle); - assert_eq!( - start_attention_test_orchestrator(&api).await.disposition, - "existing" - ); - assert!(execution.take_inputs().is_empty()); - // This is a fixture Worker, not the running dogfooding Orchestrator. - api.runtime.delete_worker(&worker).unwrap(); - api.worker_projection - .publish_ordered(|| { - api.store - .delete_worker_registry(TEST_WORKSPACE_ID, &worker) - .map(|commit| ((), commit)) - }) - .unwrap(); - let replacement = start_attention_test_orchestrator(&api).await; - assert_ne!(replacement.worker.unwrap().worker_id, worker.worker_id); - assert_eq!(execution.take_inputs().len(), 1); - } - - #[tokio::test] - async fn orchestrator_running_to_idle_recovers_queued_ticket_without_notification_memory() { - let dir = tempfile::tempdir().unwrap(); - init_clean_git_workspace(dir.path()); - let (api, execution) = test_api_with_recording_backend(dir.path()).await; - let backend = browser_ticket_backend(&api).unwrap(); - let mut input = ticket::NewTicket::new("Recover queued work"); - input.workflow_state = Some(TicketState::Open); - set_test_ticket_target(&mut input, "test-repository", "HEAD"); - let ticket_ref = backend.create(input).unwrap(); - assign_test_orchestrator(&api, &ticket_ref.id); - browser_ticket_backend(&api) - .unwrap() - .queue_ready(ticket_ref.id.clone().into(), "test-request") - .unwrap(); - - let Json(started) = scoped_start_workspace_orchestrator( - State(api.clone()), - AxumPath(ScopedWorkspacePath { - workspace_id: TEST_WORKSPACE_ID.to_string(), - }), - ) - .await - .unwrap(); - assert!(started.online); - let startup_inputs = execution.take_inputs(); - assert_eq!(startup_inputs.len(), 1); - let worker_id = started.worker.as_ref().unwrap().worker_id.clone(); - let worker = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, &worker_id); - let attention = orchestrator_attention_state(&api, &worker); - assert_eq!( - attention - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .fingerprint, - ticket_ref.id - ); - attention.lock().unwrap().accepted = None; - use protocol::subscription::SubscriptionWorkerState as S; - // Snapshot/reconnection and non-turn transitions cannot stand in for startup. - for previous in [None, Some(S::Idle), Some(S::Stopped), Some(S::Paused)] { - maybe_dispatch_orchestrator_turn_end(&api, &worker_id, previous, S::Idle); - assert!(attention.lock().unwrap().accepted.is_none()); - assert!(execution.take_inputs().is_empty()); - } - maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); - assert_eq!( - attention - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .fingerprint, - ticket_ref.id - ); - maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); - let notifications = execution.take_inputs(); - assert_eq!(notifications.len(), 1); - assert_eq!(notifications[0].0.worker_id.to_string(), worker_id); - let content = ¬ifications[0].1; - assert!(content.starts_with("Queued Tickets require attention:")); - assert!( - content.contains(&format!( - "- {} — Recover queued work", - ticket_ref.resource_key.as_deref().unwrap() - )), - "unexpected notification body: {content:?}" - ); - assert!(content.contains("Reread the current Ticket state before acting")); - assert!(!content.contains(ticket_ref.id.as_str())); - assert!(!content.contains(TEST_WORKSPACE_ID)); - assert!(!content.contains("bounded")); - assert!(!content.contains("omitted")); - - let candidates = backend - .list(ticket::TicketListQuery::states([ - ticket::TicketListState::Open, - ])) - .unwrap(); - let mut truncated_candidates = (1..=worker::OrchestratorQueueAttentionContext::MAX_TICKETS - + 1) - .map(|index| { - let mut candidate = candidates[0].clone(); - candidate.id = format!("opaque-{index}"); - candidate.resource_key = Some(format!("T-{index}")); - candidate.title = format!("Queued {index}"); - candidate - }) - .collect::>(); - let truncated = orchestrator_queue_attention_context( - TEST_WORKSPACE_ID, - TEST_WORKSPACE_ID, - &truncated_candidates, - ) - .unwrap(); - let rendered = worker::PromptCatalog::builtins_only() - .unwrap() - .orchestrator_queue_attention( - worker::OrchestratorQueueAttentionPrompt::Server, - &truncated, - ) - .unwrap(); - assert!(rendered.contains("- T-20 — Queued 20")); - assert!(!rendered.contains("T-21")); - assert!(rendered.contains("were omitted from this notice: 1")); - assert!(!rendered.contains("opaque-")); - - truncated_candidates[0].resource_key = None; - assert_eq!( - orchestrator_queue_attention_context( - TEST_WORKSPACE_ID, - TEST_WORKSPACE_ID, - &truncated_candidates - ) - .unwrap_err(), - "missing_ticket_resource_key" - ); - assert_eq!( - orchestrator_queue_attention_context( - TEST_WORKSPACE_ID, - "foreign-workspace", - &truncated_candidates - ) - .unwrap_err(), - "foreign_workspace_ticket_projection" - ); - } + include!("server_assignment_attention_tests.rs"); #[tokio::test] async fn runtime_worker_create_rejects_reserved_singleton_key_squatting_and_authority_forgery() @@ -51166,6 +50519,7 @@ mod tests { }, profile: server_api::RuntimeProfileSelector::Builtin("builtin:intake".to_string()), ticket_assignment: Some(server_api::RuntimeWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket.id.clone(), operation_id: "runtime-explicit-work".to_string(), }), @@ -51256,6 +50610,7 @@ mod tests { ticket_id ); let mut assignment = Some(CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket_resource_key.clone(), operation_id: "ticket-key-assignment".to_string(), }); @@ -51806,6 +51161,36 @@ mod tests { assert_eq!(stale.into_response().status(), StatusCode::CONFLICT); } + pub(super) fn seed_test_delivery_binding(api: &WorkspaceApi) { + if api + .config_store + .get_workspace_runtime_binding(TEST_WORKSPACE_ID, EMBEDDED_WORKER_RUNTIME_ID) + .unwrap() + .is_some() + { + return; + } + let identity = (*EMBEDDED_RUNTIME_REQUEST_IDENTITY).clone(); + api.config_store + .upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: TEST_WORKSPACE_ID.into(), + runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), + display_name: "Embedded test Runtime".into(), + base_url: "in-process://embedded".into(), + public_key: identity.public_key, + public_key_fingerprint: String::new(), + binding_id: "binding-test".into(), + authentication_mode: StoredRuntimeAuthenticationMode::LegacyServerIssuer, + created_at: TEST_CREATED_AT.into(), + updated_at: TEST_CREATED_AT.into(), + revoked_at: None, + }, + false, + ) + .unwrap(); + } + async fn test_api_with_recording_backend( workspace_root: impl Into, ) -> (WorkspaceApi, Arc) { diff --git a/crates/workspace-server/src/server/tests/ticket_worker_claim_tests.rs b/crates/workspace-server/src/server/tests/ticket_worker_claim_tests.rs index ae3340877..1e55f7564 100644 --- a/crates/workspace-server/src/server/tests/ticket_worker_claim_tests.rs +++ b/crates/workspace-server/src/server/tests/ticket_worker_claim_tests.rs @@ -448,6 +448,7 @@ async fn pending_manual_worker_claim_reconciles_same_runtime_restore_and_commits .unwrap(); let admitted_content_digest = api.authority.ticket(&ticket_id).unwrap().content_digest; let request = server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, operation_id: "claim-runtime-reconciliation".into(), principal: server_api::TicketAssignmentPrincipal::Worker { runtime_id: worker.runtime_id.clone(), diff --git a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs index 906ca566a..64ca15dc3 100644 --- a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs +++ b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs @@ -1287,23 +1287,33 @@ async fn uncertain_launch_replay_after_target_edit_keeps_claim_and_blocks_compet // Direct policy composition is intentional: this protects routing interest, // independently of the transport/Runtime launch assertions above. #[tokio::test(flavor = "multi_thread")] -async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_just_open_state() { +async fn assignment_inputs_replace_queue_interest_without_reviving_released_work() { let fixture = LaunchFixture::new(false).await; let backend = browser_ticket_backend(&fixture.api).unwrap(); let queued_id = fixture.ticket(); assign_test_orchestrator(&fixture.api, &queued_id); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &queued_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_none() + .iter() + .any(|input| input.ticket_id == queued_id) + == true ); backend .queue_ready(queued_id.clone().into(), "fixture") .unwrap(); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &queued_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_some() + .iter() + .any(|input| input.ticket_id == queued_id) + == true ); fixture.case_action( &queued_id, @@ -1313,9 +1323,14 @@ async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_jus }, ); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &queued_id, "closed") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_none() + .iter() + .any(|input| input.ticket_id == queued_id) + == false ); fixture.case_action( &queued_id, @@ -1323,9 +1338,14 @@ async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_jus ticket::TicketCaseAction::Reopen, ); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &queued_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_none() + .iter() + .any(|input| input.ticket_id == queued_id) + == false ); let active_id = fixture.ticket(); @@ -1347,9 +1367,14 @@ async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_jus .all(|event| !event.attributes.contains_key("queue_root_ticket")) ); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &active_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_some() + .iter() + .any(|input| input.ticket_id == active_id) + == true ); fixture.case_action( &active_id, @@ -1380,8 +1405,13 @@ async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_jus .is_none() ); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &active_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_none() + .iter() + .any(|input| input.ticket_id == active_id) + == false ); } diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs new file mode 100644 index 000000000..6da70dc10 --- /dev/null +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -0,0 +1,1073 @@ +async fn attention_test_api() -> ( + tempfile::TempDir, + WorkspaceApi, + Arc, +) { + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + attention_assign_ticket(&api, "Attention backlog"); + (dir, api, execution) +} + +fn attention_assign_ticket(api: &WorkspaceApi, title: &str) -> ticket::TicketRef { + let mut input = ticket::NewTicket::new(title); + input.workflow_state = Some(TicketState::Open); + set_test_ticket_target(&mut input, "test-repository", "HEAD"); + let ticket = browser_ticket_backend(api).unwrap().create(input).unwrap(); + assign_test_orchestrator(api, &ticket.id); + ticket +} + +fn attention_pending(api: &WorkspaceApi) -> Vec { + api.store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) + .unwrap() +} + +async fn start_attention_test_orchestrator( + api: &WorkspaceApi, +) -> BrowserWorkspaceOrchestratorResponse { + scoped_start_workspace_orchestrator( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + ) + .await + .unwrap() + .0 +} + +#[tokio::test] +async fn orchestrator_attention_parallel_start_and_idle_observation_send_once() { + let (_dir, api, execution) = attention_test_api().await; + let reentrant_api = api.clone(); + execution.before_input_returns(move |worker| { + use protocol::subscription::SubscriptionWorkerState as S; + // The callback is inside Runtime send_input. Neither initial Idle nor + // a real turn end may deadlock or issue another in-flight delivery. + maybe_dispatch_orchestrator_turn_end( + &reentrant_api, + &worker.worker_id.to_string(), + None, + S::Idle, + ); + maybe_dispatch_orchestrator_turn_end( + &reentrant_api, + &worker.worker_id.to_string(), + Some(S::Running), + S::Idle, + ); + }); + let barrier = Arc::new(std::sync::Barrier::new(8)); + let results = std::thread::scope(|scope| { + let joins = (0..8) + .map(|_| { + let api = api.clone(); + let barrier = barrier.clone(); + scope.spawn(move || { + barrier.wait(); + futures::executor::block_on(start_attention_test_orchestrator(&api)) + }) + }) + .collect::>(); + joins + .into_iter() + .map(|j| j.join().unwrap()) + .collect::>() + }); + assert_eq!( + results + .iter() + .filter(|r| r.disposition == "created") + .count(), + 1 + ); + assert_eq!( + results + .iter() + .filter(|r| r.disposition == "existing") + .count(), + 7 + ); + assert_eq!(execution.take_inputs().len(), 1); + assert_eq!(execution.take_input_request_ids().len(), 1); + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + assert!(execution.take_inputs().is_empty()); +} + +#[tokio::test] +async fn orchestrator_attention_barrier_callers_share_unsent_reservation() { + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + state.lock().unwrap().accepted = None; + attention_assign_ticket(&api, "Unreceived concurrent input"); + execution.take_inputs(); + execution.take_input_request_ids(); + let barrier = Arc::new(std::sync::Barrier::new(8)); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + execution.before_input_returns(move |_| { + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + }); + std::thread::scope(|scope| { + let joins = (0..8) + .map(|_| { + let api = api.clone(); + let worker = worker.clone(); + let barrier = barrier.clone(); + scope.spawn(move || { + barrier.wait(); + dispatch_orchestrator_assignment_attention(&api, &worker); + }) + }) + .collect::>(); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert!(state.lock().unwrap().sending); + assert!(state.lock().unwrap().accepted.is_none()); + // The success record does not exist yet. A ninth caller must still + // return immediately rather than compare/send twice or wait on Runtime. + dispatch_orchestrator_assignment_attention(&api, &worker); + release_tx.send(()).unwrap(); + for join in joins { + join.join().unwrap(); + } + }); + assert_eq!(execution.take_inputs().len(), 1); + assert_eq!(execution.take_input_request_ids().len(), 1); + assert!(!state.lock().unwrap().sending); + assert!(state.lock().unwrap().accepted.is_some()); +} + +#[tokio::test] +async fn orchestrator_attention_restore_rechecks_reserved_noop_before_and_after_comparison() { + for pause_before_comparison in [true, false] { + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + let acknowledged = state.lock().unwrap().accepted.as_ref().unwrap().clone(); + assert!(attention_pending(&api).is_empty()); + execution.take_inputs(); + execution.take_input_request_ids(); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + execution.accept_restores(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + std::thread::scope(|scope| { + let api_for_caller = api.clone(); + let worker_for_caller = worker.clone(); + let caller = scope.spawn(move || { + let mut checks = 0; + with_orchestrator_attention_reservation( + &api_for_caller, + &worker_for_caller, + |state, pending| { + if checks == 0 && pause_before_comparison { + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + check_orchestrator_assignment_attention( + &api_for_caller, + &worker_for_caller, + state, + pending, + ); + if checks == 0 && !pause_before_comparison { + // Durable acknowledgement matched: check returned without sending, + // but the exact same production reservation still exists. + assert_eq!( + state.lock().unwrap().accepted.as_ref().unwrap().request_id, + acknowledged.request_id + ); + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + checks += 1; + assert!(attention_pending(&api_for_caller).is_empty()); + }, + ); + assert_eq!(checks, 2, "restore must recheck the reserved no-op"); + }); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert!(state.lock().unwrap().sending); + let restored = futures::executor::block_on(start_attention_test_orchestrator(&api)); + assert_eq!(restored.disposition, "restored"); + let deferred = state.lock().unwrap().restore_recheck; + let old_retired = state.lock().unwrap().accepted.is_none(); + let inputs_before_release = execution.take_inputs(); + release_tx.send(()).unwrap(); + caller.join().unwrap(); + assert!(deferred); + assert!(old_retired); + assert!(inputs_before_release.is_empty()); + }); + assert!(execution.take_inputs().is_empty()); + assert!(execution.take_input_request_ids().is_empty()); + assert!(attention_pending(&api).is_empty()); + let state = state.lock().unwrap(); + assert!(state.accepted.is_none()); + assert!(!state.sending); + assert!(!state.restore_recheck); + } +} + +#[tokio::test] +async fn orchestrator_attention_restore_keeps_pending_unknown_delivery_and_id() { + let (_dir, api, execution) = attention_test_api().await; + execution.reject_inputs("unknown acceptance response"); + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + let pending = state.lock().unwrap().pending.clone().unwrap(); + execution.take_inputs(); + execution.take_input_request_ids(); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + execution.accept_restores(); + *execution.input_failure.lock().unwrap() = None; + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + let attempts = execution.take_inputs(); + assert_eq!(attempts.len(), 1); + assert_eq!(attempts[0].1, pending.content); + assert_eq!( + execution.take_input_request_ids(), + vec![Some(pending.request_id.clone())] + ); + assert!(state.lock().unwrap().pending.is_none()); + assert_eq!( + state.lock().unwrap().accepted.as_ref().unwrap().request_id, + pending.request_id + ); +} + +#[tokio::test] +async fn orchestrator_attention_unknown_response_retry_keeps_id_and_payload() { + let (_dir, api, execution) = attention_test_api().await; + execution.reject_inputs("accepted response lost (unknown transport outcome)"); + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + assert!(state.lock().unwrap().accepted.is_none()); + let pending = state.lock().unwrap().pending.clone().unwrap(); + let attempts = execution.take_inputs(); + let ids = execution.take_input_request_ids(); + assert_eq!(attempts.len(), 2); + assert_eq!(ids, vec![Some(pending.request_id.clone()); 2]); + // Change the backlog before retry: uncertain acceptance must never reuse + // the ID for a newly rendered payload. + attention_assign_ticket(&api, "Changed backlog"); + *execution.input_failure.lock().unwrap() = None; + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + let retry = execution.take_inputs(); + assert_eq!(retry.len(), 1); + assert_eq!(retry[0].1, pending.content); + assert_eq!( + execution.take_input_request_ids(), + vec![Some(pending.request_id)] + ); + assert!(state.lock().unwrap().pending.is_none()); + assert!(state.lock().unwrap().accepted.is_some()); + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, Some(S::Running), S::Idle); + let changed = execution.take_inputs(); + assert_eq!(changed.len(), 1); + assert!(changed[0].1.contains("Changed backlog")); +} + +#[tokio::test] +async fn orchestrator_attention_subscription_snapshots_only_establish_baselines() { + use crate::runtime_subscription::BrokerSubscriptionEvent as B; + use protocol::subscription::{ + SubscriptionEventPayload, SubscriptionSnapshot, SubscriptionWorker, + SubscriptionWorkerState as S, + }; + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let attention = orchestrator_attention_state(&api, &worker); + attention.lock().unwrap().accepted = None; + execution.take_inputs(); + attention_assign_ticket(&api, "Unreceived after reconnect"); + let summary = |state: S| -> SubscriptionWorker { + serde_json::from_value(serde_json::json!({ + "worker_id": worker.worker_id, "state": state, + })) + .unwrap() + }; + let snapshot = |state| B::Snapshot { + snapshot: SubscriptionSnapshot::Workers { + workers: vec![summary(state)], + }, + }; + let event = |state| B::Event { + payload: SubscriptionEventPayload::WorkerUpserted { + worker: summary(state), + }, + }; + let mut states = HashMap::new(); + assert!(observe_orchestrator_turn_end( + &api, + &mut states, + snapshot(S::Idle) + )); + assert!(observe_orchestrator_turn_end( + &api, + &mut states, + event(S::Idle) + )); + observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); + // Neither snapshot itself nor an Idle event after reconnect is turn-end evidence. + observe_orchestrator_turn_end( + &api, + &mut states, + B::Disconnected { + message: "fixture reconnect".into(), + }, + ); + observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); + observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); + observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Running)); + observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); + assert!(attention.lock().unwrap().accepted.is_none()); + assert!(execution.take_inputs().is_empty()); + observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); + observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); + assert_eq!(execution.take_inputs().len(), 1); +} + +// Real Controller and canonical Session storage with isolated local config. +// A closed loopback model endpoint prevents any production-provider access; +// notifications commit at the model boundary before that request can fail. +struct AttentionWorkerFactory { + root: PathBuf, +} + +impl AttentionWorkerFactory { + async fn controller( + &self, + worker_id: String, + restore: bool, + ) -> std::result::Result { + let name = format!("attention-{worker_id}"); + let root = self.root.join("workers").join(&worker_id); + let store = session_store::CombinedStore::new( + session_store::WorkerSessionStore::new(root.join("session")) + .map_err(|e| e.to_string())?, + session_store::WorkerAggregateStore::new(&root, &name).map_err(|e| e.to_string())?, + ); + let manifest = manifest::WorkerManifest::from_toml(&format!( + r#" + [worker] + name = "{name}" + pwd = "./" + [model] + scheme = "anthropic" + model_id = "attention-test" + base_url = "http://127.0.0.1:1" + auth = {{ kind = "none" }} + [engine] + max_tokens = 10 + [scope] + allow = [] + "# + )) + .map_err(|e| e.to_string())?; + let context = worker::WorkerWorkspaceContext::local_filesystem(None); + let workspace_client = context.client_handle(); + let loader = worker::PromptCatalogSource::builtins_only(); + let worker = if restore { + worker::Worker::restore_from_worker_metadata_with_context( + &name, + manifest, + store, + loader, + context, + worker::WorkerFilesystemAuthority::None, + ) + .await + } else { + worker::Worker::from_manifest_with_context( + manifest, + store, + loader, + context, + worker::WorkerFilesystemAuthority::None, + ) + .await + } + .map_err(|e| e.to_string())?; + let started = worker::PreparedWorker::new( + worker.with_notification_coalesce_delay(std::time::Duration::from_millis(10)), + worker::WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: root.join(uuid::Uuid::now_v7().to_string()), + bash_output_dir: root.join("bash-output"), + }, + worker::WorkerControllerTransport::InProcess, + ) + .start() + .await + .map_err(|e| e.to_string())?; + Ok(worker_runtime::worker_backend::RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + }) + } +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFactory { + async fn spawn_controller( + &self, + request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> std::result::Result { + self.controller(request.worker_ref.worker_id.to_string(), false) + .await + } + async fn restore_controller( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.controller(request.worker_ref.worker_id.to_string(), true) + .await + } +} + +use worker_runtime::execution as attention_execution; + +// Fault injection occurs AFTER the real Controller durably accepted input. +struct AttentionResponseLossBackend { + inner: Arc, + lose_next: std::sync::atomic::AtomicBool, + attempts: Mutex>, +} + +impl attention_execution::WorkerExecutionBackend for AttentionResponseLossBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + fn spawn_worker( + &self, + r: attention_execution::WorkerExecutionSpawnRequest, + ) -> attention_execution::WorkerExecutionSpawnResult { + self.inner.spawn_worker(r) + } + fn preflight_restore( + &self, + r: &attention_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), attention_execution::WorkerExecutionResult> { + self.inner.preflight_restore(r) + } + fn reconcile_restore( + &self, + r: attention_execution::WorkerExecutionRestoreRequest, + ) -> attention_execution::WorkerExecutionSpawnResult { + self.inner.reconcile_restore(r) + } + fn restore_worker( + &self, + r: attention_execution::WorkerExecutionRestoreRequest, + ) -> attention_execution::WorkerExecutionSpawnResult { + self.inner.restore_worker(r) + } + fn activate_restored_worker( + &self, + id: attention_execution::WorkerLifecycleOperationId, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.activate_restored_worker(id, worker_ref) + } + fn dispatch_input( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + input: worker_runtime::interaction::WorkerInput, + ) -> attention_execution::WorkerExecutionResult { + self.attempts + .lock() + .unwrap() + .push(input.submission_request_id.clone().unwrap()); + let result = self.inner.dispatch_input(worker_ref, input); + assert_eq!( + result.outcome, + attention_execution::WorkerExecutionOutcome::Accepted + ); + if self + .lose_next + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + attention_execution::WorkerExecutionResult::errored( + attention_execution::WorkerExecutionOperation::Input, + "injected loss of accepted response", + ) + } else { + result + } + } + fn dispatch_method( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + method: protocol::Method, + ) -> attention_execution::WorkerExecutionResult { + self.inner.dispatch_method(worker_ref, method) + } + fn stop_worker_operation( + &self, + r: attention_execution::WorkerExecutionStopRequest, + ) -> attention_execution::WorkerExecutionResult { + self.inner.stop_worker_operation(r) + } + fn worker_snapshot( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.worker_snapshot(worker_ref) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn orchestrator_attention_real_worker_restore_keeps_receipt_and_new_input_commits_once() { + real_attention_worker_start_restore(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() { + real_attention_worker_start_restore(true).await; +} + +async fn real_attention_worker_start_restore(lose_response: bool) { + use session_store::{LogEntry, Store, WorkerSessionStore}; + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let config = test_server_config(dir.path()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + // Real Worker Runtime adapter/controller/storage, isolated from profile + // workspace callbacks and real LLM credentials. + let execution = Arc::new( + worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new( + AttentionWorkerFactory { + root: config.embedded_runtime_store_root.clone(), + }, + ) + .unwrap(), + ); + let execution = Arc::new(AttentionResponseLossBackend { + inner: execution, + lose_next: std::sync::atomic::AtomicBool::new(lose_response), + attempts: Mutex::new(Vec::new()), + }); + let api = WorkspaceApi::new_with_execution_backend(config, store, execution.clone()) + .await + .unwrap(); + seed_test_delivery_binding(&api); + let mut input = ticket::NewTicket::new("Real notification backlog"); + input.workflow_state = Some(TicketState::Open); + set_test_ticket_target(&mut input, "test-repository", "HEAD"); + let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); + assign_test_orchestrator(&api, &ticket.id); + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let session = WorkerSessionStore::open_read_only( + api.config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id) + .join("session"), + ) + .unwrap(); + let entries = || { + let id = session.session_id().unwrap().unwrap(); + session + .list_segments(id) + .unwrap() + .into_iter() + .flat_map(|segment| session.read_all_read_only(id, segment).unwrap()) + .collect::>() + }; + let receipts = || { + entries() + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => { + Some(payload["notification_receipts"].as_array().unwrap().clone()) + } + LogEntry::AnnotatedSystemItem { extensions, .. } => extensions + .iter() + .find(|e| e.domain == "worker.pending_activations.v1") + .map(|e| { + e.payload["notification_receipts"] + .as_array() + .unwrap() + .clone() + }), + _ => None, + }) + .unwrap() + }; + let commits = || { + entries().iter().filter(|entry| matches!(entry, + LogEntry::AnnotatedSystemItem { entry, .. } + if matches!(&entry.item, session_store::SystemItem::Notification { message, .. } + if message.starts_with("Assigned Tickets have unreceived input:")) + )).count() + }; + async fn wait_for_commits usize>(count: F, expected: usize) { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let actual = count(); + assert!(actual <= expected, "duplicate canonical notifications"); + if actual == expected { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("controlled Worker notification commit"); + } + assert_eq!(receipts().len(), 1); + wait_for_commits(&commits, 1).await; + assert_eq!( + execution.attempts.lock().unwrap().len(), + if lose_response { 2 } else { 1 } + ); + let first = orchestrator_attention_state(&api, &worker) + .lock() + .unwrap() + .accepted + .clone() + .unwrap(); + let first_id = first.request_id.clone(); + assert_eq!(first.fingerprint, first.input.delivery_id); + assert!(attention_pending(&api).is_empty()); + use protocol::subscription::SubscriptionWorkerState as S; + for previous in [ + None, + Some(S::Idle), + Some(S::Stopped), + Some(S::Paused), + Some(S::Running), + ] { + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, previous, S::Idle); + } + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + assert_eq!(receipts().len(), 1); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + assert!( + orchestrator_attention_state(&api, &worker) + .lock() + .unwrap() + .accepted + .is_none() + ); + assert!(attention_pending(&api).is_empty()); + assert_eq!( + execution.attempts.lock().unwrap().len(), + if lose_response { 2 } else { 1 } + ); + assert_eq!(receipts().len(), 1); + assert_eq!(commits(), 1); + // Retry directly at the real Controller boundary: the same Worker's + // persisted receipt survives restore even after server memory is retired. + let retry = api + .runtime + .send_input( + &worker, + WorkerInputRequest { + kind: WorkerInputKind::Notify, + content: first.content.clone(), + submission_request_id: Some(first_id.clone()), + segments: None, + }, + ) + .unwrap(); + assert_eq!(retry.state, InternalWorkerOperationState::Accepted); + assert_eq!(receipts().len(), 1); + assert_eq!(commits(), 1); + // Only a new external input earns another delivery, not restore itself. + browser_ticket_backend(&api) + .unwrap() + .add_event( + ticket.id.clone().into(), + ticket::NewTicketEvent::new(ticket::TicketEventKind::Comment, "New owner answer"), + ) + .unwrap(); + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, Some(S::Running), S::Idle); + let second = orchestrator_attention_state(&api, &worker) + .lock() + .unwrap() + .accepted + .clone() + .unwrap(); + assert_eq!(first.input.assignment_id, second.input.assignment_id); + assert_ne!(first.input.input_event_id, second.input.input_event_id); + let second_id = second.request_id; + assert_ne!(first_id, second_id); + wait_for_commits(&commits, 2).await; + let attempts = execution.attempts.lock().unwrap(); + assert_eq!(attempts.len(), if lose_response { 4 } else { 3 }); + assert_eq!( + attempts.iter().filter(|id| **id == first_id).count(), + if lose_response { 3 } else { 2 } + ); + assert_eq!(attempts.iter().filter(|id| **id == second_id).count(), 1); + drop(attempts); + let receipts = receipts(); + assert_eq!(receipts.len(), 2); + for id in [first_id, second_id] { + assert_eq!( + receipts + .iter() + .filter(|r| r["notification_request_id"] == id) + .count(), + 1 + ); + } + // Stop only this isolated test fixture; no live dogfooding Worker is touched. + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); +} + +#[tokio::test] +async fn orchestrator_attention_restore_and_replacement_do_not_resend_acknowledged_input() { + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + assert_eq!(execution.take_inputs().len(), 1); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + execution.accept_restores(); + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + assert_eq!(restored.worker.unwrap().worker_id, worker.worker_id); + assert!(execution.take_inputs().is_empty()); + assert!(attention_pending(&api).is_empty()); + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, None, S::Idle); + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + assert!(execution.take_inputs().is_empty()); + // This is a fixture Worker, not the running dogfooding Orchestrator. + api.runtime.delete_worker(&worker).unwrap(); + api.worker_projection + .publish_ordered(|| { + api.store + .delete_worker_registry(TEST_WORKSPACE_ID, &worker) + .map(|commit| ((), commit)) + }) + .unwrap(); + let replacement = start_attention_test_orchestrator(&api).await; + let replacement_id = replacement.worker.unwrap().worker_id; + assert_ne!(replacement_id, worker.worker_id); + assert!(execution.take_inputs().is_empty()); + assert!(attention_pending(&api).is_empty()); + attention_assign_ticket(&api, "New assignment for replacement"); + let replacement = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, replacement_id); + dispatch_orchestrator_assignment_attention(&api, &replacement); + let inputs = execution.take_inputs(); + assert_eq!(inputs.len(), 1); + assert!(inputs[0].1.contains("New assignment for replacement")); +} + +#[tokio::test] +async fn orchestrator_attention_close_reopen_does_not_revive_old_input_but_reassignment_delivers() { + for unknown_response in [false, true] { + let (_dir, api, execution) = attention_test_api().await; + if unknown_response { + execution.reject_inputs("unknown acceptance before closure"); + } + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let attention = orchestrator_attention_state(&api, &worker); + let original = { + let state = attention.lock().unwrap(); + if unknown_response { + state.pending.clone().unwrap() + } else { + state.accepted.clone().unwrap() + } + }; + execution.take_inputs(); + execution.take_input_request_ids(); + *execution.input_failure.lock().unwrap() = None; + let backend = browser_ticket_backend(&api).unwrap(); + for (key, action) in [ + ( + "attention-close", + ticket::TicketCaseAction::Close { + reason: ticket::TicketClosureReason::NotPlanned, + }, + ), + ("attention-reopen", ticket::TicketCaseAction::Reopen), + ] { + let observed = backend + .show(original.input.ticket_id.clone().into()) + .unwrap(); + backend + .update_case( + &original.input.ticket_id, + ticket::TicketCaseUpdate { + expected_content_digest: ticket::ticket_content_digest(&observed), + expected_state: observed.meta.workflow_state, + expected_closure_operation_key: observed.closure_operation_key, + operation_key: key.into(), + action, + explanation: "Human case decision".into(), + references: vec![], + author: None, + }, + ) + .unwrap(); + dispatch_orchestrator_assignment_attention(&api, &worker); + assert!(attention_pending(&api).is_empty()); + assert!( + execution.take_inputs().is_empty(), + "case history must not revive old assignment input" + ); + assert!(attention.lock().unwrap().pending.is_none()); + } + // Even an external comment on the reopened case cannot revive the ended assignment. + backend + .add_event( + original.input.ticket_id.clone().into(), + ticket::NewTicketEvent::new( + ticket::TicketEventKind::Comment, + "Historical result annotation", + ), + ) + .unwrap(); + dispatch_orchestrator_assignment_attention(&api, &worker); + assert!(attention_pending(&api).is_empty()); + assert!(execution.take_inputs().is_empty()); + api.store + .set_current_ticket_role_assignment( + &TicketRoleAssignmentRecord { + workspace_id: TEST_WORKSPACE_ID.into(), + ticket_id: original.input.ticket_id.clone(), + assignment_id: "attention-reassigned".into(), + role: TicketAssignmentRole::Orchestrator, + principal: TicketAssignmentPrincipal::WorkspaceAgent { + agent_key: "workspace-orchestrator".into(), + }, + assigned_by: "test-user".into(), + assigned_at: "2026-09-02T00:00:00Z".into(), + }, + Some(&original.input.assignment_id), + "attention-reassign-event", + "attention-reassign-op", + true, + ) + .unwrap(); + dispatch_orchestrator_assignment_attention(&api, &worker); + assert_eq!( + execution.take_inputs().len(), + 1, + "explicit new assignment must deliver" + ); + let accepted = attention.lock().unwrap().accepted.clone().unwrap(); + assert_ne!(original.input.assignment_id, accepted.input.assignment_id); + assert_ne!(original.request_id, accepted.request_id); + assert!(attention_pending(&api).is_empty()); + } +} + +#[tokio::test] +async fn orchestrator_running_to_idle_recovers_unreceived_input_without_notification_memory() { + let (_dir, api, execution) = attention_test_api().await; + let backend = browser_ticket_backend(&api).unwrap(); + let started = start_attention_test_orchestrator(&api).await; + assert!(started.online); + assert_eq!(execution.take_inputs().len(), 1); + let worker_id = started.worker.unwrap().worker_id; + let worker = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, &worker_id); + let attention = orchestrator_attention_state(&api, &worker); + let acknowledged = attention.lock().unwrap().accepted.clone().unwrap(); + assert_eq!(acknowledged.fingerprint, acknowledged.input.delivery_id); + attention.lock().unwrap().accepted = None; + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); + assert!( + execution.take_inputs().is_empty(), + "lost memory cannot resend durable acknowledgement" + ); + assert!(attention.lock().unwrap().accepted.is_none()); + let ticket_ref = attention_assign_ticket(&api, "Recover unreceived work"); + let pending = attention_pending(&api); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].ticket_id, ticket_ref.id); + // Snapshot/reconnection and non-turn transitions cannot stand in for startup. + for previous in [None, Some(S::Idle), Some(S::Stopped), Some(S::Paused)] { + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, previous, S::Idle); + assert!(attention.lock().unwrap().accepted.is_none()); + assert!(execution.take_inputs().is_empty()); + assert_eq!(attention_pending(&api).len(), 1); + } + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); + let accepted = attention.lock().unwrap().accepted.clone().unwrap(); + assert_eq!(accepted.input.assignment_id, pending[0].assignment_id); + assert_eq!(accepted.input.input_event_id, pending[0].input_event_id); + assert_eq!(accepted.fingerprint, accepted.input.delivery_id); + assert!(attention_pending(&api).is_empty()); + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); + let notifications = execution.take_inputs(); + assert_eq!(notifications.len(), 1); + assert_eq!(notifications[0].0.worker_id.to_string(), worker_id); + let content = ¬ifications[0].1; + assert!(content.starts_with("Assigned Tickets have unreceived input:")); + assert!( + content.contains(&format!( + "- {} — Recover unreceived work", + ticket_ref.resource_key.as_deref().unwrap() + )), + "unexpected notification body: {content:?}" + ); + assert!(content.contains("Reread the current Ticket state before acting")); + assert!(!content.contains(TEST_WORKSPACE_ID)); + assert!(!content.contains("bounded")); + assert!(!content.contains("omitted")); + + let candidates = backend + .list(ticket::TicketListQuery::states([ + ticket::TicketListState::Open, + ])) + .unwrap(); + let mut truncated_candidates = (1..=worker::OrchestratorQueueAttentionContext::MAX_TICKETS + 1) + .map(|index| { + let mut candidate = candidates[0].clone(); + candidate.id = format!("opaque-{index}"); + candidate.resource_key = Some(format!("T-{index}")); + candidate.title = format!("Queued {index}"); + candidate + }) + .collect::>(); + let truncated = orchestrator_queue_attention_context( + TEST_WORKSPACE_ID, + TEST_WORKSPACE_ID, + &truncated_candidates, + ) + .unwrap(); + let rendered = worker::PromptCatalog::builtins_only() + .unwrap() + .orchestrator_queue_attention(worker::OrchestratorQueueAttentionPrompt::Server, &truncated) + .unwrap(); + assert!(rendered.contains("- T-20 — Queued 20")); + assert!(!rendered.contains("T-21")); + assert!(rendered.contains("were omitted from this notice: 1")); + assert!(!rendered.contains("opaque-")); + + truncated_candidates[0].resource_key = None; + assert_eq!( + orchestrator_queue_attention_context( + TEST_WORKSPACE_ID, + TEST_WORKSPACE_ID, + &truncated_candidates + ) + .unwrap_err(), + "missing_ticket_resource_key" + ); + assert_eq!( + orchestrator_queue_attention_context( + TEST_WORKSPACE_ID, + "foreign-workspace", + &truncated_candidates + ) + .unwrap_err(), + "foreign_workspace_ticket_projection" + ); +} diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs new file mode 100644 index 000000000..b8821a66b --- /dev/null +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -0,0 +1,962 @@ +// Included as a child module of server::tests. These tests exercise committed +// Assign + HTTP + embedded Runtime boundaries, not the old Queue predicate. +use super::*; +use session_store::{LogEntry, Store, WorkerSessionStore}; +use worker_runtime::execution as delivery_execution; + +fn delivery_assignment_path(ticket: &str) -> String { + format!("/api/w/{TEST_WORKSPACE_ID}/tickets/{ticket}/assignments/orchestrator") +} + +async fn delivery_http( + api: &WorkspaceApi, + method: &str, + path: &str, + body: Option, + expected: StatusCode, +) -> Value { + let token = format!("api-token-{TEST_WORKSPACE_ID}"); + let response = build_router(api.clone()) + .oneshot( + Request::builder() + .method(method) + .uri(path) + .header("authorization", format!("Bearer {token}")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from(body.map(|v| v.to_string()).unwrap_or_default())) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + status, + expected, + "{method} {path}: {}", + String::from_utf8_lossy(&bytes) + ); + serde_json::from_slice(&bytes).unwrap() +} + +async fn delivery_assign(api: &WorkspaceApi, ticket: &str, key: &str) -> Value { + delivery_http( + api, + "PUT", + &delivery_assignment_path(ticket), + Some(json!({ + "operation_id": key, + "principal": {"kind":"workspace_agent", "agent_key":"workspace-orchestrator"} + })), + StatusCode::OK, + ) + .await +} + +fn delivery_ticket(api: &WorkspaceApi, title: &str) -> ticket::TicketRef { + browser_ticket_backend(api) + .unwrap() + .create(ticket::NewTicket::new(title)) + .unwrap() +} + +fn delivery_tracking( + api: &WorkspaceApi, + ticket: &str, + assignment: &str, +) -> ticket::TicketAssignmentTrackingProjection { + api.store + .get_ticket_assignment_tracking(TEST_WORKSPACE_ID, ticket, assignment) + .unwrap() + .unwrap() +} + +fn delivery_worker(response: BrowserWorkspaceOrchestratorResponse) -> RuntimeWorkerRef { + assert!(response.online); + RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + response.worker.unwrap().worker_id, + ) +} + +fn delivery_scan(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { + dispatch_orchestrator_assignment_attention(api, worker); + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(api, &worker.worker_id, Some(S::Running), S::Idle); +} + +fn delivery_stop(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { + api.runtime + .stop_worker( + worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); +} + +async fn delivery_recording_fixture( + root: &Path, +) -> ( + WorkspaceApi, + Arc, + RuntimeWorkerRef, + RuntimeIdentityMaterial, +) { + let (mut api, execution) = test_api_with_recording_backend(root).await; + seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID); + let worker = delivery_worker(start_attention_test_orchestrator(&api).await); + let identity = RuntimeIdentityMaterial::generate(EMBEDDED_WORKER_RUNTIME_ID).unwrap(); + configure_runtime_request_auth(&mut api, &identity, EMBEDDED_WORKER_RUNTIME_ID); + seed_worker_source_member(&api, &worker.runtime_id, &worker.worker_id); + assert!( + execution.take_inputs().is_empty(), + "open tickets are not implicit delegations" + ); + (api, execution, worker, identity) +} + +async fn delivery_self_report( + api: &WorkspaceApi, + identity: &RuntimeIdentityMaterial, + worker: &RuntimeWorkerRef, + current: &ticket::TicketAssignmentTrackingProjection, + status: &str, + key: &str, +) -> Value { + let path = format!( + "/workspaces/{TEST_WORKSPACE_ID}/tickets/{}/assignment-tracking", + current.ticket_id + ); + let body = serde_json::to_vec(&json!({ + "assignment_id":current.assignment_id, + "expected_last_event_id":current.last_event_id, + "operation_key":key, + "tracking": {"status":status, "waiting_reason":"Need the human's choice", + "progress":"Options investigated", "next_action":"Read the answer then decide", + "result": if status == "done" { Some("Returned options without closing") } else { None }} + })).unwrap(); + let response = build_router(api.clone()) + .oneshot(runtime_source_request( + identity, + Some(&worker.worker_id), + "POST", + &path, + body, + )) + .await + .unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + status, + StatusCode::OK, + "{}", + String::from_utf8_lossy(&bytes) + ); + serde_json::from_slice(&bytes).unwrap() +} + +fn delivery_external_comment(api: &WorkspaceApi, ticket: &str, text: &str) { + browser_ticket_backend(api) + .unwrap() + .add_event( + ticket.to_owned().into(), + ticket::NewTicketEvent::new(ticket::TicketEventKind::Comment, text), + ) + .unwrap(); + notify_ticket_recipients(api, TEST_WORKSPACE_ID, ticket, "open", "open", None); +} + +#[tokio::test] +async fn assignment_waiting_self_report_does_not_loop_but_external_answer_is_delivered() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, identity) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "Choose an option"); + let assigned = delivery_assign(&api, &ticket.id, "assign-choice").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + let initial = delivery_tracking(&api, &ticket.id, assignment); + assert_eq!( + initial.tracking.as_ref().unwrap().status, + ticket::TicketAssignmentTrackingStatus::Waiting + ); + let first = execution.take_inputs(); + assert_eq!(first.len(), 1, "Assign alone delivers without Queue"); + let first_ids = execution.take_input_request_ids(); + assert!(first_ids[0].is_some()); + assert!(first[0].1.contains(ticket.resource_key.as_deref().unwrap())); + let reported = + delivery_self_report(&api, &identity, &worker, &initial, "waiting", "wait-human").await; + // Even an explicit notification of one's own mutation must not manufacture + // fresh work, and repeated idle/startup scans must not use waiting as a queue. + notify_ticket_recipients( + &api, + TEST_WORKSPACE_ID, + &ticket.id, + "open", + "open", + Some(worker.clone()), + ); + for _ in 0..3 { + delivery_scan(&api, &worker); + } + assert!( + execution.take_inputs().is_empty(), + "self progress became an input loop" + ); + let waiting = delivery_tracking(&api, &ticket.id, assignment); + assert_eq!(serde_json::to_value(&waiting).unwrap(), reported); + delivery_external_comment(&api, &ticket.id, "Human answer: choose option B"); + let answer = execution.take_inputs(); + assert_eq!( + answer.len(), + 1, + "external answer must wake the existing delegation" + ); + assert_ne!(execution.take_input_request_ids(), first_ids); + assert!( + answer[0] + .1 + .contains(ticket.resource_key.as_deref().unwrap()) + ); + assert!( + answer[0].1.contains("Need the human's choice"), + "waiting context missing: {:?}", + answer[0].1 + ); + assert!(answer[0].1.contains("Read the answer then decide")); + for _ in 0..3 { + delivery_scan(&api, &worker); + } + assert!( + execution.take_inputs().is_empty(), + "same answer was delivered again" + ); + assert_eq!( + delivery_tracking(&api, &ticket.id, assignment), + waiting, + "delivery/idle must not infer handling inprogress or done" + ); + delivery_stop(&api, &worker); +} + +#[tokio::test] +async fn assignment_done_historical_inputs_do_not_restart_work_but_reissue_is_new_delegation() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, identity) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "Return options while open"); + let assigned = delivery_assign(&api, &ticket.id, "assign-return").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + execution.take_inputs(); + execution.take_input_request_ids(); + let initial = delivery_tracking(&api, &ticket.id, assignment); + delivery_self_report(&api, &identity, &worker, &initial, "done", "return-options").await; + let done = delivery_tracking(&api, &ticket.id, assignment); + assert_eq!( + api.authority.ticket(&ticket.id).unwrap().state.as_str(), + "open" + ); + delivery_external_comment( + &api, + &ticket.id, + "Historical result annotation, not a new request", + ); + for _ in 0..3 { + delivery_scan(&api, &worker); + } + assert!( + execution.take_inputs().is_empty(), + "open + done must not execute history" + ); + assert_eq!(delivery_tracking(&api, &ticket.id, assignment), done); + for (key, action) in [ + ( + "close-return", + json!({"action":"close", "reason":"not_planned"}), + ), + ("reopen-return", json!({"action":"reopen"})), + ] { + let observed = api.authority.ticket(&ticket.id).unwrap(); + delivery_http( + &api, + "POST", + &format!("/workspaces/{TEST_WORKSPACE_ID}/tickets/{}/case", ticket.id), + Some(json!({"expected_content_digest":observed.content_digest, + "expected_state":observed.state, + "expected_closure_operation_key":observed.closure_operation_key, + "operation_key":key, "action":action, "explanation":"Human case decision"})), + StatusCode::OK, + ) + .await; + delivery_scan(&api, &worker); + assert!( + execution.take_inputs().is_empty(), + "case history must not revive done delegation" + ); + assert_eq!(delivery_tracking(&api, &ticket.id, assignment), done); + } + let reassigned = delivery_assign(&api, &ticket.id, "reissue-return").await; + let new_assignment = reassigned["assignment"]["assignment_id"].as_str().unwrap(); + assert_ne!(new_assignment, assignment); + assert_eq!( + execution.take_inputs().len(), + 1, + "explicit reissue must be reconsidered" + ); + let retained = delivery_tracking(&api, &ticket.id, assignment); + assert!(!retained.current); + assert_eq!( + retained.tracking, done.tracking, + "new request overwrote the old result" + ); + assert_eq!( + delivery_tracking(&api, &ticket.id, new_assignment) + .tracking + .unwrap() + .status, + ticket::TicketAssignmentTrackingStatus::Waiting + ); + // An exact old Assign retry returns its receipt, never revives old work. + assert_eq!( + delivery_assign(&api, &ticket.id, "assign-return").await, + assigned + ); + delivery_scan(&api, &worker); + assert!(execution.take_inputs().is_empty()); + delivery_stop(&api, &worker); +} + +#[tokio::test] +async fn assignment_clear_and_reassign_discard_old_unknown_delivery_without_stopping_worker() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, _) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "Fence old delivery"); + execution.reject_inputs("unknown acceptance response"); + let assigned = delivery_assign(&api, &ticket.id, "assign-old").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + let old_attempts = execution.take_inputs(); + let old_ids = execution.take_input_request_ids(); + assert!(!old_attempts.is_empty()); + assert!(old_ids.iter().all(|id| id == &old_ids[0])); + delivery_http( + &api, + "DELETE", + &format!( + "{}?assignment_id={assignment}&operation_id=clear-old", + delivery_assignment_path(&ticket.id) + ), + None, + StatusCode::OK, + ) + .await; + *execution.input_failure.lock().unwrap() = None; + notify_ticket_recipients(&api, TEST_WORKSPACE_ID, &ticket.id, "open", "open", None); + delivery_scan(&api, &worker); + assert!( + execution.take_inputs().is_empty(), + "cleared reservation was sent as new work" + ); + assert_eq!( + delivery_assign(&api, &ticket.id, "assign-old").await, + assigned + ); + delivery_scan(&api, &worker); + assert!( + execution.take_inputs().is_empty(), + "old operation retry revived cleared work" + ); + let reassigned = delivery_assign(&api, &ticket.id, "assign-new").await; + let new_assignment = reassigned["assignment"]["assignment_id"].as_str().unwrap(); + assert_ne!(new_assignment, assignment); + assert_eq!(execution.take_inputs().len(), 1); + let ids = execution.take_input_request_ids(); + assert_eq!(ids.len(), 1); + assert_ne!( + ids[0], old_ids[0], + "new assignment reused stale input identity" + ); + let old = delivery_tracking(&api, &ticket.id, assignment); + assert!(!old.current); + assert_eq!( + old.tracking.unwrap().status, + ticket::TicketAssignmentTrackingStatus::Waiting, + "clear must not rewrite the old status to done" + ); + assert_eq!( + find_workspace_orchestrator(&api).unwrap().worker, + worker, + "clear/reassign are not Worker stop or replacement" + ); + delivery_stop(&api, &worker); +} + +// The model boundary is scripted, but Controller, notification acceptance, +// canonical session commits, Runtime management and HTTP routing are real. +#[derive(Clone, Default)] +struct DeliveryScriptedClient { + requests: Arc>>, +} + +#[async_trait::async_trait] +impl agen::llm_client::LlmClient for DeliveryScriptedClient { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + async fn stream( + &self, + request: agen::llm_client::Request, + ) -> std::result::Result + { + self.requests.lock().unwrap().push(request); + use agen::llm_client::event::Event as LlmEvent; + Ok(Box::pin(futures::stream::iter( + [ + LlmEvent::text_block_start(0), + LlmEvent::text_delta( + 0, + "Read the delegation; returning without starting a Coder.", + ), + LlmEvent::text_block_stop(0, None), + ] + .into_iter() + .map(Ok), + ))) + } +} + +struct DeliveryWorkerFactory { + root: PathBuf, + client: DeliveryScriptedClient, +} + +impl DeliveryWorkerFactory { + async fn controller( + &self, + worker_id: String, + restore: bool, + ) -> std::result::Result { + let name = format!("delivery-{worker_id}"); + let root = self.root.join("workers").join(&worker_id); + let store = session_store::CombinedStore::new( + WorkerSessionStore::new(root.join("session")).map_err(|e| e.to_string())?, + session_store::WorkerAggregateStore::new(&root, &name).map_err(|e| e.to_string())?, + ); + let manifest = manifest::WorkerManifest::from_toml(&format!( + r#" + [worker] + name = "{name}" + pwd = "./" + [model] + scheme = "anthropic" + model_id = "scripted-delivery" + auth = {{ kind = "none" }} + [engine] + max_tokens = 64 + [scope] + allow = [] + "# + )) + .map_err(|e| e.to_string())?; + let context = worker::WorkerWorkspaceContext::local_filesystem(None); + let workspace_client = context.client_handle(); + let bootstrap = worker::WorkerBootstrap::new( + manifest, + store, + worker::PromptCatalogSource::builtins_only(), + context, + worker::WorkerFilesystemAuthority::None, + worker::WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: root.join(uuid::Uuid::now_v7().to_string()), + bash_output_dir: root.join("bash-output"), + }, + worker::WorkerControllerTransport::InProcess, + ) + .with_model_client(self.client.clone()); + let prepared = if restore { + bootstrap.prepare_restored(&name).await + } else { + bootstrap.prepare().await + } + .map_err(|e| e.to_string())?; + let started = prepared.start().await.map_err(|e| e.to_string())?; + Ok(worker_runtime::worker_backend::RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + }) + } +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFactory { + async fn spawn_controller( + &self, + request: delivery_execution::WorkerExecutionSpawnRequest, + ) -> std::result::Result { + self.controller(request.worker_ref.worker_id.to_string(), false) + .await + } + async fn restore_controller( + &self, + request: delivery_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.controller(request.worker_ref.worker_id.to_string(), true) + .await + } +} + +// Lose the response only AFTER the real Worker accepts the exact input. Unlike +// a pre-send rejection, this exercises durable Worker receipt deduplication. +struct DeliveryResponseLossBackend { + inner: Arc, + lose_responses: std::sync::atomic::AtomicBool, + attempts: Mutex>, +} + +impl delivery_execution::WorkerExecutionBackend for DeliveryResponseLossBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + fn spawn_worker( + &self, + r: delivery_execution::WorkerExecutionSpawnRequest, + ) -> delivery_execution::WorkerExecutionSpawnResult { + self.inner.spawn_worker(r) + } + fn preflight_restore( + &self, + r: &delivery_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), delivery_execution::WorkerExecutionResult> { + self.inner.preflight_restore(r) + } + fn restore_worker( + &self, + r: delivery_execution::WorkerExecutionRestoreRequest, + ) -> delivery_execution::WorkerExecutionSpawnResult { + self.inner.restore_worker(r) + } + fn reconcile_restore( + &self, + r: delivery_execution::WorkerExecutionRestoreRequest, + ) -> delivery_execution::WorkerExecutionSpawnResult { + self.inner.reconcile_restore(r) + } + fn activate_restored_worker( + &self, + id: delivery_execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.activate_restored_worker(id, worker) + } + fn dispatch_input( + &self, + worker: &worker_runtime::identity::WorkerRef, + input: worker_runtime::interaction::WorkerInput, + ) -> delivery_execution::WorkerExecutionResult { + self.attempts.lock().unwrap().push(( + input.submission_request_id.clone().unwrap(), + input.content.clone(), + )); + let result = self.inner.dispatch_input(worker, input); + if result.is_accepted() + && self + .lose_responses + .load(std::sync::atomic::Ordering::SeqCst) + { + delivery_execution::WorkerExecutionResult::errored( + delivery_execution::WorkerExecutionOperation::Input, + "accepted response lost", + ) + } else { + result + } + } + fn dispatch_method( + &self, + worker: &worker_runtime::identity::WorkerRef, + method: protocol::Method, + ) -> delivery_execution::WorkerExecutionResult { + self.inner.dispatch_method(worker, method) + } + fn stop_worker_operation( + &self, + r: delivery_execution::WorkerExecutionStopRequest, + ) -> delivery_execution::WorkerExecutionResult { + self.inner.stop_worker_operation(r) + } + fn worker_snapshot( + &self, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.worker_snapshot(worker) + } +} + +async fn delivery_real_fixture( + root: &Path, + lose_responses: bool, +) -> ( + WorkspaceApi, + Arc, + DeliveryScriptedClient, +) { + let config = test_server_config(root); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + let client = DeliveryScriptedClient::default(); + let inner = Arc::new( + worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new(DeliveryWorkerFactory { + root: config.embedded_runtime_store_root.clone(), + client: client.clone(), + }) + .unwrap(), + ); + let execution = Arc::new(DeliveryResponseLossBackend { + inner, + lose_responses: std::sync::atomic::AtomicBool::new(lose_responses), + attempts: Mutex::new(Vec::new()), + }); + let mut api = WorkspaceApi::new_with_execution_backend(config, store, execution.clone()) + .await + .unwrap(); + // The injectable constructor does not install production Runtime trust. + // Delivery resolves the registered singleton plus its active binding. + let identity = RuntimeIdentityMaterial::generate(EMBEDDED_WORKER_RUNTIME_ID).unwrap(); + configure_runtime_request_auth(&mut api, &identity, EMBEDDED_WORKER_RUNTIME_ID); + seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID); + (api, execution, client) +} + +fn delivery_session_entries(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Vec { + let session = WorkerSessionStore::open_read_only( + api.config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id) + .join("session"), + ) + .unwrap(); + let id = session.session_id().unwrap().unwrap(); + session + .list_segments(id) + .unwrap() + .into_iter() + .flat_map(|segment| session.read_all_read_only(id, segment).unwrap()) + .collect() +} + +fn delivery_receipts(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Vec { + delivery_session_entries(api, worker) + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => { + Some(payload["notification_receipts"].as_array().unwrap().clone()) + } + LogEntry::AnnotatedSystemItem { extensions, .. } => extensions + .iter() + .find(|e| e.domain == "worker.pending_activations.v1") + .map(|e| { + e.payload["notification_receipts"] + .as_array() + .unwrap() + .clone() + }), + _ => None, + }) + .unwrap_or_default() +} + +fn delivery_commits(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Vec { + delivery_session_entries(api, worker) + .into_iter() + .filter_map(|entry| match entry { + LogEntry::AnnotatedSystemItem { entry, .. } => match entry.item { + session_store::SystemItem::Notification { message, .. } => Some(message), + _ => None, + }, + _ => None, + }) + .collect() +} + +async fn delivery_wait_idle( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + expected_commits: usize, +) { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let commits = delivery_commits(api, worker).len(); + assert!( + commits <= expected_commits, + "duplicate canonical delivery: {:?}", + delivery_commits(api, worker) + ); + let summary = api.runtime.worker(worker).unwrap(); + if commits == expected_commits + && summary + .worker_state + .as_ref() + .is_some_and(|state| state.state == protocol::WorkerState::Idle) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap_or_else(|_| panic!("scripted Worker did not finish: expected_commits={expected_commits}, commits={:?}, receipts={:?}, worker={:?}", + delivery_commits(api, worker), delivery_receipts(api, worker), api.runtime.worker(worker))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn assignment_without_queue_reaches_real_worker_and_stopped_assign_survives_restore() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, client) = delivery_real_fixture(dir.path(), false).await; + let first = delivery_ticket(&api, "Assign alone is the delegation"); + let assigned = delivery_assign(&api, &first.id, "assign-before-start").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + let initial = delivery_tracking(&api, &first.id, assignment); + assert!( + execution.attempts.lock().unwrap().is_empty(), + "Assign must not auto-restore a stopped Orchestrator" + ); + let worker = delivery_worker(start_attention_test_orchestrator(&api).await); + delivery_wait_idle(&api, &worker, 1).await; + assert_eq!(delivery_receipts(&api, &worker).len(), 1); + assert!(delivery_commits(&api, &worker)[0].contains(first.resource_key.as_deref().unwrap())); + assert_eq!( + client.requests.lock().unwrap().len(), + 1, + "real scripted model did not read the request" + ); + assert_eq!(delivery_tracking(&api, &first.id, assignment), initial); + delivery_stop(&api, &worker); + let second = delivery_ticket(&api, "Assigned while stopped"); + let second_assigned = delivery_assign(&api, &second.id, "assign-while-stopped").await; + let second_id = second_assigned["assignment"]["assignment_id"] + .as_str() + .unwrap(); + let second_initial = delivery_tracking(&api, &second.id, second_id); + assert_eq!(execution.attempts.lock().unwrap().len(), 1); + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + assert_eq!(delivery_worker(restored), worker); + delivery_wait_idle(&api, &worker, 2).await; + assert_eq!(delivery_receipts(&api, &worker).len(), 2); + assert!(delivery_commits(&api, &worker)[1].contains(second.resource_key.as_deref().unwrap())); + assert_eq!( + delivery_tracking(&api, &second.id, second_id), + second_initial + ); + let accepted = execution.attempts.lock().unwrap().clone(); + delivery_scan(&api, &worker); + delivery_stop(&api, &worker); + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "restored" + ); + delivery_scan(&api, &worker); + assert_eq!( + execution.attempts.lock().unwrap().as_slice(), + accepted.as_slice(), + "restoring the same Worker must retain durable acceptance, not generate new work" + ); + assert_eq!(delivery_commits(&api, &worker).len(), 2); + assert_eq!(delivery_receipts(&api, &worker).len(), 2); + delivery_stop(&api, &worker); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commit_once() { + let dir = tempfile::tempdir().unwrap(); + let (mut api, execution, _) = delivery_real_fixture(dir.path(), true).await; + let identity = RuntimeIdentityMaterial::generate(EMBEDDED_WORKER_RUNTIME_ID).unwrap(); + configure_runtime_request_auth(&mut api, &identity, EMBEDDED_WORKER_RUNTIME_ID); + let ticket = delivery_ticket(&api, "Immutable lost-response payload"); + let assigned = delivery_assign(&api, &ticket.id, "assign-lost-response").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + let initial = delivery_tracking(&api, &ticket.id, assignment); + let worker = delivery_worker(start_attention_test_orchestrator(&api).await); + delivery_wait_idle(&api, &worker, 1).await; + let attempts = execution.attempts.lock().unwrap().clone(); + assert!(!attempts.is_empty()); + assert!( + attempts.iter().all(|attempt| attempt == &attempts[0]), + "uncertain delivery changed request id or payload: {attempts:?}" + ); + assert_eq!(delivery_receipts(&api, &worker).len(), 1); + // Change mutable rendering context while the acknowledgement is unknown. + // A new self-report is not a new input and cannot change the frozen retry. + seed_worker_source_member(&api, &worker.runtime_id, &worker.worker_id); + delivery_self_report( + &api, + &identity, + &worker, + &initial, + "waiting", + "waiting-during-response-loss", + ) + .await; + let waiting = delivery_tracking(&api, &ticket.id, assignment); + delivery_stop(&api, &worker); + execution + .lose_responses + .store(false, std::sync::atomic::Ordering::SeqCst); + browser_ticket_backend(&api) + .unwrap() + .edit_item( + ticket.id.clone().into(), + ticket::TicketItemEdit { + title: Some("Title changed while acceptance was unknown".into()), + ..Default::default() + }, + ) + .unwrap(); + // Reopen while acceptance is still unknown. This intentionally discards the + // old API's cached pending text, but not the durable input/Worker receipts. + let config = api.config.clone(); + drop(api); + tokio::task::yield_now().await; + let reopened = WorkspaceApi::new_with_execution_backend( + config.clone(), + Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()), + execution.clone(), + ) + .await + .unwrap(); + let restored = start_attention_test_orchestrator(&reopened).await; + assert_eq!(restored.disposition, "restored"); + assert_eq!(delivery_worker(restored), worker); + { + let after_restart = execution.attempts.lock().unwrap(); + let same_request = after_restart + .iter() + .filter(|attempt| attempt.0 == attempts[0].0) + .collect::>(); + assert!( + same_request.len() > attempts.len(), + "unknown outcome was not retried after restore" + ); + assert!( + same_request.iter().all(|attempt| *attempt == &attempts[0]), + "same request id changed payload after DB reopen/title/self-report change: {after_restart:?}" + ); + } + delivery_wait_idle(&reopened, &worker, 2).await; + assert_eq!(delivery_receipts(&reopened, &worker).len(), 2); + assert_eq!( + delivery_commits(&reopened, &worker) + .iter() + .filter(|message| *message == &attempts[0].1) + .count(), + 1, + "accepted retry committed the old notification twice" + ); + assert_eq!( + delivery_tracking(&reopened, &ticket.id, assignment), + waiting + ); + let before = execution.attempts.lock().unwrap().len(); + browser_ticket_backend(&reopened) + .unwrap() + .add_event( + ticket.id.clone().into(), + ticket::NewTicketEvent::new( + ticket::TicketEventKind::Comment, + "External answer after server restart", + ), + ) + .unwrap(); + let barrier = Arc::new(std::sync::Barrier::new(8)); + std::thread::scope(|scope| { + let joins = (0..8) + .map(|_| { + let api = reopened.clone(); + let worker = worker.clone(); + let barrier = barrier.clone(); + scope.spawn(move || { + barrier.wait(); + dispatch_orchestrator_assignment_attention(&api, &worker); + }) + }) + .collect::>(); + for join in joins { + join.join().unwrap(); + } + }); + delivery_wait_idle(&reopened, &worker, 3).await; + assert_eq!( + execution.attempts.lock().unwrap().len(), + before + 1, + "concurrent scans must share the committed reservation" + ); + assert_eq!(delivery_receipts(&reopened, &worker).len(), 3); + delivery_scan(&reopened, &worker); + assert_eq!(execution.attempts.lock().unwrap().len(), before + 1); + assert_eq!( + delivery_tracking(&reopened, &ticket.id, assignment), + waiting + ); + delivery_stop(&reopened, &worker); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn assignment_worker_replacement_receives_pending_input_without_recreating_delegation() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, _) = delivery_real_fixture(dir.path(), false).await; + let handled = delivery_ticket(&api, "Already received before replacement"); + let saved = delivery_assign(&api, &handled.id, "assign-before-replacement").await; + let assignment = saved["assignment"]["assignment_id"].as_str().unwrap(); + let initial = delivery_tracking(&api, &handled.id, assignment); + let old_worker = delivery_worker(start_attention_test_orchestrator(&api).await); + delivery_wait_idle(&api, &old_worker, 1).await; + delivery_stop(&api, &old_worker); + // Delete only this isolated fixture's stopped identity. A new live Worker + // has a new session; it is not a restore and cannot reuse old session receipts. + api.runtime.delete_worker(&old_worker).unwrap(); + api.worker_projection + .publish_ordered(|| { + api.store + .delete_worker_registry(TEST_WORKSPACE_ID, &old_worker) + .map(|commit| ((), commit)) + }) + .unwrap(); + let pending = delivery_ticket(&api, "Pending input for replacement"); + let new_assign = delivery_assign(&api, &pending.id, "assign-pending-replacement").await; + let pending_assignment = new_assign["assignment"]["assignment_id"].as_str().unwrap(); + let pending_initial = delivery_tracking(&api, &pending.id, pending_assignment); + let started = start_attention_test_orchestrator(&api).await; + assert_eq!(started.disposition, "created"); + let replacement = delivery_worker(started); + assert_ne!(replacement, old_worker); + delivery_wait_idle(&api, &replacement, 1).await; + assert_eq!(delivery_receipts(&api, &replacement).len(), 1); + let commits = delivery_commits(&api, &replacement); + assert!(commits[0].contains(pending.resource_key.as_deref().unwrap())); + assert!( + !commits[0].contains(handled.resource_key.as_deref().unwrap()), + "replacement must not execute acknowledged history" + ); + assert_eq!(execution.attempts.lock().unwrap().len(), 2); + assert_eq!( + delivery_tracking(&api, &handled.id, assignment), + initial, + "Worker replacement must not reassign the stable principal" + ); + assert_eq!( + delivery_tracking(&api, &pending.id, pending_assignment), + pending_initial + ); + delivery_scan(&api, &replacement); + assert_eq!(execution.attempts.lock().unwrap().len(), 2); + delivery_stop(&api, &replacement); +} diff --git a/crates/workspace-server/src/server_assignment_start_tests.rs b/crates/workspace-server/src/server_assignment_start_tests.rs new file mode 100644 index 000000000..11e0cdee6 --- /dev/null +++ b/crates/workspace-server/src/server_assignment_start_tests.rs @@ -0,0 +1,752 @@ +// Include from server.rs's `tests` module as a child module: +// mod assignment_start_tests { include!("server_assignment_start_tests.rs"); } +// These exercise the authenticated router with the existing in-process Runtime +// adapter; they do not start product processes or claim model/E2E coverage. +use super::*; + +struct StartSource { + identity: RuntimeIdentityMaterial, + worker: RuntimeWorkerRef, +} + +async fn start_source(api: &mut WorkspaceApi, runtime: &WorkdirlessFixtureRuntime) -> StartSource { + let runtime_id = WorkdirlessFixtureRuntime::RUNTIME_ID; + let identity = RuntimeIdentityMaterial::generate(runtime_id).unwrap(); + configure_runtime_request_auth(api, &identity, runtime_id); + let memory = api + .config_store + .get_workspace_memory_settings(TEST_WORKSPACE_ID) + .unwrap(); + let reservation = api + .config_store + .reserve_worker_create( + TEST_WORKSPACE_ID, + runtime_id, + "start-orchestrator", + "sha256:start-orchestrator", + Some(crate::hosts::WORKSPACE_ORCHESTRATOR_SINGLETON_KEY), + &memory, + ) + .unwrap(); + let mut summary = WorkdirlessFixtureRuntime::worker_summary( + WorkerCreateBinding { + worker_id: reservation.worker_id, + create_fingerprint: reservation.create_fingerprint, + }, + &WorkerSpawnRequest { + requested_worker_name: Some("Start Orchestrator".into()), + singleton_key: Some(crate::hosts::WORKSPACE_ORCHESTRATOR_SINGLETON_KEY.into()), + intent: WorkerSpawnIntent::WorkspaceCompanion, + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 0, + }, + profile: ProfileSelector::Builtin("builtin:orchestrator".into()), + ticket_assignment: None, + initial_submit: Vec::new(), + workdir_attachment_requests: Vec::new(), + resolved_workdir_attachment_requests: Vec::new(), + resolved_workdir_attachments: Vec::new(), + resolved_config_bundle: None, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_control_operation: None, + }, + ); + summary.tags = vec!["orchestrator".into(), "singleton".into()]; + let worker = summary.worker.clone(); + runtime.workers.lock().unwrap().push(summary); + seed_worker_source_member(api, runtime_id, &worker.worker_id); + api.config_store + .complete_worker_create_reservation(TEST_WORKSPACE_ID, worker.worker_id.parse().unwrap()) + .unwrap(); + assert_eq!(find_workspace_orchestrator(api).unwrap().worker, worker); + StartSource { identity, worker } +} + +fn add_unfinished_start_dependency(api: &WorkspaceApi, ticket_id: &str) -> String { + let backend = browser_ticket_backend(api).unwrap(); + let dependency = backend + .create(ticket::NewTicket::new("Unfinished prerequisite")) + .unwrap(); + backend + .add_ticket_relation( + ticket_id.into(), + ticket::NewTicketRelation { + kind: ticket::TicketRelationKind::DependsOn, + target: dependency.id.clone(), + note: Some("Partial work is intentional".into()), + author: Some("owner".into()), + }, + ) + .unwrap(); + dependency.id +} + +fn start_http_request( + source: Option<&StartSource>, + token: &str, + method: &str, + path: &str, + body: Value, +) -> Request { + let body = serde_json::to_vec(&body).unwrap(); + match source { + Some(source) => runtime_source_request( + &source.identity, + Some(&source.worker.worker_id), + method, + path, + body, + ), + None => Request::builder() + .method(method) + .uri(path) + .header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from(body)) + .unwrap(), + } +} + +async fn start_http_response( + api: &WorkspaceApi, + request: Request, + expected: StatusCode, +) -> Value { + let response = build_router(api.clone()).oneshot(request).await.unwrap(); + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&body)); + serde_json::from_slice(&body) + .unwrap_or_else(|_| serde_json::json!({"message": String::from_utf8_lossy(&body)})) +} + +fn observed_start_delegation( + api: &WorkspaceApi, + ticket_id: &str, + source: Option<&StartSource>, +) -> Option { + source.map(|_| { + api.store + .get_current_ticket_role_assignment( + TEST_WORKSPACE_ID, + ticket_id, + TicketAssignmentRole::Orchestrator, + ) + .unwrap() + .unwrap() + .assignment_id + }) +} + +fn start_worker_body( + ticket_id: Option<&str>, + attachments: Value, + delegation: Option<&str>, +) -> Value { + let mut body = serde_json::json!({ + "runtime_id": WorkdirlessFixtureRuntime::RUNTIME_ID, + "display_name": "Partial investigation", + "profile": "builtin:intake", + "initial_submit": [], + "workdir_attachments": attachments, + "feature_connections": {}, + "control_operation_id": "partial-control-start" + }); + if let Some(ticket_id) = ticket_id { + body["ticket_assignment"] = serde_json::json!({ + "ticket_id": ticket_id, + "operation_id": "partial-start" + }); + if let Some(delegation) = delegation { + body["ticket_assignment"]["orchestrator_assignment_id"] = + Value::String(delegation.into()); + } + } + body +} + +fn assert_start_dependency_diagnostic_only(api: &WorkspaceApi, ticket: &str, dependency: &str) { + let Json(detail) = browser_ticket_detail(api, ticket).unwrap(); + assert_eq!(detail.relations.blockers.len(), 1); + assert_eq!(detail.relations.blockers[0].blocking_ticket, dependency); + assert!(!detail.action_eligibility.can_start_manual_worker); + assert!(detail.current_worker.is_some()); + let Json(prerequisite) = browser_ticket_detail(api, dependency).unwrap(); + assert_eq!(prerequisite.state.as_str(), "open"); + assert!(prerequisite.assignments.is_empty()); + assert!(prerequisite.assignment_tracking.is_empty()); +} + +#[tokio::test] +async fn unfinished_dependencies_allow_manual_and_orchestrator_resource_free_start_and_assign() { + for orchestrator in [false, true] { + for existing in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let mut api = test_api(directory.path()).await; + register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; + let runtime = WorkdirlessFixtureRuntime::default(); + api.runtime.register_or_replace(runtime.clone()); + let source = if orchestrator { + Some(start_source(&mut api, &runtime).await) + } else { + None + }; + let token = seed_test_api_token(api.store.as_ref(), "partial-start-owner"); + let ticket = browser_ticket_backend(&api) + .unwrap() + .create(ticket::NewTicket::new( + "Partial resource-free investigation", + )) + .unwrap(); + let dependency = add_unfinished_start_dependency(&api, &ticket.id); + if orchestrator { + assign_test_orchestrator(&api, &ticket.id); + } + let delegation = observed_start_delegation(&api, &ticket.id, source.as_ref()); + assert!( + browser_ticket_detail(&api, &ticket.id) + .unwrap() + .0 + .action_eligibility + .can_start_manual_worker + ); + let create_path = if orchestrator { + format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers") + } else { + format!("/api/w/{TEST_WORKSPACE_ID}/workers") + }; + let created = start_http_response( + &api, + start_http_request( + source.as_ref(), + &token, + "POST", + &create_path, + start_worker_body( + (!existing).then_some(ticket.id.as_str()), + serde_json::json!([]), + delegation.as_deref(), + ), + ), + StatusCode::OK, + ) + .await; + let worker_id = created["worker_id"].as_str().unwrap(); + assert_eq!(created["worker"]["profile"], "builtin:intake"); + if existing { + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + ticket.id + ); + let mut body = serde_json::json!({ + "operation_id": "partial-existing-assign", + "principal": { + "kind": "worker", + "runtime_id": WorkdirlessFixtureRuntime::RUNTIME_ID, + "worker_id": worker_id + }, + "workdir_bindings": [] + }); + if let Some(delegation) = &delegation { + body["orchestrator_assignment_id"] = Value::String(delegation.clone()); + } + let assigned = start_http_response( + &api, + start_http_request(source.as_ref(), &token, "PUT", &path, body.clone()), + StatusCode::OK, + ) + .await; + let replay = start_http_response( + &api, + start_http_request(source.as_ref(), &token, "PUT", &path, body), + StatusCode::OK, + ) + .await; + assert_eq!(assigned, replay); + } + let worker = RuntimeWorkerRef::new(WorkdirlessFixtureRuntime::RUNTIME_ID, worker_id); + assert!( + api.store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &worker) + .unwrap() + .is_empty() + ); + assert_start_dependency_diagnostic_only(&api, &ticket.id, &dependency); + // Partial work never permits a second unfinished Worker. + let mut competing = start_worker_body( + Some(&ticket.id), + serde_json::json!([]), + delegation.as_deref(), + ); + competing["ticket_assignment"]["operation_id"] = + Value::String("competing-start".into()); + competing["control_operation_id"] = Value::String("competing-control-start".into()); + let spawn_count = runtime.spawn_requests().len(); + start_http_response( + &api, + start_http_request(source.as_ref(), &token, "POST", &create_path, competing), + StatusCode::CONFLICT, + ) + .await; + assert_eq!(runtime.spawn_requests().len(), spawn_count); + if !orchestrator { + assert!( + api.store + .get_current_ticket_role_assignment( + TEST_WORKSPACE_ID, + &ticket.id, + TicketAssignmentRole::Orchestrator + ) + .unwrap() + .is_none() + ); + } + } + } +} + +#[tokio::test] +async fn unfinished_dependencies_allow_read_only_start_and_assign_without_write_or_command_grants() +{ + for orchestrator in [false, true] { + for existing in [false, true] { + let mut fixture = manual_worker_assignment_fixture().await; + let source = if orchestrator { + Some(start_source(&mut fixture.api, &fixture.runtime).await) + } else { + None + }; + let api = &fixture.api; + // Spawn finalization observes Runtime Workdir identity; do not use + // the adapter's unrelated default Repository id for this checkout. + fixture.runtime.set_workdir_repository( + &fixture.docs_workdir_id, + &test_repository_id_by_key(api, "docs"), + ); + let backend = browser_ticket_backend(api).unwrap(); + // Keep only the reference target. Partial investigation needs no writable Repository. + backend + .edit_item( + fixture.ticket_id.clone().into(), + ticket::TicketItemEdit { + targets: Some(ticket::TicketTargetsEdit::Set { + targets: vec![test_ticket_target( + "docs", + "develop", + TicketTargetAccess::ReadOnly, + )], + }), + ..Default::default() + }, + ) + .unwrap(); + let dependency = add_unfinished_start_dependency(api, &fixture.ticket_id); + if orchestrator { + assign_test_orchestrator(api, &fixture.ticket_id); + } + let delegation = observed_start_delegation(api, &fixture.ticket_id, source.as_ref()); + let token = seed_test_api_token(api.store.as_ref(), "read-only-start-owner"); + let worker = if existing { + // Drop the unrelated checkout so the entire attachment set is reference-only. + let main = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .into_iter() + .find(|link| link.workdir_id == fixture.main_workdir_id) + .unwrap(); + api.store + .detach_worker_workdir_connection( + TEST_WORKSPACE_ID, + &fixture.worker, + &main.alias, + &main.connection_id, + &now_registry_timestamp(), + ) + .unwrap(); + sync_runtime_worker_workdir_attachments(api, &fixture.worker).unwrap(); + let mut body = serde_json::to_value(manual_worker_assignment_request( + &fixture, + "read-only-existing", + )) + .unwrap(); + if let Some(delegation) = &delegation { + body["orchestrator_assignment_id"] = Value::String(delegation.clone()); + } + body["workdir_bindings"] = serde_json::json!([{ + "alias": "docs", "working_directory_id": fixture.docs_workdir_id, + "connection_id": api.store.list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker).unwrap().into_iter().find(|link| link.workdir_id == fixture.docs_workdir_id && link.unlinked_at.is_none()).unwrap().connection_id + }]); + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + fixture.ticket_id + ); + start_http_response( + api, + start_http_request(source.as_ref(), &token, "PUT", &path, body), + StatusCode::OK, + ) + .await; + fixture.worker.clone() + } else { + // This matrix tests read-only start, not sharing an occupied + // Workdir. Release the fixture candidate's reference attachment. + let docs = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .into_iter() + .find(|link| { + link.workdir_id == fixture.docs_workdir_id && link.unlinked_at.is_none() + }) + .unwrap(); + api.store + .detach_worker_workdir_connection( + TEST_WORKSPACE_ID, + &fixture.worker, + &docs.alias, + &docs.connection_id, + &now_registry_timestamp(), + ) + .unwrap(); + sync_runtime_worker_workdir_attachments(api, &fixture.worker).unwrap(); + let path = if orchestrator { + format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers") + } else { + format!("/api/w/{TEST_WORKSPACE_ID}/workers") + }; + let body = start_worker_body( + Some(&fixture.ticket_id), + serde_json::json!([{ + "alias": "docs", "working_directory_id": fixture.docs_workdir_id + }]), + delegation.as_deref(), + ); + let created = start_http_response( + api, + start_http_request(source.as_ref(), &token, "POST", &path, body), + StatusCode::OK, + ) + .await; + assert_eq!(created["worker"]["profile"], "builtin:intake"); + RuntimeWorkerRef::new( + WorkdirlessFixtureRuntime::RUNTIME_ID, + created["worker_id"].as_str().unwrap(), + ) + }; + let links = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &worker) + .unwrap() + .into_iter() + .filter(|link| link.unlinked_at.is_none()) + .collect::>(); + assert_eq!(links.len(), 1); + assert_eq!(links[0].workdir_id, fixture.docs_workdir_id); + assert_eq!( + links[0].capabilities, + workdir::WorkdirSessionCapabilities::READ_ONLY + ); + if existing { + let attachments = fixture.runtime.logical_attachments(&worker.worker_id); + assert_eq!(attachments.len(), 1); + assert_eq!( + attachments[0].capabilities, + workdir::WorkdirSessionCapabilities::READ_ONLY + ); + } else { + // This adapter records spawn inputs separately from the + // replacement-attachment calls used by existing-Worker assign. + let request = fixture.runtime.spawn_requests().last().unwrap().clone(); + assert_eq!(request.resolved_workdir_attachments.len(), 1); + assert_eq!( + request.resolved_workdir_attachments[0].working_directory_id, + fixture.docs_workdir_id + ); + assert_eq!( + request.resolved_workdir_attachments[0].capabilities, + workdir::WorkdirSessionCapabilities::READ_ONLY + ); + } + assert_start_dependency_diagnostic_only(api, &fixture.ticket_id, &dependency); + } + } +} + +#[tokio::test] +async fn unfinished_dependencies_do_not_bypass_http_auth_closed_or_resource_binding_guards() { + let fixture = manual_worker_assignment_fixture().await; + let api = &fixture.api; + add_unfinished_start_dependency(api, &fixture.ticket_id); + let token = seed_test_api_token(api.store.as_ref(), "guard-start-owner"); + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + fixture.ticket_id + ); + let valid = + serde_json::to_value(manual_worker_assignment_request(&fixture, "guarded-assign")).unwrap(); + let denied = start_http_response( + api, + start_http_request(None, "invalid-token", "PUT", &path, valid.clone()), + StatusCode::UNAUTHORIZED, + ) + .await; + assert!(denied["message"].is_string()); + let mut stale = valid.clone(); + stale["workdir_bindings"][0]["connection_id"] = Value::String("stale-connection".into()); + let denied = start_http_response( + api, + start_http_request(None, &token, "PUT", &path, stale), + StatusCode::CONFLICT, + ) + .await; + assert!(denied["message"].as_str().unwrap().contains("connection")); + assert!( + api.store + .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &fixture.ticket_id) + .unwrap() + .is_none() + ); + // The existing case/work-release trigger is separate from relation diagnostics. + rusqlite::Connection::open(&api.config.database_path).unwrap().execute( + "UPDATE typed_tickets SET workflow_state='closed' WHERE workspace_id=?1 AND ticket_id=?2", + rusqlite::params![TEST_WORKSPACE_ID, fixture.ticket_id], + ).unwrap(); + let denied = start_http_response( + api, + start_http_request(None, &token, "PUT", &path, valid), + StatusCode::CONFLICT, + ) + .await; + assert!( + denied["message"].as_str().unwrap().contains("open") + || denied["message"].as_str().unwrap().contains("closed") + ); + assert!( + api.store + .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &fixture.ticket_id) + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn orchestrator_start_and_assign_reject_missing_stale_cleared_or_reissued_delegation() { + for existing in [false, true] { + for invalidation in ["missing", "stale", "clear", "reissue"] { + let directory = tempfile::tempdir().unwrap(); + let mut api = test_api(directory.path()).await; + register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; + let runtime = WorkdirlessFixtureRuntime::default(); + api.runtime.register_or_replace(runtime.clone()); + let source = start_source(&mut api, &runtime).await; + let token = seed_test_api_token(api.store.as_ref(), "delegation-owner"); + let ticket = browser_ticket_backend(&api) + .unwrap() + .create(ticket::NewTicket::new("Observed delegation start")) + .unwrap(); + assign_test_orchestrator(&api, &ticket.id); + let observed = observed_start_delegation(&api, &ticket.id, Some(&source)).unwrap(); + let worker = if existing { + Some( + start_http_response( + &api, + start_http_request( + None, + &token, + "POST", + &format!("/api/w/{TEST_WORKSPACE_ID}/workers"), + start_worker_body(None, serde_json::json!([]), None), + ), + StatusCode::OK, + ) + .await, + ) + } else { + None + }; + if matches!(invalidation, "clear" | "reissue") { + let clear = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/orchestrator?assignment_id={observed}&operation_id=clear-observed", + ticket.id + ); + start_http_response( + &api, + start_http_request(None, &token, "DELETE", &clear, Value::Null), + StatusCode::OK, + ) + .await; + if invalidation == "reissue" { + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/orchestrator", + ticket.id + ); + start_http_response(&api, start_http_request(None, &token, "PUT", &path, + serde_json::json!({"operation_id":"reissue-observed", "principal":{"kind":"workspace_agent", "agent_key":"workspace-orchestrator"}})), StatusCode::OK).await; + assert_ne!( + observed_start_delegation(&api, &ticket.id, Some(&source)).unwrap(), + observed + ); + } + } + let submitted = match invalidation { + "missing" => None, + "stale" => Some("unrelated-assignment"), + _ => Some(observed.as_str()), + }; + let (method, path, body) = if let Some(worker) = worker { + let mut body = serde_json::json!({ + "operation_id":"invalid-existing-start", + "principal":{"kind":"worker", "runtime_id":WorkdirlessFixtureRuntime::RUNTIME_ID, "worker_id":worker["worker_id"]}, + "workdir_bindings":[] + }); + if let Some(submitted) = submitted { + body["orchestrator_assignment_id"] = Value::String(submitted.into()); + } + ( + "PUT", + format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + ticket.id + ), + body, + ) + } else { + ( + "POST", + format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers"), + start_worker_body(Some(&ticket.id), serde_json::json!([]), submitted), + ) + }; + let spawns = runtime.spawn_requests().len(); + let denied = start_http_response( + &api, + start_http_request(Some(&source), &token, method, &path, body), + StatusCode::CONFLICT, + ) + .await; + assert!( + denied["message"].as_str().unwrap().contains("assignment") + || denied["message"].as_str().unwrap().contains("delegation"), + "{invalidation}: {denied}" + ); + assert_eq!(runtime.spawn_requests().len(), spawns); + assert!( + api.store + .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &ticket.id) + .unwrap() + .is_none() + ); + let operation = if existing { + "invalid-existing-start" + } else { + "partial-start" + }; + assert!( + api.store + .get_ticket_assignment_operation(TEST_WORKSPACE_ID, operation) + .unwrap() + .is_none() + ); + } + } +} + +#[tokio::test] +async fn replaced_orchestrator_cannot_start_or_assign_under_retained_current_delegation() { + for existing in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let mut api = test_api(directory.path()).await; + register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; + let runtime = WorkdirlessFixtureRuntime::default(); + api.runtime.register_or_replace(runtime.clone()); + let source = start_source(&mut api, &runtime).await; + let token = seed_test_api_token(api.store.as_ref(), "obsolete-worker-owner"); + let ticket = browser_ticket_backend(&api) + .unwrap() + .create(ticket::NewTicket::new( + "Retained delegation after Worker replacement", + )) + .unwrap(); + assign_test_orchestrator(&api, &ticket.id); + let delegation = observed_start_delegation(&api, &ticket.id, Some(&source)).unwrap(); + let worker = if existing { + Some( + start_http_response( + &api, + start_http_request( + None, + &token, + "POST", + &format!("/api/w/{TEST_WORKSPACE_ID}/workers"), + start_worker_body(None, serde_json::json!([]), None), + ), + StatusCode::OK, + ) + .await, + ) + } else { + None + }; + // Backend ownership changes independently of the retained assignment row. + let replacement = + RuntimeWorkerRef::new(&source.worker.runtime_id, WorkerId::now_v7().to_string()); + seed_worker_source_member(&api, &replacement.runtime_id, &replacement.worker_id); + let mut summary = runtime.workers.lock().unwrap()[0].clone(); + summary.worker = replacement.clone(); + runtime.workers.lock().unwrap().push(summary); + rusqlite::Connection::open(&api.config.database_path).unwrap().execute( + "UPDATE worker_singleton_owners SET worker_id=?1,generation=generation+1 WHERE workspace_id=?2 AND singleton_key=?3", + rusqlite::params![replacement.worker_id, TEST_WORKSPACE_ID, crate::hosts::WORKSPACE_ORCHESTRATOR_SINGLETON_KEY], + ).unwrap(); + assert_eq!( + find_workspace_orchestrator(&api).unwrap().worker, + replacement + ); + assert_eq!( + observed_start_delegation(&api, &ticket.id, Some(&source)).unwrap(), + delegation + ); + let (method, path, body) = if let Some(worker) = worker { + ( + "PUT", + format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + ticket.id + ), + serde_json::json!({ + "operation_id":"obsolete-existing-start", "orchestrator_assignment_id":delegation, + "principal":{"kind":"worker", "runtime_id":WorkdirlessFixtureRuntime::RUNTIME_ID, "worker_id":worker["worker_id"]}, + "workdir_bindings":[] + }), + ) + } else { + ( + "POST", + format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers"), + start_worker_body(Some(&ticket.id), serde_json::json!([]), Some(&delegation)), + ) + }; + let spawns = runtime.spawn_requests().len(); + let denied = start_http_response( + &api, + start_http_request(Some(&source), &token, method, &path, body), + // The authenticated Worker-source middleware fences the replaced + // singleton before the start/assign handler is admitted. + StatusCode::UNAUTHORIZED, + ) + .await; + assert!(denied["message"].is_string()); + assert_eq!(runtime.spawn_requests().len(), spawns); + assert!( + api.store + .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &ticket.id) + .unwrap() + .is_none() + ); + } +} diff --git a/crates/workspace-server/src/server_assignment_tracking_tests.rs b/crates/workspace-server/src/server_assignment_tracking_tests.rs index 138fae02b..7ad464549 100644 --- a/crates/workspace-server/src/server_assignment_tracking_tests.rs +++ b/crates/workspace-server/src/server_assignment_tracking_tests.rs @@ -68,7 +68,9 @@ async fn runtime_and_account_assign_retries_return_saved_result_after_clear_and_ })).unwrap(); tracking_response( app.clone(), - request("PUT", &worker_path, worker_input), + Request::builder().method("PUT").uri(&worker_path).header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from(worker_input)).unwrap(), StatusCode::OK, ) .await; @@ -117,15 +119,15 @@ async fn runtime_and_account_assign_retries_return_saved_result_after_clear_and_ .await, saved ); - // Only a genuinely new operation is refused; a receipt never restores - // the cleared current pointer, initializes tracking, or starts work. - tracking_response( + // Receipt replay does not recreate a delegation or transfer manual work. + assert_eq!(counts(), before); + let fresh = tracking_response( app, request("PUT", &path, input("new-assign")), - StatusCode::CONFLICT, + StatusCode::OK, ) .await; - assert_eq!(counts(), before); + assert_ne!(fresh["assignment"]["assignment_id"].as_str(), Some(assignment)); assert_eq!( api.store .get_ticket_assignment_tracking(TEST_WORKSPACE_ID, &ticket, assignment) @@ -144,16 +146,9 @@ async fn runtime_and_account_assign_retries_return_saved_result_after_clear_and_ .unwrap(), active ); - assert!( - api.store - .get_current_ticket_role_assignment( - TEST_WORKSPACE_ID, - &ticket, - TicketAssignmentRole::Orchestrator - ) - .unwrap() - .is_none() - ); + assert_eq!(api.store.get_current_ticket_role_assignment(TEST_WORKSPACE_ID, &ticket, + TicketAssignmentRole::Orchestrator).unwrap().unwrap().assignment_id, + fresh["assignment"]["assignment_id"].as_str().unwrap()); } } diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index c88faeec8..a709fa672 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -27,8 +27,11 @@ use crate::backend_job::{ use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; +#[path = "assignment_delivery.rs"] +mod assignment_delivery; #[path = "assignment_tracking.rs"] mod assignment_tracking; +pub use assignment_delivery::TicketAssignmentDelivery; pub use assignment_tracking::TicketOrchestratorAssignmentSourceContext; pub use ticket::{ TicketAssignmentTracking, TicketAssignmentTrackingHistoryEntry, @@ -37,7 +40,7 @@ pub use ticket::{ }; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 92; +const LATEST_SCHEMA_VERSION: i64 = 93; const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants"; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; @@ -305,6 +308,11 @@ const MIGRATIONS: &[Migration] = &[ name: "Ticket-owned assignment tracking", apply: migrate_ticket_tracking_v91_to_v92, }, + Migration { + version: 93, + name: "Durable assignment deliveries and source-bound startup fences", + apply: migrate_assignment_delivery_v92_to_v93, + }, ]; #[derive(Clone, Copy)] @@ -1804,6 +1812,52 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { request_fingerprint: &str, created_at: &str, ) -> Result; + /// Common scan for committed current delegation inputs, not handling status. + fn list_pending_ticket_assignment_deliveries( + &self, + workspace_id: &str, + limit: usize, + ) -> Result>; + /// Frozen payload/request identity for exact response-loss retry. + fn reserve_ticket_assignment_delivery( + &self, + workspace_id: &str, + assignment_id: &str, + input_event_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result; + /// Freeze exact rendered Runtime request bytes before dispatch. + fn prepare_ticket_assignment_delivery_content( + &self, + workspace_id: &str, + delivery_id: &str, + content: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result; + fn ack_ticket_assignment_delivery( + &self, + workspace_id: &str, + delivery_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result<()>; + fn reserve_ticket_assignment_operation_for_orchestrator( + &self, + workspace_id: &str, + operation_id: &str, + ticket_id: &str, + runtime_id: &str, + worker_id: Option<&str>, + request_fingerprint: &str, + created_at: &str, + orchestrator_assignment_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result; + /// Recovery/pre-dispatch check; write boundaries also revalidate this fence. + fn validate_ticket_assignment_operation_fence( + &self, + workspace_id: &str, + operation_id: &str, + ) -> Result<()>; fn bind_ticket_assignment_operation_worker( &self, workspace_id: &str, @@ -7598,92 +7652,90 @@ impl ControlPlaneStore for SqliteWorkspaceStore { request_fingerprint: &str, created_at: &str, ) -> Result { - validate_non_empty("operation_id", operation_id)?; - validate_non_empty("request_fingerprint", request_fingerprint)?; + self.reserve_ticket_assignment_operation_with_fence( + workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + request_fingerprint, + created_at, + None, + ) + } + fn reserve_ticket_assignment_operation_for_orchestrator( + &self, + workspace_id: &str, + operation_id: &str, + ticket_id: &str, + runtime_id: &str, + worker_id: Option<&str>, + request_fingerprint: &str, + created_at: &str, + orchestrator_assignment_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result { + self.reserve_ticket_assignment_operation_with_fence( + workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + request_fingerprint, + created_at, + Some((orchestrator_assignment_id, source)), + ) + } + fn validate_ticket_assignment_operation_fence( + &self, + workspace_id: &str, + operation_id: &str, + ) -> Result<()> { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - if let Some(existing) = read_assignment_operation(&tx, workspace_id, operation_id)? { - if existing.action == "assign" - && existing.ticket_id == ticket_id - && existing.runtime_id.as_deref() == Some(runtime_id) - && (worker_id.is_none() - || existing - .worker - .as_ref() - .map(|worker| worker.worker_id.as_str()) - == worker_id) - && existing.expected_assignment_id.is_none() - && existing.request_fingerprint.as_deref() == Some(request_fingerprint) - { - tx.commit()?; - return Ok(false); - } - return Err(Error::TicketAssignmentConflict(format!( - "assignment operation {operation_id} was already used with different input" - ))); - } - let worker = worker_id.map(|id| RuntimeWorkerRef::new(runtime_id, id)); - ensure_no_competing_ticket_claim( - &tx, - workspace_id, - ticket_id, - worker.as_ref(), - operation_id, - )?; - if let Some(worker) = &worker { - ensure_worker_assignment_available(&tx, workspace_id, worker)?; - let occupied: bool = tx.query_row( - "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments - WHERE workspace_id=?1 AND role='worker' AND runtime_id=?2 AND worker_id=?3)", - params![workspace_id, worker.runtime_id, worker.worker_id], - |row| row.get(0), - )?; - if occupied { - return Err(Error::TicketAssignmentConflict( - "Worker has unfinished Ticket work".into(), - )); - } - } - let state: String = tx.query_row( - "SELECT workflow_state FROM typed_tickets WHERE workspace_id=?1 AND ticket_id=?2", - params![workspace_id, ticket_id], - |row| row.get(0), - )?; - if state != "open" { - return Err(Error::TicketAssignmentConflict(format!( - "Ticket `{ticket_id}` cannot claim work in state `{state}`" - ))); - } - let occupied: bool = tx.query_row( - "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments - WHERE workspace_id=?1 AND ticket_id=?2 AND role='worker')", - params![workspace_id, ticket_id], - |row| row.get(0), - )?; - if occupied { - return Err(Error::TicketAssignmentConflict( - "Ticket has unfinished Worker work".into(), - )); - } - tx.execute( - "INSERT INTO ticket_assignment_operations ( - workspace_id,operation_id,action,ticket_id,runtime_id,worker_id,assignment_id, - expected_assignment_id,created_at,request_fingerprint,claim_state) - VALUES(?1,?2,'assign',?3,?4,?5,NULL,NULL,?6,?7,'pending')", - params![ - workspace_id, - operation_id, - ticket_id, - runtime_id, - worker_id, - created_at, - request_fingerprint - ], - )?; + assignment_delivery::operation_guard(&tx, workspace_id, operation_id)?; tx.commit()?; - Ok(true) + Ok(()) }) } + fn list_pending_ticket_assignment_deliveries( + &self, + workspace_id: &str, + limit: usize, + ) -> Result> { + if !(1..=1000).contains(&limit) { + return Err(Error::InvalidInput( + "Assignment delivery scan limit must be 1..=1000".into(), + )); + } + self.with_conn(|conn| assignment_delivery::list_in_tx(conn, workspace_id, limit)) + } + fn reserve_ticket_assignment_delivery( + &self, + workspace_id: &str, + assignment_id: &str, + input_event_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result { + assignment_delivery::reserve(self, workspace_id, assignment_id, input_event_id, source) + } + fn prepare_ticket_assignment_delivery_content( + &self, + workspace_id: &str, + delivery_id: &str, + content: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result { + assignment_delivery::prepare_content(self, workspace_id, delivery_id, content, source) + } + fn ack_ticket_assignment_delivery( + &self, + workspace_id: &str, + delivery_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result<()> { + assignment_delivery::ack(self, workspace_id, delivery_id, source) + } fn bind_ticket_assignment_operation_worker( &self, @@ -7693,6 +7745,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result<()> { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,workspace_id,operation_id)?; let receipt = read_assignment_operation(&tx, workspace_id, operation_id)? .ok_or_else(|| Error::TicketAssignmentConflict(format!("missing Ticket claim `{operation_id}`")))?; // Exact binding replay is safe even after assignment finalization. @@ -8036,6 +8089,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,&record.workspace_id,operation_id)?; if let Some(worker) = record.principal.worker() { ensure_worker_assignment_available(&tx, &record.workspace_id, &worker)?; } @@ -8091,29 +8145,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { "manual Worker assignment requires an open Ticket; current state is `{state}`" ))); } - let unresolved_blockers: i64 = tx.query_row( - "SELECT COUNT(*) - FROM typed_ticket_relations AS relation - JOIN typed_tickets AS related - ON related.workspace_id = relation.workspace_id - AND related.ticket_id = CASE - WHEN relation.ticket_id = ?2 THEN relation.target - ELSE relation.ticket_id - END - WHERE relation.workspace_id = ?1 - AND ( - (relation.ticket_id = ?2 AND relation.kind = 'depends_on') - OR (relation.target = ?2 AND relation.kind = 'blocks') - ) - AND related.workflow_state = 'open'", - params![record.workspace_id, record.ticket_id], - |row| row.get(0), - )?; - if unresolved_blockers != 0 { - return Err(Error::TicketAssignmentConflict( - "manual Worker assignment is blocked by unresolved Ticket relations".to_string(), - )); - } + // Relations remain diagnostics; they do not reserve work or block partial starts. let conflicting: i64 = tx.query_row( "SELECT COUNT(*) FROM ticket_active_worker_assignments WHERE workspace_id = ?1 AND ticket_id = ?2 @@ -8335,6 +8367,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,&record.workspace_id,operation_id)?; let removal_blocks_assignment: bool = tx.query_row( "SELECT EXISTS( SELECT 1 FROM worker_removal_operations @@ -11360,6 +11393,119 @@ fn ensure_worker_assignment_available( } impl SqliteWorkspaceStore { + fn reserve_ticket_assignment_operation_with_fence( + &self, + workspace_id: &str, + operation_id: &str, + ticket_id: &str, + runtime_id: &str, + worker_id: Option<&str>, + request_fingerprint: &str, + created_at: &str, + fence: Option<(&str, &TicketAssignmentTrackingSourceContext)>, + ) -> Result { + validate_non_empty("operation_id", operation_id)?; + validate_non_empty("request_fingerprint", request_fingerprint)?; + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,workspace_id,operation_id)?; + if fence.is_none() { + let fenced:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_assignment_operation_fences WHERE workspace_id=?1 AND operation_id=?2)",params![workspace_id,operation_id],|r|r.get(0))?; + if fenced {return Err(Error::TicketAssignmentConflict("Source-bound reservation requires authenticated Orchestrator context".into()));} + } + if let Some(existing) = read_assignment_operation(&tx, workspace_id, operation_id)? { + if existing.action == "assign" + && existing.ticket_id == ticket_id + && existing.runtime_id.as_deref() == Some(runtime_id) + && (worker_id.is_none() + || existing + .worker + .as_ref() + .map(|worker| worker.worker_id.as_str()) + == worker_id) + && existing.expected_assignment_id.is_none() + && existing.request_fingerprint.as_deref() == Some(request_fingerprint) + { + if let Some((assignment_id,source))=fence { + assignment_delivery::save_operation_fence(&tx,workspace_id,operation_id,ticket_id,assignment_id,source,true)?; + } + tx.commit()?; + return Ok(false); + } + return Err(Error::TicketAssignmentConflict(format!( + "assignment operation {operation_id} was already used with different input" + ))); + } + if let Some((assignment_id,source))=fence { + assignment_tracking::require_runtime_source(&tx,workspace_id,&source.runtime_id,&source.worker_id,&source.agent_key,&source.authentication_binding_id)?; + assignment_delivery::require_current(&tx,workspace_id,ticket_id,assignment_id)?; + } + let worker = worker_id.map(|id| RuntimeWorkerRef::new(runtime_id, id)); + ensure_no_competing_ticket_claim( + &tx, + workspace_id, + ticket_id, + worker.as_ref(), + operation_id, + )?; + if let Some(worker) = &worker { + ensure_worker_assignment_available(&tx, workspace_id, worker)?; + let occupied: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments + WHERE workspace_id=?1 AND role='worker' AND runtime_id=?2 AND worker_id=?3)", + params![workspace_id, worker.runtime_id, worker.worker_id], + |row| row.get(0), + )?; + if occupied { + return Err(Error::TicketAssignmentConflict( + "Worker has unfinished Ticket work".into(), + )); + } + } + let state: String = tx.query_row( + "SELECT workflow_state FROM typed_tickets WHERE workspace_id=?1 AND ticket_id=?2", + params![workspace_id, ticket_id], + |row| row.get(0), + )?; + if state != "open" { + return Err(Error::TicketAssignmentConflict(format!( + "Ticket `{ticket_id}` cannot claim work in state `{state}`" + ))); + } + let occupied: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments + WHERE workspace_id=?1 AND ticket_id=?2 AND role='worker')", + params![workspace_id, ticket_id], + |row| row.get(0), + )?; + if occupied { + return Err(Error::TicketAssignmentConflict( + "Ticket has unfinished Worker work".into(), + )); + } + tx.execute( + "INSERT INTO ticket_assignment_operations ( + workspace_id,operation_id,action,ticket_id,runtime_id,worker_id,assignment_id, + expected_assignment_id,created_at,request_fingerprint,claim_state) + VALUES(?1,?2,'assign',?3,?4,?5,NULL,NULL,?6,?7,'pending')", + params![ + workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + created_at, + request_fingerprint + ], + )?; + if let Some((assignment_id,source))=fence { + assignment_delivery::save_operation_fence(&tx,workspace_id,operation_id,ticket_id,assignment_id,source,false)?; + } + tx.commit()?; + Ok(true) + }) + } + fn set_ticket_role_assignment_with_source( &self, record: &TicketRoleAssignmentRecord, @@ -11421,6 +11567,7 @@ impl SqliteWorkspaceStore { }; self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,&record.workspace_id,operation_id)?; if let Some(source)=source {assignment_tracking::require_assignment_source(&tx,&record.workspace_id,source)?;} if let Some(worker) = record.principal.worker() { ensure_worker_assignment_available(&tx, &record.workspace_id, &worker)?; @@ -11456,24 +11603,13 @@ impl SqliteWorkspaceStore { return Ok(persisted); } - // This is a constraint on a new delegation, not on a saved result. - // Authenticate the source and resolve its full-input receipt first; - // work that began after the original Assign must not block replay. - if record.role == TicketAssignmentRole::Orchestrator { - let active_worker: bool = tx.query_row( - "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments - WHERE workspace_id=?1 AND ticket_id=?2 AND role='worker')", - params![record.workspace_id, record.ticket_id], - |row| row.get(0), - )?; - if active_worker { - return Err(Error::TicketAssignmentConflict( - "Orchestrator assignment conflicts with an active Worker assignment".into(), - )); - } + // Delegation has no Runtime/resource occupancy. It can invalidate + // an in-flight startup fence; existing actual Worker/claim resource + // exclusion is checked independently at the next work admission. + if record.role != TicketAssignmentRole::Orchestrator { + ensure_no_competing_ticket_claim(&tx, &record.workspace_id, &record.ticket_id, + record.principal.worker().as_ref(), operation_id)?; } - ensure_no_competing_ticket_claim(&tx, &record.workspace_id, &record.ticket_id, - record.principal.worker().as_ref(), operation_id)?; let current_sql = ticket_role_assignment_select_sql( "WHERE current.workspace_id = ?1 AND current.ticket_id = ?2 AND current.role = ?3 \ ORDER BY a.assigned_at, a.assignment_id", @@ -11816,13 +11952,15 @@ fn clear_current_ticket_role_assignment_with_source_in_tx( } return Ok(true); } - ensure_no_competing_ticket_claim( - tx, - workspace_id, - ticket_id, - current.principal.worker().as_ref(), - operation_id, - )?; + if role != TicketAssignmentRole::Orchestrator { + ensure_no_competing_ticket_claim( + tx, + workspace_id, + ticket_id, + current.principal.worker().as_ref(), + operation_id, + )?; + } let is_current:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_current_worker_assignments WHERE workspace_id=?1 AND ticket_id=?2 AND role=?3 AND assignment_id=?4)",params![workspace_id,ticket_id,role.as_str(),assignment_id],|row|row.get(0))?; if !is_current { return Ok(false); @@ -13457,8 +13595,22 @@ fn migrate_ticket_tracking_v91_to_v92(conn: &Connection) -> Result<()> { Ok(()) } +fn migrate_assignment_delivery_v92_to_v93(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 92 { + return Err(Error::Store( + "Assignment delivery migration requires schema 92".into(), + )); + } + let tx = conn.unchecked_transaction()?; + assignment_delivery::install(&tx)?; + tx.execute("INSERT INTO __yoi_schema_migrations(version,name) VALUES(93,'Durable assignment deliveries and source-bound startup fences')",[])?; + tx.commit()?; + Ok(()) +} + fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { create_workspace_schema_structure(conn)?; + assignment_delivery::install(conn)?; ticket::verify_sqlite_ticket_schema(conn).map_err(|error| { Error::Store(format!( "fresh Workspace Ticket schema verification failed: {error}" @@ -13652,6 +13804,37 @@ fn verify_workspace_resource_constraints(conn: &Connection) -> Result<()> { } } } + if current_schema_version(conn)? >= 93 { + for (kind, name) in [ + ("table", "ticket_assignment_delivery_origins"), + ("table", "ticket_assignment_deliveries"), + ("table", "ticket_assignment_delivery_requests"), + ("table", "ticket_assignment_delivery_acknowledgements"), + ("table", "ticket_assignment_operation_fences"), + ("trigger", "ticket_assignment_delivery_origin_insert"), + ("trigger", "ticket_assignment_delivery_origin_no_update"), + ("trigger", "ticket_assignment_delivery_origin_no_delete"), + ("trigger", "ticket_assignment_delivery_payload_no_update"), + ("trigger", "ticket_assignment_delivery_payload_no_delete"), + ("trigger", "ticket_assignment_delivery_request_no_update"), + ("trigger", "ticket_assignment_delivery_request_no_delete"), + ("trigger", "ticket_assignment_delivery_ack_no_update"), + ("trigger", "ticket_assignment_delivery_ack_no_delete"), + ("trigger", "ticket_assignment_operation_fence_no_update"), + ("trigger", "ticket_assignment_operation_fence_no_delete"), + ] { + let present: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type=?1 AND name=?2)", + params![kind, name], + |row| row.get(0), + )?; + if !present { + return Err(Error::Store(format!( + "Assignment delivery constraint {name} is missing" + ))); + } + } + } Ok(()) } @@ -14405,6 +14588,7 @@ fn table_columns(conn: &Connection, table_name: &str) -> Result> { #[cfg(test)] mod tests { include!("assignment_tracking_tests.rs"); + include!("assignment_delivery_tests.rs"); include!("store_workspace_config_tests.rs"); fn drive_grant_fixture(store: &SqliteWorkspaceStore) -> server_api::DriveGrantResponse { @@ -16614,6 +16798,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx name: "Ticket open/closed cases and retained routing evidence".to_string() }, WorkspaceSchemaMigrationStep {version:92, name:"Ticket-owned assignment tracking".into()}, + WorkspaceSchemaMigrationStep {version:93, name:"Durable assignment deliveries and source-bound startup fences".into()}, ] ); @@ -16716,6 +16901,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx (90, "Runtime binding connection configuration and historical verification archive".to_string()), (91, "Ticket open/closed cases and retained routing evidence".to_string()), (92, "Ticket-owned assignment tracking".to_string()), +(93, "Durable assignment deliveries and source-bound startup fences".to_string()), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -19313,9 +19499,9 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .map(|step| step.version) .collect::>(), if version == 89 { - vec![90, 91, 92] + vec![90, 91, 92, 93] } else { - vec![91, 92] + vec![91, 92, 93] } ); drop(conn); @@ -19330,7 +19516,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx assert_eq!(workspace_schema_migration_history(conn)?.into_iter().filter(|(version, _)| *version >= 90).collect::>(), vec![(90, "Runtime binding connection configuration and historical verification archive".into()), (91, "Ticket open/closed cases and retained routing evidence".into()), - (92, "Ticket-owned assignment tracking".into())]); + (92, "Ticket-owned assignment tracking".into()), + (93, "Durable assignment deliveries and source-bound startup fences".into())]); for (table, rows) in unchanged.iter().zip(before.iter()) { assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}"); } @@ -20048,7 +20235,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx } #[tokio::test] - async fn pending_claims_fence_all_assignment_writers_and_worker_binding_until_owner_commits() { + async fn pending_claims_fence_work_writers_but_allow_delegation_clear() { let (_dir, store) = manual_worker_start_store().await; seed_manual_start_ticket(&store, "first", "open"); seed_manual_start_ticket(&store, "second", "open"); @@ -20100,7 +20287,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx "2", None ) - .is_err() + .unwrap() ); assert!( store diff --git a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs index 0f1e65591..0ea9c64e7 100644 --- a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs +++ b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs @@ -265,7 +265,7 @@ fn assert_retained_classification_upgrade(workspace_version: i64, component: i64 drop(conn); let store = SqliteWorkspaceStore::open(&path).unwrap(); store.with_conn(|conn| { - assert_eq!(current_schema_version(conn)?,92); + assert_eq!(current_schema_version(conn)?,93); assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations",[],|row|row.get::<_,i64>(0))?,11); assert_eq!(fixture_auth_rows(conn),auth_before); for (table, rows) in preserved_tables.iter().zip(&preserved) { @@ -493,7 +493,7 @@ fn workspace90_cases9_add_classification_without_reinterpreting_reasoned_history let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { - assert_eq!(current_schema_version(conn)?, 92); + assert_eq!(current_schema_version(conn)?,93); assert_eq!( conn.query_row( "SELECT MAX(version) FROM ticket_schema_migrations", diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index f81403e7e..e241f8aee 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -45,6 +45,11 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ "repository_ssh_host_trust_keys", "repository_ssh_host_trusts", "server_secret_objects", + "ticket_assignment_delivery_acknowledgements", + "ticket_assignment_delivery_requests", + "ticket_assignment_deliveries", + "ticket_assignment_delivery_origins", + "ticket_assignment_operation_fences", "ticket_assignment_operations", "ticket_assignment_work_releases", "ticket_assignment_ticket_tombstones", @@ -479,6 +484,11 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { ("ticket_worker_assignments", "typed_ticket_assignment_identity_no_delete"), ("ticket_worker_assignment_events", "typed_ticket_assignment_events_no_delete"), ("ticket_assignment_operations", "typed_ticket_assignment_receipt_no_delete"), + ("ticket_assignment_deliveries", "ticket_assignment_delivery_payload_no_delete"), + ("ticket_assignment_delivery_requests", "ticket_assignment_delivery_request_no_delete"), + ("ticket_assignment_delivery_acknowledgements", "ticket_assignment_delivery_ack_no_delete"), + ("ticket_assignment_delivery_origins", "ticket_assignment_delivery_origin_no_delete"), + ("ticket_assignment_operation_fences", "ticket_assignment_operation_fence_no_delete"), ("typed_ticket_assignment_tracking_history", "typed_ticket_assignment_tracking_history_no_delete"), ("typed_ticket_assignment_tracking_operations", "typed_ticket_assignment_tracking_operations_no_delete"), ("typed_ticket_case_history", "typed_ticket_case_history_no_delete"), diff --git a/docs/development/ticket-assignment-delivery.md b/docs/development/ticket-assignment-delivery.md new file mode 100644 index 000000000..9945531e1 --- /dev/null +++ b/docs/development/ticket-assignment-delivery.md @@ -0,0 +1,60 @@ +# Assign-driven Orchestrator delivery and start admission (T-742) + +## Separate authorities + +[Ticket assignment tracking](ticket-assignment-tracking.md) remains the sole delegation/handling authority (Ticket component schema 11). Workspace schema 93 adds **operational** input origins, delivery reservations, exact rendered requests, acceptance acknowledgements, and assignment/source-bound start fences. These tables do not create another assignment identity or handling status. + +Assign commits the current pointer, initial waiting, assignment event and source-bound operation receipt together. Its committed tracking-history insertion records an initial input origin in that same transaction. Delivery runs only after commit. Runtime acceptance, running/idle/stop, MR changes, and delivery acknowledgement never update `waiting`/`inprogress`/`done`. + +A delivery acknowledgement means the Runtime accepted the immutable Notify request. It is not an assertion that the model considered the input, satisfied the Ticket, or started execution. Runtime notification/activation receipts provide the corresponding durable session boundary. Self-reported progress/result remains a separate explicit mutation. + +## Common input scan and delivery + +Mutation notification, Backend startup, Orchestrator create (including already-online), accepted restore, replacement, and running→idle all use `list_pending_ticket_assignment_deliveries`. There is no parallel Queue/open-state routing predicate. Openness is a new-work safety boundary, not delegation intent. + +The scan selects current `workspace_agent:workspace-orchestrator` assignments with actionable handling and an explicit committed input origin. It excludes released identities, unknown legacy relationships, done/history and closed cases. External content/answer events after the origin are new inputs; handling self-reports and case/routing history are not. Waiting alone is never a resend reason. Close releases operational work; reopen alone does not revive the old delegation. Explicit reissue creates a new assignment and initial input. + +Schema 92→93 preserves original assignment/event/Queue evidence. Native Assign history establishes its input origin; migrated tracking contributes an input only where its already-reconciled assignment-specific Queue fence is present. Ambiguous/historical rows remain diagnostic data, not fresh requests. Earlier migrations and Ticket schema 11 are unchanged. + +Delivery order: + +1. Resolve the currently registered online Worker and active Runtime binding from Backend authority (a stopped Worker is not automatically restored). +2. Reserve the exact `assignment_id`/`input_event_id` under IMMEDIATE, rechecking current delegation, singleton and binding. +3. Render using the reservation's metadata, including waiting reason and next action. Persist the **entire rendered content**, not just its metadata or a title fingerprint. +4. Send the saved content with the saved `request_id`, outside the transaction. Response loss retains the same request/payload. Concurrent reservations converge on the same identity, and Runtime request receipts deduplicate actual acceptance. +5. Acknowledge accepted delivery under another IMMEDIATE transaction after rechecking current authority. The acknowledgement never changes handling. + +Process-local attention locks prevent concurrent/reentrant send loops; they are not backlog authority. Losing that cache does not lose a delegation or resend acknowledged input. Replacement does not create assignment rows; only pending, unacknowledged inputs are recovered. Accepted history is not replayed into new work merely because a Worker was exchanged. + +## Public start/action handoff for T-743 + +Existing canonical tracking query/history/update endpoints, Tools/WIP, Ticket query/detail, and retained assignment projections remain available; see the tracking document. Model-visible notices include the committed delegation/input identity and bounded waiting/next-action context. The Server attention template is updated here; final Web/TUI/all-Prompt operation alignment belongs to T-743. + +Start request contracts now carry optional `orchestrator_assignment_id`: + +- `CreateWorkspaceWorkerRequest.ticket_assignment` (including Worker control spawn). +- `RuntimeWorkerSpawnRequest.ticket_assignment`. +- `SetTicketRoleAssignmentRequest` for an existing Worker. +- `SpawnTicketWorker` and generic `WorkerSpawn` tool input/lifecycle forwarding. + +An authenticated current Orchestrator **must** supply its observed delegation identity. Backend checks it with the actual `ServerRequestContext.runtime_source.authentication_binding_id`; it never derives the source from model role text or reacquires another binding as authentication proof. Missing/stale/cleared/reissued identities and obsolete Workers cannot admit work. Manual/generic authorized starts omit the delegation field and do not implicitly assign the Orchestrator. New starts and existing Worker assignments share the same reservation/start fence. + +The fenced reservation extends the existing Ticket operation receipt and Worker allocation/recovery identity. Reserve/retry, bind and finalization validate the saved assignment/source under IMMEDIATE. Recovery/pre-dispatch checks use the same fence. Clear/reassign can invalidate a reservation but do not stop, transfer or clean up already-running work. Unknown external outcomes keep their reservation/resource fences for explicit reconciliation rather than permitting a second start. Runtime I/O is never inside a database transaction. + +Unfinished dependencies remain query/relation diagnostics and do not disable manual/Orchestrator new starts or existing Worker assignment. They are never implicitly started. Case closure, authenticated permissions, actual unfinished work/claims, resource/grant conflicts, target containment and recovery remain independent guards. Repository/Workdir-free work, read-only work and caller-selected published Profiles are preserved. + +Action projection permits delegation independently of existing Worker occupancy; actual Worker start eligibility still excludes unfinished Worker work and closed cases. `can_start_manual_worker` no longer includes an unresolved-dependency predicate. `can_assign_orchestrator` permits an explicit fresh Assign/reissue operation even when a current row exists; callers can supply the observed expected assignment for CAS. `can_unassign_orchestrator` uses the retained current pointer, not active-work or case-state filtering. These authoring actions are not Worker transfer. Final browser controls must use these Backend decisions rather than recreating old phase/dependency gates. + +Delivery diagnostics are available through the typed Store scan/reservation/ack records and structured `ticket_assignment_delivery_pending` warning (`assignment_id`, `input_event_id`, `delivery_id`, recipient and error category, never content). Existing `ticket_notification_delivery_failed` remains for recipient failures. No browser acknowledgement/status-edit endpoint is introduced: successful delivery is not a user handling update. T-743 can consume the operational query independently of handling without inventing a third status in tracking. + +## DB caller/transaction handoff for T-748 and protocol boundaries + +Store callers are in `store.rs`; operational SQL/mutations are in `assignment_delivery.rs`. The public Store methods are `list_pending_ticket_assignment_deliveries`, `reserve_ticket_assignment_delivery`, `prepare_ticket_assignment_delivery_content`, `ack_ticket_assignment_delivery`, `reserve_ticket_assignment_operation_for_orchestrator`, and `validate_ticket_assignment_operation_fence`. Existing bind/finalization writes call the saved-operation guard inside their own IMMEDIATE transaction. + +Operational tables: `ticket_assignment_delivery_origins`, `ticket_assignment_deliveries`, `ticket_assignment_delivery_requests`, `ticket_assignment_delivery_acknowledgements`, `ticket_assignment_operation_fences`. Workspace explicit purge inventory includes their scoped rows and temporarily suspends/restores their immutable-delete guards in the existing guarded purge transaction. Ordinary operations cannot delete or rewrite receipts. + +T-741 set/clear/update/history remain unchanged in transaction ownership: actual source binding/singleton checks, receipt comparison, current/event CAS, audit and result must stay together. A future DB blocking adapter must move the whole synchronous transaction as one unit, not split its queries around awaits. Server Runtime method admission and `ServerRuntimeMutationSourceContext.authentication_binding_id` are not changed. T-739 close/reopen and T-715 identity-specific work releases remain execution boundaries. + +## Validation limits + +Local Store tests, authenticated HTTP tests, real embedded Controller/session tests, and loopback remote Runtime HTTP tests prove persistence, concurrency, source/assignment fencing, payload replay, resource contracts and activation dedupe. They do not prove model judgment quality or final Web/TUI/all-Prompt behavior (T-743/T-744). No live DB migration, environment update, target integration, Ticket conclusion or Runtime Worker/Workdir cleanup is part of this handoff. diff --git a/openapi/server-api.json b/openapi/server-api.json index 78819a744..a8b846fa9 100644 --- a/openapi/server-api.json +++ b/openapi/server-api.json @@ -1606,6 +1606,13 @@ "operation_id": { "type": "string" }, + "orchestrator_assignment_id": { + "description": "Observed delegation required for Orchestrator Ticket starts; manual starts omit it.", + "type": [ + "string", + "null" + ] + }, "ticket_id": { "type": "string" } @@ -8953,6 +8960,13 @@ "operation_id": { "type": "string" }, + "orchestrator_assignment_id": { + "description": "Observed delegation required for Orchestrator Ticket starts; manual starts omit it.", + "type": [ + "string", + "null" + ] + }, "ticket_id": { "type": "string" } @@ -9304,6 +9318,13 @@ "operation_id": { "type": "string" }, + "orchestrator_assignment_id": { + "description": "Observed delegation required for Orchestrator Ticket starts; manual starts omit it.", + "type": [ + "string", + "null" + ] + }, "principal": { "$ref": "#/components/schemas/TicketAssignmentPrincipal" }, @@ -16813,7 +16834,7 @@ "info": { "title": "Yoi Server API", "version": "0.1.0", - "x-yoi-source-digest": "sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774" + "x-yoi-source-digest": "sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b" }, "openapi": "3.1.0", "paths": { diff --git a/resources/prompts/internal/workspace_orchestrator_queue_attention.md b/resources/prompts/internal/workspace_orchestrator_queue_attention.md index eee42a88e..87c914e12 100644 --- a/resources/prompts/internal/workspace_orchestrator_queue_attention.md +++ b/resources/prompts/internal/workspace_orchestrator_queue_attention.md @@ -1,8 +1,17 @@ -Queued Tickets require attention: +Assigned Tickets have unreceived input: {% for ticket in tickets -%} - {{ ticket.resource_key }} {{ separator }} {{ ticket.title }} +{% if ticket.assignment_id -%} + Delegation: {{ ticket.assignment_id }}; input: {{ ticket.input_event_id }}. +{% endif -%} +{% if ticket.waiting_reason -%} + Waiting reason: {{ ticket.waiting_reason }} +{% endif -%} +{% if ticket.next_action -%} + Next action: {{ ticket.next_action }} +{% endif -%} {% endfor -%} {% if omitted_ticket_count > 0 -%} -Additional queued Tickets were omitted from this notice: {{ omitted_ticket_count }}. Re-query current Ticket authority for the complete set. +Additional assigned Tickets were omitted from this notice: {{ omitted_ticket_count }}. Re-query current Ticket authority for the complete set. {% endif -%} -Reread the current Ticket state before acting. Preserve the human queue gate and current assignment, dependency, Worker, and Workdir authority; do not create duplicate work. +Reread the current Ticket state before acting, including current assignment tracking and new input. Assign is delegation; no ready/Queue step is required. Handling status is your self-report, not delivery or Worker state. Waiting alone and done history are not new work. Dependencies are diagnostics for partial work, not a blanket start gate; never implicitly start dependencies. Preserve current assignment identity, authenticated Worker, unfinished work, and Workdir/resource authority; do not create duplicate work. diff --git a/web/workspace/src/lib/generated/companion-api.ts b/web/workspace/src/lib/generated/companion-api.ts index 88c4da3bb..d40e0565a 100644 --- a/web/workspace/src/lib/generated/companion-api.ts +++ b/web/workspace/src/lib/generated/companion-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-companion-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/companion-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check diff --git a/web/workspace/src/lib/generated/repository-api.ts b/web/workspace/src/lib/generated/repository-api.ts index a3f8fc78e..307f65277 100644 --- a/web/workspace/src/lib/generated/repository-api.ts +++ b/web/workspace/src/lib/generated/repository-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-repository-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output // Output: web/workspace/src/lib/generated/repository-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check diff --git a/web/workspace/src/lib/generated/runtime-api.ts b/web/workspace/src/lib/generated/runtime-api.ts index 7a78b1505..516e54f07 100644 --- a/web/workspace/src/lib/generated/runtime-api.ts +++ b/web/workspace/src/lib/generated/runtime-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-runtime-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/runtime-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check @@ -151,7 +151,7 @@ export type RuntimeWorkerSpawnWorkingDirectoryRequest = { repository_key: string export type RuntimeWorkerSummary = { diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: unknown; workspace: WorkerWorkspaceSummary; }; -export type RuntimeWorkerTicketAssignmentRequest = { operation_id: string; ticket_id: string; }; +export type RuntimeWorkerTicketAssignmentRequest = { operation_id: string; orchestrator_assignment_id?: string | null; ticket_id: string; }; export type Segment = { content: string; kind: "text"; } | { chars: number; content: string; id: number; kind: "paste"; lines: number; } | { artifact: PasteArtifactRef; kind: "paste_artifact"; } | { file: UploadedFileRef; kind: "uploaded_file"; } | { kind: "file_ref"; path: string; } | { invocation: FeatureInvocation; kind: "feature_invoke"; } | { kind: "flow"; selector: string; } | { kind: "unknown"; }; diff --git a/web/workspace/src/lib/generated/ticket-api.ts b/web/workspace/src/lib/generated/ticket-api.ts index 60e817c7a..7c9829198 100644 --- a/web/workspace/src/lib/generated/ticket-api.ts +++ b/web/workspace/src/lib/generated/ticket-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/ticket-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check @@ -83,7 +83,7 @@ export type ReviewDecision = "approve" | "request_changes"; export type ReviewFinding = { body: string; code?: string | null; line?: number | null; path?: string | null; severity: FindingSeverity; }; -export type SetTicketRoleAssignmentRequest = { expected_assignment_id?: string | null; operation_id: string; principal: TicketAssignmentPrincipal; workdir_bindings?: Array; }; +export type SetTicketRoleAssignmentRequest = { expected_assignment_id?: string | null; operation_id: string; orchestrator_assignment_id?: string | null; principal: TicketAssignmentPrincipal; workdir_bindings?: Array; }; export type TicketActionEligibility = { blockers: Array; can_assign_orchestrator: boolean; can_queue: boolean; can_start_manual_worker: boolean; can_unassign_orchestrator: boolean; queue_tickets: Array; }; diff --git a/web/workspace/src/lib/generated/worker-launch-api.ts b/web/workspace/src/lib/generated/worker-launch-api.ts index 8db0ab85e..3868137c5 100644 --- a/web/workspace/src/lib/generated/worker-launch-api.ts +++ b/web/workspace/src/lib/generated/worker-launch-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-worker-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/worker-launch-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check @@ -13,7 +13,7 @@ export type BrowserWorkspaceOrchestratorResponse = { diagnostics: Array; profile?: string | null; runtime_id: string; singleton_key?: string | null; ticket_assignment?: CreateWorkspaceWorkerTicketAssignmentRequest | null; workdir_attachments?: Array; }; -export type CreateWorkspaceWorkerTicketAssignmentRequest = { operation_id: string; ticket_id: string; }; +export type CreateWorkspaceWorkerTicketAssignmentRequest = { operation_id: string; orchestrator_assignment_id?: string | null; ticket_id: string; }; export type Diagnostic = { code: string; message: string; severity: DiagnosticSeverity; }; From 56795fb92cb9def9bcd701bd51d92e99526f8eff Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 12 Oct 2026 05:39:33 +0900 Subject: [PATCH 2/2] fix: drain concurrent assignment inputs and discard stale pending retries --- crates/workspace-server/src/server.rs | 47 ++-- .../src/server_assignment_attention_tests.rs | 23 +- .../src/server_assignment_delivery_tests.rs | 210 +++++++++++++----- .../development/ticket-assignment-delivery.md | 2 +- 4 files changed, 199 insertions(+), 83 deletions(-) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index f536b55aa..5e5742bb5 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1528,7 +1528,7 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { #[derive(Default)] struct OrchestratorAttentionState { sending: bool, - restore_recheck: bool, + input_recheck: bool, accepted: Option, pending: Option, } @@ -1547,10 +1547,11 @@ impl OrchestratorAttentionReservation { fn complete(mut self) -> bool { let state = self.0.take().expect("active attention reservation"); let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); - // Release and take deferred work atomically. A Restore arriving after - // release can reserve its own check; one arriving before is ours to drain. + // Release and take deferred input checks atomically. A committed input or + // Restore arriving before release is ours to drain; one after release + // can reserve its own check. state.sending = false; - std::mem::take(&mut state.restore_recheck) + std::mem::take(&mut state.input_recheck) } } @@ -18865,12 +18866,22 @@ fn maybe_dispatch_orchestrator_turn_end( if orchestrator.worker.runtime_id == EMBEDDED_WORKER_RUNTIME_ID && orchestrator.worker.worker_id == worker_id { - dispatch_orchestrator_assignment_attention(api, &orchestrator.worker); + // Idle can be a reentrant consequence of this very Runtime send. It is + // not evidence of a new input and must not retry a failed send forever. + // Concurrent committed-input notifications independently defer a check. + with_orchestrator_attention_reservation( + api, + &orchestrator.worker, + false, + |state, pending| { + check_orchestrator_assignment_attention(api, &orchestrator.worker, state, pending); + }, + ); } } fn dispatch_orchestrator_assignment_attention(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { - with_orchestrator_attention_reservation(api, worker, |state, pending| { + with_orchestrator_attention_reservation(api, worker, true, |state, pending| { check_orchestrator_assignment_attention(api, worker, state, pending); }); } @@ -18878,6 +18889,7 @@ fn dispatch_orchestrator_assignment_attention(api: &WorkspaceApi, worker: &Runti fn with_orchestrator_attention_reservation( api: &WorkspaceApi, worker: &RuntimeWorkerRef, + defer_input_check_if_reserved: bool, mut check: impl FnMut( &Arc>, Option, @@ -18888,6 +18900,9 @@ fn with_orchestrator_attention_reservation( let pending = { let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); if state.sending { + // A no-op scan may already have read its empty snapshot. Retain + // committed input/startup/Restore triggers until its completion. + state.input_recheck |= defer_input_check_if_reserved; return; } state.sending = true; @@ -18898,8 +18913,8 @@ fn with_orchestrator_attention_reservation( if !reservation.complete() { return; } - // Accepted Restore is an explicit durable-input recheck even when the - // current check was a no-op. A local cache is never backlog authority. + // Drain coalesced input triggers even when the first scan was a no-op. + // No recheck is created by a failed send or its own idle observation. } } @@ -18951,10 +18966,19 @@ fn check_orchestrator_assignment_attention( .is_ok() { send_orchestrator_attention(api, worker, state, delivery); + if state + .lock() + .unwrap_or_else(|p| p.into_inner()) + .pending + .is_some() + { + return; // Unknown/failed acceptance needs an explicit later trigger. + } } else { state.lock().unwrap_or_else(|p| p.into_inner()).pending = None; } - return; + // Discarding or acknowledging an old retry must not consume this new- + // input trigger. Continue the common scan in the same reservation. } let Ok(inputs) = api .store @@ -22969,11 +22993,6 @@ async fn scoped_start_workspace_orchestrator( { attention.accepted = None; } - // A reserved caller may already have decided not to send. Carry - // the explicit Restore check through its completion in that case. - if attention.sending { - attention.restore_recheck = true; - } } drop(_guard); dispatch_orchestrator_assignment_attention(&api, &existing.worker); diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index 6da70dc10..d72a8e735 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -154,7 +154,7 @@ async fn orchestrator_attention_barrier_callers_share_unsent_reservation() { } #[tokio::test] -async fn orchestrator_attention_restore_rechecks_reserved_noop_before_and_after_comparison() { +async fn orchestrator_attention_input_rechecks_reserved_noop_before_and_after_comparison() { for pause_before_comparison in [true, false] { let (_dir, api, execution) = attention_test_api().await; let started = start_attention_test_orchestrator(&api).await; @@ -187,6 +187,7 @@ async fn orchestrator_attention_restore_rechecks_reserved_noop_before_and_after_ with_orchestrator_attention_reservation( &api_for_caller, &worker_for_caller, + true, |state, pending| { if checks == 0 && pause_before_comparison { entered_tx.send(()).unwrap(); @@ -224,7 +225,7 @@ async fn orchestrator_attention_restore_rechecks_reserved_noop_before_and_after_ assert!(state.lock().unwrap().sending); let restored = futures::executor::block_on(start_attention_test_orchestrator(&api)); assert_eq!(restored.disposition, "restored"); - let deferred = state.lock().unwrap().restore_recheck; + let deferred = state.lock().unwrap().input_recheck; let old_retired = state.lock().unwrap().accepted.is_none(); let inputs_before_release = execution.take_inputs(); release_tx.send(()).unwrap(); @@ -239,7 +240,7 @@ async fn orchestrator_attention_restore_rechecks_reserved_noop_before_and_after_ let state = state.lock().unwrap(); assert!(state.accepted.is_none()); assert!(!state.sending); - assert!(!state.restore_recheck); + assert!(!state.input_recheck); } } @@ -308,19 +309,19 @@ async fn orchestrator_attention_unknown_response_retry_keeps_id_and_payload() { "existing" ); let retry = execution.take_inputs(); - assert_eq!(retry.len(), 1); + assert_eq!(retry.len(), 2, "acknowledged retry must also drain the changed backlog"); assert_eq!(retry[0].1, pending.content); - assert_eq!( - execution.take_input_request_ids(), - vec![Some(pending.request_id)] - ); + assert!(retry[1].1.contains("Changed backlog")); + let ids = execution.take_input_request_ids(); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], Some(pending.request_id)); + assert_ne!(ids[1], ids[0]); assert!(state.lock().unwrap().pending.is_none()); assert!(state.lock().unwrap().accepted.is_some()); + assert!(attention_pending(&api).is_empty()); use protocol::subscription::SubscriptionWorkerState as S; maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, Some(S::Running), S::Idle); - let changed = execution.take_inputs(); - assert_eq!(changed.len(), 1); - assert!(changed[0].1.contains("Changed backlog")); + assert!(execution.take_inputs().is_empty()); } #[tokio::test] diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index b8821a66b..489674c19 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -36,7 +36,11 @@ async fn delivery_http( "{method} {path}: {}", String::from_utf8_lossy(&bytes) ); - serde_json::from_slice(&bytes).unwrap() + if bytes.is_empty() { + Value::Null + } else { + serde_json::from_slice(&bytes).unwrap() + } } async fn delivery_assign(api: &WorkspaceApi, ticket: &str, key: &str) -> Value { @@ -330,65 +334,157 @@ async fn assignment_done_historical_inputs_do_not_restart_work_but_reissue_is_ne #[tokio::test] async fn assignment_clear_and_reassign_discard_old_unknown_delivery_without_stopping_worker() { + for clear_first in [true, false] { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, _) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "Fence old delivery"); + execution.reject_inputs("unknown acceptance response"); + let assigned = delivery_assign(&api, &ticket.id, "assign-old").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + let old_attempts = execution.take_inputs(); + let old_ids = execution.take_input_request_ids(); + assert!(!old_attempts.is_empty()); + assert!(old_ids.iter().all(|id| id == &old_ids[0])); + if clear_first { + delivery_http( + &api, + "DELETE", + &format!( + "{}?assignment_id={assignment}&operation_id=clear-old", + delivery_assignment_path(&ticket.id) + ), + None, + StatusCode::OK, + ) + .await; + } + *execution.input_failure.lock().unwrap() = None; + // No extra scan, old-operation retry, or synthetic turn end between + // clearing/reissuing the failed input and this committed new Assign. + let reassigned = delivery_assign(&api, &ticket.id, "assign-new").await; + let new_assignment = reassigned["assignment"]["assignment_id"].as_str().unwrap(); + assert_ne!(new_assignment, assignment); + let inputs = execution.take_inputs(); + assert_eq!(inputs.len(), 1, "new Assign must not need another scan"); + assert!(inputs[0].1.contains(new_assignment)); + let ids = execution.take_input_request_ids(); + assert_eq!(ids.len(), 1); + assert_ne!(ids[0], old_ids[0], "new assignment reused stale input identity"); + assert_eq!( + delivery_assign(&api, &ticket.id, "assign-old").await, + assigned + ); + assert!(execution.take_inputs().is_empty(), "old retry revived work"); + let old = delivery_tracking(&api, &ticket.id, assignment); + assert!(!old.current); + assert_eq!( + old.tracking.unwrap().status, + ticket::TicketAssignmentTrackingStatus::Waiting, + "clear/reissue must not rewrite the old status to done" + ); + assert_eq!( + find_workspace_orchestrator(&api).unwrap().worker, + worker, + "clear/reassign are not Worker stop or replacement" + ); + delivery_stop(&api, &worker); + } +} + +#[tokio::test] +async fn assignment_or_answer_committed_during_empty_scan_is_delivered_without_another_trigger() { + for answer in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, identity) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "Input racing an empty scan"); + if answer { + let assigned = delivery_assign(&api, &ticket.id, "initial-assign").await; + let current = delivery_tracking( + &api, + &ticket.id, + assigned["assignment"]["assignment_id"].as_str().unwrap(), + ); + delivery_self_report(&api, &identity, &worker, ¤t, "waiting", "wait-answer").await; + execution.take_inputs(); + execution.take_input_request_ids(); + } + assert!(attention_pending(&api).is_empty()); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + std::thread::scope(|scope| { + let caller_api = api.clone(); + let caller_worker = worker.clone(); + let caller = scope.spawn(move || { + let mut checks = 0; + with_orchestrator_attention_reservation( + &caller_api, + &caller_worker, + true, + |state, pending| { + check_orchestrator_assignment_attention( + &caller_api, &caller_worker, state, pending, + ); + if checks == 0 { + // The production scan has read an empty snapshot but + // still owns its reservation. Race a real HTTP commit. + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); + } + checks += 1; + }, + ); + assert_eq!(checks, 2, "committed input must defer a scan"); + }); + entered_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); + if answer { + futures::executor::block_on(delivery_http( + &api, + "POST", + &format!("/api/w/{TEST_WORKSPACE_ID}/tickets/{}/thread-events", ticket.id), + Some(serde_json::to_value(NewTicketEvent::new( + TicketEventKind::Comment, "Human answer: go ahead", + )).unwrap()), + StatusCode::NO_CONTENT, + )); + } else { + futures::executor::block_on(delivery_assign(&api, &ticket.id, "racing-assign")); + } + // No manual dispatch, Restore, or synthetic Running -> Idle. + assert!(execution.take_inputs().is_empty()); + release_tx.send(()).unwrap(); + caller.join().unwrap(); + }); + let inputs = execution.take_inputs(); + assert_eq!(inputs.len(), 1, "idle Orchestrator was stranded"); + assert!(inputs[0].1.contains(ticket.resource_key.as_deref().unwrap())); + if answer { + assert!(inputs[0].1.contains("Need the human's choice")); + } + assert_eq!(execution.take_input_request_ids().len(), 1); + assert!(attention_pending(&api).is_empty()); + delivery_stop(&api, &worker); + } +} + +#[tokio::test] +async fn assignment_failed_delivery_and_its_reentrant_idle_do_not_schedule_retry_loop() { let dir = tempfile::tempdir().unwrap(); let (api, execution, worker, _) = delivery_recording_fixture(dir.path()).await; - let ticket = delivery_ticket(&api, "Fence old delivery"); - execution.reject_inputs("unknown acceptance response"); - let assigned = delivery_assign(&api, &ticket.id, "assign-old").await; - let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); - let old_attempts = execution.take_inputs(); - let old_ids = execution.take_input_request_ids(); - assert!(!old_attempts.is_empty()); - assert!(old_ids.iter().all(|id| id == &old_ids[0])); - delivery_http( - &api, - "DELETE", - &format!( - "{}?assignment_id={assignment}&operation_id=clear-old", - delivery_assignment_path(&ticket.id) - ), - None, - StatusCode::OK, - ) - .await; - *execution.input_failure.lock().unwrap() = None; - notify_ticket_recipients(&api, TEST_WORKSPACE_ID, &ticket.id, "open", "open", None); - delivery_scan(&api, &worker); - assert!( - execution.take_inputs().is_empty(), - "cleared reservation was sent as new work" - ); - assert_eq!( - delivery_assign(&api, &ticket.id, "assign-old").await, - assigned - ); - delivery_scan(&api, &worker); - assert!( - execution.take_inputs().is_empty(), - "old operation retry revived cleared work" - ); - let reassigned = delivery_assign(&api, &ticket.id, "assign-new").await; - let new_assignment = reassigned["assignment"]["assignment_id"].as_str().unwrap(); - assert_ne!(new_assignment, assignment); - assert_eq!(execution.take_inputs().len(), 1); + let ticket = delivery_ticket(&api, "No failed-delivery busy loop"); + execution.reject_inputs("Runtime currently cannot accept notifications"); + let callback_api = api.clone(); + execution.before_input_returns(move |worker| { + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end( + &callback_api, &worker.worker_id.to_string(), Some(S::Running), S::Idle, + ); + }); + delivery_assign(&api, &ticket.id, "assign-failed").await; + assert_eq!(execution.take_inputs().len(), 2, "idle must not add retries beyond the bounded send"); let ids = execution.take_input_request_ids(); - assert_eq!(ids.len(), 1); - assert_ne!( - ids[0], old_ids[0], - "new assignment reused stale input identity" - ); - let old = delivery_tracking(&api, &ticket.id, assignment); - assert!(!old.current); - assert_eq!( - old.tracking.unwrap().status, - ticket::TicketAssignmentTrackingStatus::Waiting, - "clear must not rewrite the old status to done" - ); - assert_eq!( - find_workspace_orchestrator(&api).unwrap().worker, - worker, - "clear/reassign are not Worker stop or replacement" - ); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + assert_eq!(attention_pending(&api).len(), 1, "failure must retain durable input"); delivery_stop(&api, &worker); } diff --git a/docs/development/ticket-assignment-delivery.md b/docs/development/ticket-assignment-delivery.md index 9945531e1..59fa87214 100644 --- a/docs/development/ticket-assignment-delivery.md +++ b/docs/development/ticket-assignment-delivery.md @@ -24,7 +24,7 @@ Delivery order: 4. Send the saved content with the saved `request_id`, outside the transaction. Response loss retains the same request/payload. Concurrent reservations converge on the same identity, and Runtime request receipts deduplicate actual acceptance. 5. Acknowledge accepted delivery under another IMMEDIATE transaction after rechecking current authority. The acknowledgement never changes handling. -Process-local attention locks prevent concurrent/reentrant send loops; they are not backlog authority. Losing that cache does not lose a delegation or resend acknowledged input. Replacement does not create assignment rows; only pending, unacknowledged inputs are recovered. Accepted history is not replayed into new work merely because a Worker was exchanged. +Process-local attention locks prevent concurrent/reentrant send loops; they are not backlog authority. A committed-input/startup/Restore trigger arriving during a reserved scan is coalesced into an input recheck and drained on reservation release, including when that scan read an empty snapshot. Discarding or acknowledging a pending retry continues the common scan in the same reservation; a successful retry cannot consume the trigger for newer input. Reentrant idle observations caused by a send do not schedule another retry of a failed send. These liveness repairs change no DB transaction or public API boundary. Losing that cache does not lose a delegation or resend acknowledged input. Replacement does not create assignment rows; only pending, unacknowledged inputs are recovered. Accepted history is not replayed into new work merely because a Worker was exchanged. ## Public start/action handoff for T-743