diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index 3acc5956a..87eaa3a01 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -1977,6 +1977,7 @@ pub trait ServerApi { )] async fn runtime_worker_create_alias( &self, + #[extension] context: ServerRequestContext, #[path] runtime_id: String, #[body] request: RuntimeWorkerSpawnRequest, ) -> Result; @@ -2006,6 +2007,7 @@ pub trait ServerApi { )] async fn runtime_worker_create( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] runtime_id: String, #[body] request: RuntimeWorkerSpawnRequest, @@ -7940,6 +7942,10 @@ pub struct TicketWorkerAttachmentBinding { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(deny_unknown_fields)] pub struct SetTicketRoleAssignmentRequest { + /// Observed delegation required for Orchestrator Ticket starts; manual starts omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub orchestrator_assignment_id: Option, + pub operation_id: String, pub principal: TicketAssignmentPrincipal, pub expected_assignment_id: Option, @@ -8580,6 +8586,10 @@ pub enum RuntimeWorkerSpawnAcceptanceRequirement { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RuntimeWorkerTicketAssignmentRequest { + /// Observed delegation required for Orchestrator Ticket starts; manual starts omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub orchestrator_assignment_id: Option, + pub ticket_id: String, pub operation_id: String, } @@ -9909,6 +9919,10 @@ pub struct BrowserWorkerWorkingDirectorySelection { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(deny_unknown_fields)] pub struct CreateWorkspaceWorkerTicketAssignmentRequest { + /// Observed delegation required for Orchestrator Ticket starts; manual starts omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub orchestrator_assignment_id: Option, + pub ticket_id: String, pub operation_id: String, } diff --git a/crates/worker/src/feature/builtin/manage_worker.rs b/crates/worker/src/feature/builtin/manage_worker.rs index 96dab0b7e..b320afec6 100644 --- a/crates/worker/src/feature/builtin/manage_worker.rs +++ b/crates/worker/src/feature/builtin/manage_worker.rs @@ -473,6 +473,9 @@ pub struct WorkerLifecycleWorkdirAttachment { #[derive(Debug, Clone)] pub struct WorkerLifecycleSpawnRequest { + /// Observed delegation required for Orchestrator Ticket starts. + pub orchestrator_assignment_id: Option, + pub runtime_id: String, /// Ordered alias-keyed selections forwarded as canonical Workdir attachments. pub workdir_attachments: Vec, @@ -494,6 +497,7 @@ fn workspace_worker_create_request( let ticket_assignment = match (request.ticket_id, request.operation_id) { (Some(ticket_id), Some(operation_id)) => { Some(CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: request.orchestrator_assignment_id, ticket_id, operation_id, }) @@ -729,6 +733,9 @@ struct WorkerSpawnInput { /// neither the profile nor an initial Flow segment grants review authority. #[serde(default)] ticket_id: Option, + /// Observed current Orchestrator delegation, required for a Ticket start by the Orchestrator. + #[serde(default)] + orchestrator_assignment_id: Option, #[serde(default)] display_name: Option, /// Normal typed initial user submission delivered after spawn. An empty @@ -913,6 +920,10 @@ impl Tool for WorkspaceWorkerTool { .transpose()?; self.control .spawn_worker(WorkerLifecycleSpawnRequest { + orchestrator_assignment_id: input + .orchestrator_assignment_id + .map(|id| authority_id(&id, "orchestrator_assignment_id")) + .transpose()?, runtime_id: authority_id(&input.runtime_id, "runtime_id")?, workdir_attachments: vec![WorkerLifecycleWorkdirAttachment { alias: INITIAL_WORKDIR_ALIAS.to_string(), @@ -1895,6 +1906,7 @@ mod tests { #[test] fn worker_lifecycle_spawn_request_preserves_canonical_attachment_collection() { let request = workspace_worker_create_request(WorkerLifecycleSpawnRequest { + orchestrator_assignment_id: None, runtime_id: "runtime-1".to_string(), workdir_attachments: vec![ WorkerLifecycleWorkdirAttachment { @@ -1954,6 +1966,7 @@ mod tests { #[test] fn worker_lifecycle_spawn_serializes_ticket_assignment_without_workdirs_or_flow() { let request = workspace_worker_create_request(WorkerLifecycleSpawnRequest { + orchestrator_assignment_id: Some("delegation-742".into()), runtime_id: "runtime-1".to_string(), workdir_attachments: Vec::new(), profile: "project:ticket-worker".to_string(), @@ -1971,6 +1984,7 @@ mod tests { value["ticket_assignment"], serde_json::json!({ "ticket_id": "T-719", + "orchestrator_assignment_id": "delegation-742", "operation_id": "spawn-ticket-worker:T-719:call-1", }) ); diff --git a/crates/worker/src/feature/builtin/orchestration.rs b/crates/worker/src/feature/builtin/orchestration.rs index abbdca4e2..495c382cf 100644 --- a/crates/worker/src/feature/builtin/orchestration.rs +++ b/crates/worker/src/feature/builtin/orchestration.rs @@ -21,7 +21,7 @@ use crate::feature::{ const FEATURE_ID: &str = "orchestration"; const TOOL_NAME: &str = "SpawnTicketWorker"; -const TOOL_DESCRIPTION: &str = "Spawn and atomically assign a generic Ticket Worker for the natural-language initial request. Profile is optional (defaults to builtin:ticket-worker); Flow and repository/Workdir attachments are optional. Results and return choices follow user intent, not a universal approval or MR gate. Backend validates claims and derives attachment access from Ticket targets; acceptance follows spawn, initial input, assignment, and resource finalization. Profile/Flow selection grants neither trusted review nor merge authority."; +const TOOL_DESCRIPTION: &str = "Spawn and atomically assign a generic Ticket Worker for the natural-language initial request. Profile is optional (defaults to builtin:ticket-worker); Flow and repository/Workdir attachments are optional. Results and return choices follow user intent, not a universal approval or MR gate. Orchestrator starts must supply the observed orchestrator_assignment_id; stale delegations cannot start new work. Assign is enough (no Queue/ready); unresolved dependencies are diagnostics for partial work and never start implicitly. Backend validates claims and derives attachment access from Ticket targets; acceptance follows spawn, initial input, assignment, and resource finalization. Profile/Flow selection grants neither trusted review nor merge authority."; #[derive(Debug, Default)] pub struct OrchestrationFeature; @@ -78,6 +78,9 @@ struct SpawnTicketWorkerWorkdirInput { #[serde(deny_unknown_fields)] struct SpawnTicketWorkerInput { ticket_id: String, + /// Observed current delegation identity. Required by Backend for an Orchestrator start; manual callers omit it. + #[serde(default)] + orchestrator_assignment_id: Option, runtime_id: String, /// Optional registered profile selector; omitted/null selects builtin:ticket-worker. /// Backend resolves the profile and validates claims, not natural-language satisfaction. @@ -132,6 +135,10 @@ impl Tool for SpawnTicketWorkerTool { let response = self .worker_service .spawn(WorkerLifecycleSpawnRequest { + orchestrator_assignment_id: input + .orchestrator_assignment_id + .map(|id| authority_id(id, "orchestrator_assignment_id")) + .transpose()?, runtime_id, workdir_attachments, profile, @@ -307,6 +314,7 @@ mod tests { let service = Arc::new(RecordingService::default()); let mut input = input(); input["flow"] = serde_json::json!("project:investigation"); + input["orchestrator_assignment_id"] = serde_json::json!("delegation-742"); input["workdir_attachments"] = serde_json::json!([ { "alias": "checkout", @@ -327,6 +335,10 @@ mod tests { assert_eq!(requests.len(), 1); let request = &requests[0]; assert_eq!(request.runtime_id, "runtime-1"); + assert_eq!( + request.orchestrator_assignment_id.as_deref(), + Some("delegation-742") + ); assert_eq!(request.profile, "project:ticket-worker"); assert_eq!(request.ticket_id.as_deref(), Some("00001KZXN51C7")); assert_eq!( @@ -475,6 +487,16 @@ mod tests { "initial_request must not be empty", ), ("flow", serde_json::json!(" "), "flow must not be empty"), + ( + "orchestrator_assignment_id", + serde_json::json!("http://invalid/id"), + "orchestrator_assignment_id must be an authority id", + ), + ( + "orchestrator_assignment_id", + serde_json::json!(" "), + "orchestrator_assignment_id must not be empty", + ), ( "runtime_id", serde_json::json!("runtime/1"), diff --git a/crates/worker/src/prompt/catalog.rs b/crates/worker/src/prompt/catalog.rs index cdb30cfae..4d2497661 100644 --- a/crates/worker/src/prompt/catalog.rs +++ b/crates/worker/src/prompt/catalog.rs @@ -156,11 +156,15 @@ impl WorkerPrompt { ]; } -/// Model-visible queued Ticket projection shared by Server and TUI backlog attention paths. +/// Model-visible Ticket projection shared by Server and TUI attention paths. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct OrchestratorQueueAttentionTicket { resource_key: String, title: String, + assignment_id: Option, + input_event_id: Option, + waiting_reason: Option, + next_action: Option, } impl OrchestratorQueueAttentionTicket { @@ -175,8 +179,28 @@ impl OrchestratorQueueAttentionTicket { Ok(Self { resource_key, title: bounded_queue_attention_text(&title.into(), 240), + assignment_id: None, + input_event_id: None, + waiting_reason: None, + next_action: None, }) } + + /// Committed delegation/input identity, not a handling-status transition. + /// The receiving Worker must reread current authority before a new action. + pub fn with_assignment_input( + mut self, + assignment_id: &str, + input_event_id: &str, + waiting_reason: Option<&str>, + next_action: Option<&str>, + ) -> Self { + self.assignment_id = Some(bounded_queue_attention_text(assignment_id, 1024)); + self.input_event_id = Some(bounded_queue_attention_text(input_event_id, 1024)); + self.waiting_reason = waiting_reason.map(|text| bounded_queue_attention_text(text, 1024)); + self.next_action = next_action.map(|text| bounded_queue_attention_text(text, 1024)); + self + } } /// Shared model-visible context for every Orchestrator backlog attention renderer. @@ -200,7 +224,7 @@ impl OrchestratorQueueAttentionContext { } } -/// Prompt-catalog entries that must share the same backlog-attention body contract. +/// Prompt-catalog entries sharing a bounded Ticket resource-key projection. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OrchestratorQueueAttentionPrompt { Server, @@ -841,8 +865,10 @@ mod tests { .orchestrator_queue_attention(OrchestratorQueueAttentionPrompt::Tui, &context) .unwrap(); - assert_eq!(server, tui); - assert!(server.starts_with("Queued Tickets require attention:")); + assert!(server.starts_with("Assigned Tickets have unreceived input:")); + // TUI's final operation model is a separate downstream scope; both + // renderers still consume the same bounded resource-key projection. + assert!(tui.starts_with("Queued Tickets require attention:")); assert!(server.contains("- T-1 — Ticket 1 with control ")); assert!(!server.contains("T-21")); assert!(server.contains("were omitted from this notice: 1")); @@ -859,6 +885,28 @@ mod tests { } } + #[test] + fn assignment_attention_carries_input_identity_and_bounded_waiting_context() { + let ticket = OrchestratorQueueAttentionTicket::new("T-742", "Partial work") + .unwrap() + .with_assignment_input( + "assignment-a", + "answer-event", + Some("human\nanswer"), + Some(&"x".repeat(2048)), + ); + let context = OrchestratorQueueAttentionContext::new(vec![ticket]); + let output = PromptCatalog::builtins_only() + .unwrap() + .orchestrator_queue_attention(OrchestratorQueueAttentionPrompt::Server, &context) + .unwrap(); + assert!(output.contains("Delegation: assignment-a; input: answer-event.")); + assert!(output.contains("Waiting reason: human answer")); + assert!(output.contains(&format!("Next action: {}…", "x".repeat(1024)))); + assert!(!output.contains(&"x".repeat(1025))); + assert!(output.contains("no ready/Queue step is required")); + } + #[test] fn queue_attention_prompt_omits_truncation_text_for_complete_list() { let catalog = PromptCatalog::builtins_only().unwrap(); diff --git a/crates/workspace-server/src/assignment_delivery.rs b/crates/workspace-server/src/assignment_delivery.rs new file mode 100644 index 000000000..db59d0f51 --- /dev/null +++ b/crates/workspace-server/src/assignment_delivery.rs @@ -0,0 +1,380 @@ +//! Durable operational delivery and startup fencing, independent of handling. +use super::*; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TicketAssignmentDelivery { + pub ticket_id: String, + pub assignment_id: String, + pub input_event_id: String, + pub delivery_id: String, + pub request_id: String, + pub waiting_reason: Option, + pub next_action: Option, +} + +pub(super) const SCHEMA_SQL: &str = r#" +CREATE TABLE ticket_assignment_delivery_origins ( + workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, assignment_id TEXT NOT NULL, + input_event_id TEXT NOT NULL, after_ticket_event_index INTEGER NOT NULL, + PRIMARY KEY(workspace_id,assignment_id), + FOREIGN KEY(workspace_id,ticket_id,assignment_id) REFERENCES ticket_worker_assignments(workspace_id,ticket_id,assignment_id) +); +CREATE TABLE ticket_assignment_deliveries ( + workspace_id TEXT NOT NULL, assignment_id TEXT NOT NULL, input_event_id TEXT NOT NULL, + delivery_id TEXT NOT NULL, payload_json TEXT NOT NULL CHECK(json_valid(payload_json)), + reserved_source_json TEXT NOT NULL CHECK(json_valid(reserved_source_json)), + PRIMARY KEY(workspace_id,assignment_id,input_event_id), UNIQUE(workspace_id,delivery_id), + FOREIGN KEY(workspace_id,assignment_id) REFERENCES ticket_worker_assignments(workspace_id,assignment_id) +); +CREATE TABLE ticket_assignment_delivery_requests ( + workspace_id TEXT NOT NULL, delivery_id TEXT NOT NULL, content TEXT NOT NULL, + PRIMARY KEY(workspace_id,delivery_id), + FOREIGN KEY(workspace_id,delivery_id) REFERENCES ticket_assignment_deliveries(workspace_id,delivery_id) +); +CREATE TRIGGER ticket_assignment_delivery_request_no_update BEFORE UPDATE ON ticket_assignment_delivery_requests +BEGIN SELECT RAISE(ABORT,'Assignment delivery request is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_request_no_delete BEFORE DELETE ON ticket_assignment_delivery_requests +BEGIN SELECT RAISE(ABORT,'Assignment delivery request is immutable'); END; +CREATE TABLE ticket_assignment_delivery_acknowledgements ( + workspace_id TEXT NOT NULL, delivery_id TEXT NOT NULL, + source_json TEXT NOT NULL CHECK(json_valid(source_json)), PRIMARY KEY(workspace_id,delivery_id), + FOREIGN KEY(workspace_id,delivery_id) REFERENCES ticket_assignment_deliveries(workspace_id,delivery_id) +); +CREATE TABLE ticket_assignment_operation_fences ( + workspace_id TEXT NOT NULL, operation_id TEXT NOT NULL, orchestrator_assignment_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, agent_key TEXT NOT NULL, + authentication_binding_id TEXT NOT NULL, source_json TEXT NOT NULL CHECK(json_valid(source_json)), + PRIMARY KEY(workspace_id,operation_id), + FOREIGN KEY(workspace_id,operation_id) REFERENCES ticket_assignment_operations(workspace_id,operation_id), + FOREIGN KEY(workspace_id,orchestrator_assignment_id) REFERENCES ticket_worker_assignments(workspace_id,assignment_id) +); +CREATE TRIGGER ticket_assignment_delivery_origin_insert +AFTER INSERT ON typed_ticket_assignment_tracking_history +WHEN json_extract(NEW.source_context_json,'$.kind')='assignment' +BEGIN + INSERT INTO ticket_assignment_delivery_origins + VALUES(NEW.workspace_id,NEW.ticket_id,NEW.assignment_id,'assignment-event:' || NEW.event_id, + COALESCE((SELECT MAX(event_index) FROM typed_ticket_events + WHERE workspace_id=NEW.workspace_id AND ticket_id=NEW.ticket_id),-1)); +END; +CREATE TRIGGER ticket_assignment_delivery_payload_no_update BEFORE UPDATE ON ticket_assignment_deliveries +BEGIN SELECT RAISE(ABORT,'Assignment delivery payload is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_payload_no_delete BEFORE DELETE ON ticket_assignment_deliveries +BEGIN SELECT RAISE(ABORT,'Assignment delivery payload is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_ack_no_update BEFORE UPDATE ON ticket_assignment_delivery_acknowledgements +BEGIN SELECT RAISE(ABORT,'Assignment delivery acknowledgement is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_ack_no_delete BEFORE DELETE ON ticket_assignment_delivery_acknowledgements +BEGIN SELECT RAISE(ABORT,'Assignment delivery acknowledgement is immutable'); END; +CREATE TRIGGER ticket_assignment_operation_fence_no_update BEFORE UPDATE ON ticket_assignment_operation_fences +BEGIN SELECT RAISE(ABORT,'Assignment startup fence is immutable'); END; +CREATE TRIGGER ticket_assignment_operation_fence_no_delete BEFORE DELETE ON ticket_assignment_operation_fences +BEGIN SELECT RAISE(ABORT,'Assignment startup fence is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_origin_no_update BEFORE UPDATE ON ticket_assignment_delivery_origins +BEGIN SELECT RAISE(ABORT,'Assignment input origin is immutable'); END; +CREATE TRIGGER ticket_assignment_delivery_origin_no_delete BEFORE DELETE ON ticket_assignment_delivery_origins +BEGIN SELECT RAISE(ABORT,'Assignment input origin is immutable'); END; +"#; + +/// Additive Workspace schema only. Native pre-upgrade assignments use their +/// committed initial audit time to exclude earlier Ticket inputs; ambiguous +/// legacy relationships are NOT treated as fresh delegations. Migration inputs +/// retain the exact Queue fence already reconciled by Ticket schema 11. +pub(super) fn install(conn: &Connection) -> Result<()> { + conn.execute_batch(SCHEMA_SQL)?; + conn.execute_batch(r#" +INSERT INTO ticket_assignment_delivery_origins +SELECT h.workspace_id,h.ticket_id,h.assignment_id,'assignment-event:' || h.event_id, + COALESCE((SELECT MAX(e.event_index) FROM typed_ticket_events e + WHERE e.workspace_id=h.workspace_id AND e.ticket_id=h.ticket_id AND e.ata.after_ticket_event_index + -- External content/answer inputs, not case/routing/history or own self-report. + -- Ticket supports typed extension events; an answer need not be named comment. + AND e.kind NOT IN ('create','close','state_changed','status_changed','history') + AND e.kind NOT LIKE 'assignment_tracking%' + AND e.kind NOT LIKE 'assignment_delivery%' + AND COALESCE(e.state_field,'') NOT IN ('case','routing','assignment_tracking') +) +SELECT i.ticket_id,i.assignment_id,i.input_event_id, + json_extract(i.tracking_json,'$.waiting_reason'),json_extract(i.tracking_json,'$.next_action'),d.payload_json +FROM inputs i +LEFT JOIN ticket_assignment_deliveries d ON d.workspace_id=?1 AND d.assignment_id=i.assignment_id AND d.input_event_id=i.input_event_id +WHERE NOT EXISTS(SELECT 1 FROM ticket_assignment_delivery_acknowledgements ack + WHERE ack.workspace_id=?1 AND ack.delivery_id=d.delivery_id) +AND (?3 IS NULL OR i.assignment_id=?3) AND (?4 IS NULL OR i.input_event_id=?4) +"#; + +fn decode_payload(payload: &str) -> Result { + serde_json::from_str(payload).map_err(|e| Error::Store(e.to_string())) +} + +pub(super) fn list_in_tx( + conn: &Connection, + workspace_id: &str, + limit: usize, +) -> Result> { + inputs_in_tx(conn, workspace_id, limit, None, None) +} +fn inputs_in_tx( + conn: &Connection, + workspace_id: &str, + limit: usize, + assignment_id: Option<&str>, + input_event_id: Option<&str>, +) -> Result> { + let sql = format!("{INPUTS_SQL} ORDER BY i.ticket_id,i.assignment_id,i.input_order LIMIT ?2"); + let mut statement = conn.prepare(&sql)?; + let rows = statement.query_map( + params![workspace_id, limit as i64, assignment_id, input_event_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + )) + }, + )?; + rows.map(|row| { + let (ticket_id, assignment_id, input_event_id, waiting_reason, next_action, payload) = row?; + if let Some(payload) = payload { + return decode_payload(&payload); + } + // Hash the identities, not mutable handling. Scope is Workspace-qualified. + let identities = serde_json::to_string(&(workspace_id, &assignment_id, &input_event_id)) + .map_err(|e| Error::Store(e.to_string()))?; + let digest = Sha256::digest(identities.as_bytes()); + let hex = digest + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let delivery_id = format!("assignment-delivery:{hex}"); + Ok(TicketAssignmentDelivery { + ticket_id, + assignment_id, + input_event_id, + request_id: delivery_id.clone(), + delivery_id, + waiting_reason, + next_action, + }) + }) + .collect() +} + +fn source_guard( + conn: &Connection, + workspace_id: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result<()> { + assignment_tracking::require_runtime_source( + conn, + workspace_id, + &source.runtime_id, + &source.worker_id, + &source.agent_key, + &source.authentication_binding_id, + ) +} + +pub(super) fn require_current( + conn: &Connection, + workspace_id: &str, + ticket_id: &str, + assignment_id: &str, +) -> Result<()> { + let current:bool=conn.query_row(r#"SELECT EXISTS(SELECT 1 FROM ticket_current_worker_assignments c + JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.assignment_id=c.assignment_id + JOIN typed_tickets t ON t.workspace_id=a.workspace_id AND t.ticket_id=a.ticket_id + WHERE c.workspace_id=?1 AND c.ticket_id=?2 AND c.assignment_id=?3 AND c.role='orchestrator' + AND a.principal_kind='workspace_agent' AND a.principal_id='workspace-orchestrator' + AND t.workflow_state='open' AND json_extract(a.tracking_json,'$.status') IN ('waiting','inprogress') + AND EXISTS(SELECT 1 FROM ticket_assignment_delivery_origins o WHERE o.workspace_id=a.workspace_id AND o.assignment_id=a.assignment_id) + AND NOT EXISTS(SELECT 1 FROM ticket_assignment_work_releases r WHERE r.workspace_id=a.workspace_id AND r.assignment_id=a.assignment_id))"#,params![workspace_id,ticket_id,assignment_id],|r|r.get(0))?; + if !current { + return Err(Error::TicketAssignmentConflict( + "Assignment is no longer a current actionable delegation".into(), + )); + } + Ok(()) +} + +pub(super) fn reserve( + store: &SqliteWorkspaceStore, + workspace_id: &str, + assignment_id: &str, + input_event_id: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result { + store.with_conn_mut(|conn| { + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + source_guard(&tx,workspace_id,source)?; + let ticket_id:String=tx.query_row("SELECT ticket_id FROM ticket_worker_assignments WHERE workspace_id=?1 AND assignment_id=?2",params![workspace_id,assignment_id],|r|r.get(0))?; + require_current(&tx,workspace_id,&ticket_id,assignment_id)?; + let saved:Option=tx.query_row("SELECT payload_json FROM ticket_assignment_deliveries WHERE workspace_id=?1 AND assignment_id=?2 AND input_event_id=?3",params![workspace_id,assignment_id,input_event_id],|r|r.get(0)).optional()?; + if let Some(saved)=saved {let result=decode_payload(&saved)?;tx.commit()?;return Ok(result);} + // A caller cannot fabricate an input. Restrict the same common query by + // exact assignment/input identity rather than trusting a scan snapshot. + let candidates=inputs_in_tx(&tx,workspace_id,1,Some(assignment_id),Some(input_event_id))?; + let delivery=candidates.into_iter().find(|d|d.assignment_id==assignment_id && d.input_event_id==input_event_id) + .ok_or_else(||Error::TicketAssignmentConflict("Assignment input is not pending".into()))?; + tx.execute("INSERT INTO ticket_assignment_deliveries VALUES(?1,?2,?3,?4,?5,?6)",params![workspace_id,assignment_id,input_event_id,delivery.delivery_id,serde_json::to_string(&delivery).map_err(|e|Error::Store(e.to_string()))?,serde_json::to_string(source).map_err(|e|Error::Store(e.to_string()))?])?; + tx.commit()?; + Ok(delivery) + }) +} + +/// Persist the exact rendered Runtime request before I/O. A losing renderer or +/// retry receives the original bytes, not a new payload under the same ID. +pub(super) fn prepare_content( + store: &SqliteWorkspaceStore, + workspace_id: &str, + delivery_id: &str, + content: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result { + if content.is_empty() || content.len() > 262144 || content.contains('\0') { + return Err(Error::InvalidInput( + "Assignment delivery content must be nonempty, NUL-free and at most 262144 bytes" + .into(), + )); + } + store.with_conn_mut(|conn| { + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + source_guard(&tx,workspace_id,source)?; + let payload:String=tx.query_row("SELECT payload_json FROM ticket_assignment_deliveries WHERE workspace_id=?1 AND delivery_id=?2",params![workspace_id,delivery_id],|r|r.get(0))?; + let delivery=decode_payload(&payload)?; + require_current(&tx,workspace_id,&delivery.ticket_id,&delivery.assignment_id)?; + let saved:Option=tx.query_row("SELECT content FROM ticket_assignment_delivery_requests WHERE workspace_id=?1 AND delivery_id=?2",params![workspace_id,delivery_id],|r|r.get(0)).optional()?; + let result=if let Some(saved)=saved {saved} else { + tx.execute("INSERT INTO ticket_assignment_delivery_requests VALUES(?1,?2,?3)",params![workspace_id,delivery_id,content])?; + content.to_owned() + }; + tx.commit()?; + Ok(result) + }) +} + +pub(super) fn ack( + store: &SqliteWorkspaceStore, + workspace_id: &str, + delivery_id: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result<()> { + store.with_conn_mut(|conn| { + let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + source_guard(&tx,workspace_id,source)?; + let payload:String=tx.query_row("SELECT payload_json FROM ticket_assignment_deliveries WHERE workspace_id=?1 AND delivery_id=?2",params![workspace_id,delivery_id],|r|r.get(0))?; + let delivery=decode_payload(&payload)?; + require_current(&tx,workspace_id,&delivery.ticket_id,&delivery.assignment_id)?; + let source_json=serde_json::to_string(source).map_err(|e|Error::Store(e.to_string()))?; + let saved:Option=tx.query_row("SELECT source_json FROM ticket_assignment_delivery_acknowledgements WHERE workspace_id=?1 AND delivery_id=?2",params![workspace_id,delivery_id],|r|r.get(0)).optional()?; + if let Some(saved)=saved { + if saved!=source_json {return Err(Error::TicketAssignmentConflict("Delivery acknowledgement source differs from saved receipt".into()));} + } else {tx.execute("INSERT INTO ticket_assignment_delivery_acknowledgements VALUES(?1,?2,?3)",params![workspace_id,delivery_id,source_json])?;} + tx.commit()?; + Ok(()) + }) +} + +/// Must run before receipt replay as well as every bind/finalize write. Absence +/// is the existing manual reservation contract, not inferred delegation. +pub(super) fn operation_guard( + conn: &Connection, + workspace_id: &str, + operation_id: &str, +) -> Result<()> { + let fence=conn.query_row("SELECT o.ticket_id,f.orchestrator_assignment_id,f.runtime_id,f.worker_id,f.agent_key,f.authentication_binding_id FROM ticket_assignment_operation_fences f JOIN ticket_assignment_operations o ON o.workspace_id=f.workspace_id AND o.operation_id=f.operation_id WHERE f.workspace_id=?1 AND f.operation_id=?2",params![workspace_id,operation_id],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?,r.get::<_,String>(2)?,r.get::<_,String>(3)?,r.get::<_,String>(4)?,r.get::<_,String>(5)?))).optional()?; + if let Some((ticket_id, assignment_id, runtime_id, worker_id, agent_key, binding_id)) = fence { + assignment_tracking::require_runtime_source( + conn, + workspace_id, + &runtime_id, + &worker_id, + &agent_key, + &binding_id, + )?; + require_current(conn, workspace_id, &ticket_id, &assignment_id)?; + } + Ok(()) +} + +pub(super) fn save_operation_fence( + conn: &Connection, + workspace_id: &str, + operation_id: &str, + ticket_id: &str, + assignment_id: &str, + source: &TicketAssignmentTrackingSourceContext, + existing: bool, +) -> Result<()> { + source_guard(conn, workspace_id, source)?; + require_current(conn, workspace_id, ticket_id, assignment_id)?; + let source_json = serde_json::to_string(source).map_err(|e| Error::Store(e.to_string()))?; + let saved:Option<(String,String)>=conn.query_row("SELECT orchestrator_assignment_id,source_json FROM ticket_assignment_operation_fences WHERE workspace_id=?1 AND operation_id=?2",params![workspace_id,operation_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()?; + if let Some(saved) = saved { + if saved != (assignment_id.to_owned(), source_json) { + return Err(Error::TicketAssignmentConflict( + "Startup operation assignment/source differs from saved fence".into(), + )); + } + } else { + if existing { + return Err(Error::TicketAssignmentConflict( + "Cannot add a delegation fence to an existing manual operation".into(), + )); + } + conn.execute( + "INSERT INTO ticket_assignment_operation_fences VALUES(?1,?2,?3,?4,?5,?6,?7,?8)", + params![ + workspace_id, + operation_id, + assignment_id, + source.runtime_id, + source.worker_id, + source.agent_key, + source.authentication_binding_id, + source_json + ], + )?; + } + Ok(()) +} diff --git a/crates/workspace-server/src/assignment_delivery_tests.rs b/crates/workspace-server/src/assignment_delivery_tests.rs new file mode 100644 index 000000000..7a93d3c12 --- /dev/null +++ b/crates/workspace-server/src/assignment_delivery_tests.rs @@ -0,0 +1,794 @@ +// Included in store::tests: exercise persisted transaction boundaries, not Runtime I/O. +fn delivery_store_external_event( + store: &SqliteWorkspaceStore, + index: i64, + kind: &str, + state_field: Option<&str>, +) { + store.with_conn(|conn| { + conn.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,author,at,state_field,body) VALUES('w','tracking',?1,?2,'user','2026-10-12T00:00:00Z',?3,'Answer arrived')",params![index,kind,state_field])?; + Ok(()) + }).unwrap(); +} + +fn delivery_store_reserve_initial( + store: &SqliteWorkspaceStore, + source: &TicketAssignmentTrackingSourceContext, +) -> TicketAssignmentDelivery { + let inputs = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!(inputs.len(), 1); + store + .reserve_ticket_assignment_delivery( + "w", + &inputs[0].assignment_id, + &inputs[0].input_event_id, + source, + ) + .unwrap() +} + +#[tokio::test] +async fn assignment_delivery_waiting_self_reports_do_not_busy_loop_and_answers_are_pending() { + let (_directory, store) = tracking_store_fixture().await; + // Existing history before Assign must not become a new consideration input. + delivery_store_external_event(&store, 0, "comment", None); + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let initial = delivery_store_reserve_initial(&store, &source); + store + .ack_ticket_assignment_delivery("w", &initial.delivery_id, &source) + .unwrap(); + let mut update = tracking_store_input("a", "event-assign-a", "waiting-a"); + update.tracking.status = TicketAssignmentTrackingStatus::Waiting; + update.tracking.waiting_reason = Some("Waiting for owner answer".into()); + update.tracking.next_action = Some("Read answer and decide".into()); + store + .update_ticket_assignment_tracking("w", "tracking", &update, &source) + .unwrap(); + assert!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .is_empty() + ); + delivery_store_external_event( + &store, + 1, + "assignment_tracking", + Some("assignment_tracking"), + ); + delivery_store_external_event(&store, 2, "comment", None); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].input_event_id, "ticket-event:2"); + assert_eq!( + pending[0].waiting_reason.as_deref(), + Some("Waiting for owner answer") + ); + assert_eq!( + pending[0].next_action.as_deref(), + Some("Read answer and decide") + ); + let answer = store + .reserve_ticket_assignment_delivery("w", "a", "ticket-event:2", &source) + .unwrap(); + store + .ack_ticket_assignment_delivery("w", &answer.delivery_id, &source) + .unwrap(); + assert!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .is_empty() + ); + // Typed extension inputs must not disappear merely because they are not + // comments; own tracking/history events remain excluded. + delivery_store_external_event(&store, 3, "assignment_tracking_updated", None); + delivery_store_external_event(&store, 4, "history", None); + delivery_store_external_event(&store, 5, "owner_answer", None); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].input_event_id, "ticket-event:5"); + let saved = store + .get_ticket_assignment_tracking("w", "tracking", "a") + .unwrap() + .unwrap(); + assert_eq!( + saved.tracking.unwrap().status, + TicketAssignmentTrackingStatus::Waiting + ); +} + +#[tokio::test] +async fn assignment_delivery_response_loss_returns_immutable_payload_after_progress_changes_and_reopen() + { + let (directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let initial = delivery_store_reserve_initial(&store, &source); + let mut update = tracking_store_input("a", "event-assign-a", "waiting-a"); + update.tracking.status = TicketAssignmentTrackingStatus::Waiting; + update.tracking.waiting_reason = Some("New reason not part of original delivery".into()); + store + .update_ticket_assignment_tracking("w", "tracking", &update, &source) + .unwrap(); + drop(store); + let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + assert_eq!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(), + vec![initial.clone()] + ); + assert_eq!( + store + .reserve_ticket_assignment_delivery("w", "a", &initial.input_event_id, &source) + .unwrap(), + initial + ); + store + .ack_ticket_assignment_delivery("w", &initial.delivery_id, &source) + .unwrap(); + store + .ack_ticket_assignment_delivery("w", &initial.delivery_id, &source) + .unwrap(); + assert!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .is_empty() + ); + store + .with_conn(|conn| { + assert!( + conn.execute( + "UPDATE ticket_assignment_deliveries SET payload_json='{}'", + [] + ) + .is_err() + ); + assert!( + conn.execute( + "DELETE FROM ticket_assignment_delivery_acknowledgements", + [] + ) + .is_err() + ); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_delivery_simultaneous_reservations_converge_to_one_receipt() { + let (directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .remove(0); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let threads = (0..2) + .map(|_| { + let barrier = barrier.clone(); + let path = directory.path().join("workspace.db"); + let input = pending.input_event_id.clone(); + std::thread::spawn(move || { + let store = SqliteWorkspaceStore::open(path).unwrap(); + barrier.wait(); + store + .reserve_ticket_assignment_delivery("w", "a", &input, &tracking_store_source()) + .unwrap() + }) + }) + .collect::>(); + let receipts = threads + .into_iter() + .map(|t| t.join().unwrap()) + .collect::>(); + assert_eq!(receipts[0], receipts[1]); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM ticket_assignment_deliveries", + [], + |r| r.get(0), + )?; + assert_eq!(count, 1); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_delivery_done_close_reopen_and_unknown_history_do_not_revive_delegation() { + for change in ["done", "close-reopen", "unknown"] { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let initial = delivery_store_reserve_initial(&store, &source); + match change { + "done" => { + store + .update_ticket_assignment_tracking( + "w", + "tracking", + &tracking_store_input("a", "event-assign-a", "done-a"), + &source, + ) + .unwrap(); + } + "close-reopen" => { + store.with_conn(|conn| {conn.execute("UPDATE typed_tickets SET workflow_state='closed' WHERE ticket_id='tracking'",[])?;conn.execute("UPDATE typed_tickets SET workflow_state='open' WHERE ticket_id='tracking'",[])?;Ok(())}).unwrap(); + } + _ => { + store.with_conn(|conn| {conn.execute("UPDATE ticket_worker_assignments SET tracking_json=NULL,last_tracking_event_id=NULL WHERE assignment_id='a'",[])?;Ok(())}).unwrap(); + } + } + delivery_store_external_event(&store, 1, "comment", None); + assert!( + store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap() + .is_empty(), + "{change}" + ); + assert!( + store + .reserve_ticket_assignment_delivery("w", "a", &initial.input_event_id, &source) + .is_err(), + "{change}" + ); + } +} + +#[tokio::test] +async fn assignment_delivery_clear_reassign_fences_scanned_and_reserved_inputs() { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let old = delivery_store_reserve_initial(&store, &source); + store + .clear_current_ticket_role_assignment( + "w", + "tracking", + TicketAssignmentRole::Orchestrator, + "a", + "clear-a", + "clear-a", + "owner", + "2", + None, + ) + .unwrap(); + let new = tracking_store_assign(&store, "b", "assign-b"); + assert_eq!(new.assignment_id, "b"); + assert!( + store + .reserve_ticket_assignment_delivery("w", "a", &old.input_event_id, &source) + .is_err() + ); + assert!( + store + .ack_ticket_assignment_delivery("w", &old.delivery_id, &source) + .is_err() + ); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].assignment_id, "b"); + assert_ne!(pending[0].request_id, old.request_id); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM ticket_worker_assignments WHERE role='orchestrator'", + [], + |r| r.get(0), + )?; + assert_eq!(count, 2); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_delivery_old_worker_or_authenticated_binding_cannot_reserve_or_ack() { + for change in ["replacement", "revoke", "binding-change"] { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let old = delivery_store_reserve_initial(&store, &source); + store.with_conn(|conn| { + match change { + "replacement" => {conn.execute("UPDATE worker_singleton_owners SET worker_id='replacement' WHERE workspace_id='w'",[])?;}, + "revoke" => {conn.execute("UPDATE workspace_runtime_bindings SET revoked_at='2' WHERE workspace_id='w'",[])?;}, + _ => {conn.execute("UPDATE workspace_runtime_bindings SET binding_id='new-binding' WHERE workspace_id='w'",[])?;}, + } Ok(()) + }).unwrap(); + assert!( + matches!( + store.reserve_ticket_assignment_delivery("w", "a", &old.input_event_id, &source), + Err(Error::WorkspacePermissionDenied(_)) + ), + "{change}" + ); + assert!( + matches!( + store.ack_ticket_assignment_delivery("w", &old.delivery_id, &source), + Err(Error::WorkspacePermissionDenied(_)) + ), + "{change}" + ); + if change == "replacement" { + let mut replacement = source.clone(); + replacement.worker_id = "replacement".into(); + assert_eq!( + store + .reserve_ticket_assignment_delivery("w", "a", &old.input_event_id, &replacement) + .unwrap(), + old + ); + store + .ack_ticket_assignment_delivery("w", &old.delivery_id, &replacement) + .unwrap(); + assert_eq!( + store + .list_current_ticket_assignment_tracking("w", "tracking") + .unwrap() + .len(), + 1 + ); + } + } +} + +fn delivery_store_start_reserve( + store: &SqliteWorkspaceStore, + key: &str, + assignment: &str, + source: &TicketAssignmentTrackingSourceContext, +) -> Result { + store.reserve_ticket_assignment_operation_for_orchestrator( + "w", + key, + "tracking", + "r", + Some("replacement"), + "start-input", + "3", + assignment, + source, + ) +} + +#[tokio::test] +async fn assignment_startup_clear_reassign_fences_reservation_replay_bind_and_finalize() { + for change in ["clear", "reassign", "close-reopen"] { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + assert!(delivery_store_start_reserve(&store, "start-a", "a", &source).unwrap()); + assert!(!delivery_store_start_reserve(&store, "start-a", "a", &source).unwrap()); + if change == "close-reopen" { + store.with_conn(|conn| {conn.execute("UPDATE typed_tickets SET workflow_state='closed' WHERE ticket_id='tracking'",[])?;conn.execute("UPDATE typed_tickets SET workflow_state='open' WHERE ticket_id='tracking'",[])?;Ok(())}).unwrap(); + } else { + // Clear/reassign must not be blocked by a pending startup claim: + // assignment authority can invalidate it without Runtime side effects. + store + .clear_current_ticket_role_assignment( + "w", + "tracking", + TicketAssignmentRole::Orchestrator, + "a", + "clear-a", + "clear-a", + "owner", + "4", + None, + ) + .unwrap(); + if change == "reassign" { + tracking_store_assign(&store, "b", "assign-b"); + } + } + assert!( + delivery_store_start_reserve(&store, "start-a", "a", &source).is_err(), + "{change}" + ); + assert!( + delivery_store_start_reserve(&store, "start-new", "a", &source).is_err(), + "{change}" + ); + assert!( + store + .validate_ticket_assignment_operation_fence("w", "start-a") + .is_err(), + "{change}" + ); + assert!( + store + .bind_ticket_assignment_operation_worker("w", "start-a", "replacement") + .is_err(), + "{change}" + ); + let record = TicketWorkerAssignmentRecord { + workspace_id: "w".into(), + ticket_id: "tracking".into(), + assignment_id: "target-a".into(), + worker: RuntimeWorkerRef::new("r", "replacement"), + assigned_by: "owner".into(), + assigned_at: "4".into(), + }; + assert!( + store + .set_current_ticket_worker_assignment( + &record, + None, + "worker-event", + "start-a", + false + ) + .is_err(), + "{change}" + ); + assert!( + store + .get_active_ticket_role_assignment("w", "tracking", TicketAssignmentRole::Worker) + .unwrap() + .is_none() + ); + } +} + +#[tokio::test] +async fn assignment_startup_replaced_worker_cannot_start_or_replay_saved_operations() { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + delivery_store_start_reserve(&store, "start-a", "a", &source).unwrap(); + store + .with_conn(|conn| { + conn.execute( + "UPDATE worker_singleton_owners SET worker_id='replacement' WHERE workspace_id='w'", + [], + )?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + delivery_store_start_reserve(&store, "start-a", "a", &source), + Err(Error::WorkspacePermissionDenied(_)) + )); + assert!(matches!( + delivery_store_start_reserve(&store, "start-new", "a", &source), + Err(Error::WorkspacePermissionDenied(_)) + )); + assert!(matches!( + store.validate_ticket_assignment_operation_fence("w", "start-a"), + Err(Error::WorkspacePermissionDenied(_)) + )); + assert!(matches!( + store.bind_ticket_assignment_operation_worker("w", "start-a", "replacement"), + Err(Error::WorkspacePermissionDenied(_)) + )); + // Even the new Worker cannot adopt the old sender's startup operation key. + let mut replacement = source.clone(); + replacement.worker_id = "replacement".into(); + assert!(delivery_store_start_reserve(&store, "start-a", "a", &replacement).is_err()); +} + +#[tokio::test] +async fn assignment_startup_saved_fence_cannot_be_bypassed_or_attached_to_manual_receipt() { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + delivery_store_start_reserve(&store, "start-a", "a", &source).unwrap(); + assert!( + store + .reserve_ticket_assignment_operation( + "w", + "start-a", + "tracking", + "r", + Some("replacement"), + "start-input", + "3" + ) + .is_err() + ); + store.with_conn(|conn| {assert!(conn.execute("UPDATE ticket_assignment_operation_fences SET orchestrator_assignment_id='different'",[]).is_err());Ok(())}).unwrap(); + store + .fail_ticket_assignment_operation("w", "start-a", "explicit cancellation") + .unwrap(); + store + .reserve_ticket_assignment_operation( + "w", + "manual", + "tracking", + "r", + Some("replacement"), + "start-input", + "3", + ) + .unwrap(); + assert!(delivery_store_start_reserve(&store, "manual", "a", &source).is_err()); +} + +#[tokio::test] +async fn assignment_delivery_rendered_content_is_immutable_across_restart_and_concurrent_renderers() +{ + let (directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + let initial = delivery_store_reserve_initial(&store, &source); + assert_eq!( + store + .prepare_ticket_assignment_delivery_content( + "w", + &initial.delivery_id, + "original rendered prompt", + &source + ) + .unwrap(), + "original rendered prompt" + ); + drop(store); + let store = SqliteWorkspaceStore::open(directory.path().join("workspace.db")).unwrap(); + assert_eq!( + store + .prepare_ticket_assignment_delivery_content( + "w", + &initial.delivery_id, + "different title/handling/config prompt", + &source + ) + .unwrap(), + "original rendered prompt" + ); + store + .with_conn(|conn| { + assert!( + conn.execute( + "UPDATE ticket_assignment_delivery_requests SET content='changed'", + [] + ) + .is_err() + ); + assert!( + conn.execute("DELETE FROM ticket_assignment_delivery_requests", []) + .is_err() + ); + Ok(()) + }) + .unwrap(); + store + .clear_current_ticket_role_assignment( + "w", + "tracking", + TicketAssignmentRole::Orchestrator, + "a", + "clear-a", + "clear-a", + "owner", + "4", + None, + ) + .unwrap(); + assert!( + store + .prepare_ticket_assignment_delivery_content( + "w", + &initial.delivery_id, + "original rendered prompt", + &source + ) + .is_err() + ); +} + +#[test] +fn assignment_delivery_schema92_upgrade_uses_reconciled_migration_evidence_not_legacy_history() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("schema92.db"); + // A genuine retained Ticket-10/Workspace-91 fixture goes through its actual + // applied tracking migration; no current DB markers or columns are lowered. + let conn = tracking_schema91_fixture(&path); + migrate_ticket_tracking_v91_to_v92(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 92); + let assignments = stored_fixture_rows(&conn, "ticket_worker_assignments"); + let operations = stored_fixture_rows(&conn, "ticket_assignment_operations"); + let tracking = stored_fixture_rows(&conn, "typed_ticket_assignment_tracking_history"); + drop(conn); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + let pending = store + .list_pending_ticket_assignment_deliveries("w", 100) + .unwrap(); + assert_eq!( + pending + .iter() + .map(|d| d.ticket_id.as_str()) + .collect::>(), + vec!["pending", "progress"] + ); + assert_eq!( + pending[0].input_event_id, + "assignment-event:migration-tracking:a-pending" + ); + store + .with_conn(|conn| { + assert_eq!(current_schema_version(conn)?, 93); + assert_eq!( + stored_fixture_rows(conn, "ticket_worker_assignments"), + assignments + ); + assert_eq!( + stored_fixture_rows(conn, "ticket_assignment_operations"), + operations + ); + assert_eq!( + stored_fixture_rows(conn, "typed_ticket_assignment_tracking_history"), + tracking + ); + let violations: i64 = + conn.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |r| { + r.get(0) + })?; + assert_eq!(violations, 0); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_startup_simultaneous_reservations_admit_runtime_effects_once() { + let (directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let threads = (0..2) + .map(|_| { + let barrier = barrier.clone(); + let path = directory.path().join("workspace.db"); + std::thread::spawn(move || { + let store = SqliteWorkspaceStore::open(path).unwrap(); + barrier.wait(); + delivery_store_start_reserve(&store, "start-a", "a", &tracking_store_source()) + .unwrap() + }) + }) + .collect::>(); + let admitted = threads + .into_iter() + .map(|t| t.join().unwrap()) + .collect::>(); + assert_eq!(admitted.iter().filter(|result| **result).count(), 1); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM ticket_assignment_operation_fences", + [], + |r| r.get(0), + )?; + assert_eq!(count, 1); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn assignment_delivery_fresh_and_upgraded_operational_schemas_are_identical() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("retained.db"); + let conn = tracking_schema91_fixture(&path); + migrate_ticket_tracking_v91_to_v92(&conn).unwrap(); + drop(conn); + let upgraded = SqliteWorkspaceStore::open(&path).unwrap(); + let fresh = SqliteWorkspaceStore::in_memory().unwrap(); + let objects = |store: &SqliteWorkspaceStore| { + store.with_conn(|conn| { + let mut statement=conn.prepare("SELECT type,name,sql FROM sqlite_schema WHERE name LIKE 'ticket_assignment_delivery_%' OR name LIKE 'ticket_assignment_operation_fence%' ORDER BY name")?; + Ok(statement.query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?,r.get::<_,String>(2)?)))?.collect::>>()?) + }).unwrap() + }; + assert_eq!(objects(&fresh), objects(&upgraded)); +} + +#[tokio::test] +async fn assignment_delivery_missing_immutable_request_guard_refuses_startup() { + let (directory, store) = tracking_store_fixture().await; + store + .with_conn(|conn| { + conn.execute( + "DROP TRIGGER ticket_assignment_delivery_request_no_update", + [], + )?; + Ok(()) + }) + .unwrap(); + drop(store); + assert!( + matches!(SqliteWorkspaceStore::open(directory.path().join("workspace.db")),Err(Error::Store(message)) if message.contains("ticket_assignment_delivery_request_no_update")) + ); +} + +#[tokio::test] +async fn assignment_delivery_fabricated_inputs_and_foreign_workspace_cannot_create_receipts() { + let (_directory, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "a", "assign-a"); + let source = tracking_store_source(); + assert!( + store + .reserve_ticket_assignment_delivery("w", "a", "ticket-event:9999", &source) + .is_err() + ); + assert!( + store + .reserve_ticket_assignment_delivery( + "other", + "a", + "assignment-event:event-assign-a", + &source + ) + .is_err() + ); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM ticket_assignment_deliveries", + [], + |r| r.get(0), + )?; + assert_eq!(count, 0); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn assignment_startup_unfinished_dependencies_remain_diagnostics_not_store_admission_gates() { + let (_directory, store) = manual_worker_start_store().await; + seed_manual_start_ticket(&store, "partial", "open"); + seed_manual_start_ticket(&store, "dependency", "open"); + store.with_conn(|conn| { + conn.execute("INSERT INTO typed_ticket_relations(workspace_id,ticket_id,kind,target,author,at) VALUES('w','partial','depends_on','dependency','owner','1')",[])?; + Ok(()) + }).unwrap(); + let assignment = manual_worker_assignment("partial", "partial-assignment"); + store + .start_ticket_with_worker_assignment( + &assignment, + "partial-event", + "partial-operation", + None, + ) + .unwrap(); + assert!( + store + .get_active_ticket_role_assignment("w", "partial", TicketAssignmentRole::Worker) + .unwrap() + .is_some() + ); + assert!( + store + .get_active_ticket_role_assignment("w", "dependency", TicketAssignmentRole::Worker) + .unwrap() + .is_none() + ); + store + .with_conn(|conn| { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM typed_ticket_relations WHERE kind='depends_on'", + [], + |r| r.get(0), + )?; + assert_eq!(count, 1); + Ok(()) + }) + .unwrap(); +} diff --git a/crates/workspace-server/src/assignment_tracking_tests.rs b/crates/workspace-server/src/assignment_tracking_tests.rs index 3fd7f8b6b..7c85c4812 100644 --- a/crates/workspace-server/src/assignment_tracking_tests.rs +++ b/crates/workspace-server/src/assignment_tracking_tests.rs @@ -439,12 +439,12 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip .iter() .map(|m| m.version) .collect::>(), - vec![92] + vec![92, 93] ); let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { - assert_eq!(current_schema_version(conn)?, 92); + assert_eq!(current_schema_version(conn)?, 93); assert_eq!( conn.query_row( "SELECT applied_at FROM ticket_schema_migrations WHERE version=10", @@ -457,7 +457,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip workspace_schema_migration_history(conn)?, vec![ (91, "workspace schema baseline".into()), - (92, "Ticket-owned assignment tracking".into()) + (92, "Ticket-owned assignment tracking".into()), + (93, "Durable assignment deliveries and source-bound startup fences".into()) ] ); for (table, rows) in retained.iter().zip(original.iter()) { @@ -595,7 +596,7 @@ fn tracking_store_schema91_closed_unknown_assignment_uses_assign_audit_for_first .is_empty() ); store.with_conn(|conn| { - assert_eq!(current_schema_version(conn)?, 92); + assert_eq!(current_schema_version(conn)?, 93); assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations", [], |row| row.get::<_,i64>(0))?, 11); assert_eq!(conn.query_row("SELECT workflow_state FROM typed_tickets WHERE ticket_id='historical-done'", [], |row| row.get::<_,String>(0))?, "closed"); let stored: (Option, Option) = conn.query_row("SELECT tracking_json,last_tracking_event_id FROM ticket_worker_assignments WHERE assignment_id='a-historical-done'", [], |row| Ok((row.get(0)?,row.get(1)?)))?; diff --git a/crates/workspace-server/src/authority.rs b/crates/workspace-server/src/authority.rs index 686be3115..abda3e7b2 100644 --- a/crates/workspace-server/src/authority.rs +++ b/crates/workspace-server/src/authority.rs @@ -783,12 +783,12 @@ impl SqliteWorkspaceAuthority { }) }) .transpose()?; - // Responsibility remains visible after work ends; action eligibility must - // use only active unfinished work, including after a Ticket is reopened. + // Work start eligibility uses unfinished work. Delegation authoring and + // clearing use the retained current pointer, including after close. let active_role_assignments = self .store .list_active_ticket_role_assignments(&self.workspace_id, id)?; - let has_orchestrator = active_role_assignments + let has_orchestrator = role_assignments .iter() .any(|assignment| assignment.role == TicketAssignmentRole::Orchestrator); let has_worker = active_role_assignments @@ -811,7 +811,6 @@ impl SqliteWorkspaceAuthority { }) .collect::>>()?; let write_repository_keys = ticket_write_target_repository_keys(&targets); - let has_blockers = !ticket.relations.blockers.is_empty(); let mut assignment_diagnostics = Vec::new(); if let Some(legacy_assignee) = ticket .meta @@ -827,12 +826,18 @@ impl SqliteWorkspaceAuthority { let action_eligibility = TicketActionEligibility { // Case openness is not readiness. These are explicit operations; // occupancy remains an independent resource constraint. - can_assign_orchestrator: !has_orchestrator && !has_worker, + // Delegation neither occupies nor transfers existing Worker work. + // Start eligibility below separately retains unfinished-work guards. + // A fresh operation explicitly delegates/reissues even to the same + // principal. Expected-assignment CAS (when supplied) and source + // authority are checked by the mutation, not by Ticket phase. + can_assign_orchestrator: true, can_unassign_orchestrator: has_orchestrator, can_queue: ticket.meta.workflow_state == TicketState::Open, - can_start_manual_worker: ticket.meta.workflow_state == TicketState::Open - && !has_worker - && !has_blockers, + // Unfinished dependencies remain relation diagnostics, not a blanket + // start refusal. Backend start/assign still validates authorization, + // resource grants, reservations and actual unfinished work. + can_start_manual_worker: ticket.meta.workflow_state == TicketState::Open && !has_worker, queue_tickets: dependency_check.queue_tickets.clone(), blockers: action_blockers, }; @@ -3232,6 +3237,81 @@ mod tests { ); } + #[tokio::test] + async fn unfinished_dependencies_remain_diagnostic_without_disabling_manual_start() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("workspace.db"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: "w".into(), + owner_account_id: "owner-account".into(), + display_name: "W".into(), + state: "active".into(), + created_at: "1".into(), + updated_at: "1".into(), + }) + .await + .unwrap(); + let backend = SqliteTicketBackend::open_verified(&path, "w").unwrap(); + let dependent = backend + .create(ticket::NewTicket::new( + "Partial investigation without resources", + )) + .unwrap(); + let prerequisite = backend + .create(ticket::NewTicket::new("Unfinished prerequisite")) + .unwrap(); + backend + .add_ticket_relation( + dependent.id.clone().into(), + ticket::NewTicketRelation { + kind: ticket::TicketRelationKind::DependsOn, + target: prerequisite.id.clone(), + note: Some("Partial work can proceed".into()), + author: Some("owner-account".into()), + }, + ) + .unwrap(); + let authority = SqliteWorkspaceAuthority::new(&path, "w").unwrap(); + let detail = authority.ticket(&dependent.id).unwrap(); + assert!(detail.targets.is_empty()); + assert!(detail.action_eligibility.can_start_manual_worker); + assert!(detail.action_eligibility.blockers.is_empty()); + assert_eq!(detail.relations.blockers.len(), 1); + assert_eq!( + detail.relations.blockers[0].blocking_ticket, + prerequisite.id + ); + assert_eq!(detail.relations.outgoing[0].kind, "depends_on"); + let blocked = authority + .query_tickets(TicketQueryRequest { + attention: vec![server_api::TicketAttention::Blocked], + ..TicketQueryRequest::default() + }) + .unwrap(); + assert_eq!(blocked.items.len(), 1); + assert_eq!(blocked.items[0].id, dependent.id); + assert_eq!(blocked.items[0].unresolved_blocker_count, 1); + let prerequisite_detail = authority.ticket(&prerequisite.id).unwrap(); + assert!(prerequisite_detail.assignments.is_empty()); + assert_eq!(prerequisite_detail.state.as_str(), "open"); + rusqlite::Connection::open(&path) + .unwrap() + .execute( + "UPDATE typed_tickets SET workflow_state='closed' WHERE ticket_id=?1", + [&dependent.id], + ) + .unwrap(); + assert!( + !authority + .ticket(&dependent.id) + .unwrap() + .action_eligibility + .can_start_manual_worker + ); + } + #[tokio::test] async fn manual_worker_eligibility_accepts_explicit_states_without_reviving_released_work() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 73733162b..765a557a7 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -453,6 +453,10 @@ pub struct WorkerSpawnWorkingDirectoryRequest { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct WorkerTicketAssignmentRequest { + /// Observed delegation required for Orchestrator Ticket starts; manual starts omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub orchestrator_assignment_id: Option, + pub ticket_id: String, pub operation_id: String, } @@ -7653,6 +7657,7 @@ mod tests { fn worker_spawn_idempotency_fingerprint_covers_canonical_initial_submit() { let mut request = embedded_spawn_request(); request.ticket_assignment = Some(WorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: "00001KVZSGT0Q".to_string(), operation_id: "operation-1".to_string(), }); diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index f6535f9c8..3dc7aa8fe 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1540,7 +1540,7 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { #[derive(Default)] struct OrchestratorAttentionState { sending: bool, - restore_recheck: bool, + input_recheck: bool, accepted: Option, pending: Option, } @@ -1550,6 +1550,7 @@ struct OrchestratorAttentionDelivery { fingerprint: String, request_id: String, content: String, + input: crate::store::TicketAssignmentDelivery, } struct OrchestratorAttentionReservation(Option>>); @@ -1558,10 +1559,11 @@ impl OrchestratorAttentionReservation { fn complete(mut self) -> bool { let state = self.0.take().expect("active attention reservation"); let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); - // Release and take deferred work atomically. A Restore arriving after - // release can reserve its own check; one arriving before is ours to drain. + // Release and take deferred input checks atomically. A committed input or + // Restore arriving before release is ours to drain; one after release + // can reserve its own check. state.sending = false; - std::mem::take(&mut state.restore_recheck) + std::mem::take(&mut state.input_recheck) } } @@ -3840,59 +3842,86 @@ impl WorkspaceApi { )) .map_err(|message| Error::Config(message.to_string()))?; } - let expired_external_workdirs = api.store.reconcile_external_workdir_grants_after_restart( - &api.config.workspace_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; + let recovery_api = api.clone(); + let expired_external_workdirs = api + .store + .db_executor() + .run(move || { + recovery_api + .store + .reconcile_external_workdir_grants_after_restart( + &recovery_api.config.workspace_id, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + }) + .await?; for workdir_id in expired_external_workdirs { cleanup_external_workdir_attachments(&api, &workdir_id) .await .map_err(|error| error.error)?; } - for grant in api + let grant_api = api.clone(); + let grants = api .store - .list_external_workdir_grants(&api.config.workspace_id)? + .db_executor() + .run(move || { + grant_api + .store + .list_external_workdir_grants(&grant_api.config.workspace_id) + }) + .await?; + for grant in grants .into_iter() .filter(|grant| !matches!(grant.status.as_str(), "revoked" | "expired")) { schedule_external_workdir_expiry(&api, &grant)?; } - recover_workdir_removals(&api)?; + let workdir_recovery_api = api.clone(); + host_io_executor()? + .run(move || recover_workdir_removals(&workdir_recovery_api)) + .await?; recover_runtime_removals(&api).await; - let recovery_now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - let deliveries = api - .store - .recover_backend_job_deliveries(&api.config.workspace_id, &recovery_now)?; - for delivery in deliveries { - let Some(job) = api - .store - .get_backend_job(&api.config.workspace_id, &delivery.job_id)? - else { - continue; - }; - let Some(attempt) = api.store.get_backend_job_attempt( - &api.config.workspace_id, - &delivery.job_id, - &delivery.attempt_id, - )? - else { - continue; - }; - if job.state == BackendJobState::Completed - && attempt.state == BackendJobAttemptState::Completed - { - api.deliver_backend_job_result( - &BackendJobResultAcceptance { - job, - attempt, - replayed: true, - }, - &delivery.target, - ); - } - } - api.recover_backend_jobs()?; - Ok(api) + host_io_executor()? + .run(move || { + let recovery_now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let deliveries = api + .store + .recover_backend_job_deliveries(&api.config.workspace_id, &recovery_now)?; + for delivery in deliveries { + let Some(job) = api + .store + .get_backend_job(&api.config.workspace_id, &delivery.job_id)? + else { + continue; + }; + let Some(attempt) = api.store.get_backend_job_attempt( + &api.config.workspace_id, + &delivery.job_id, + &delivery.attempt_id, + )? + else { + continue; + }; + if job.state == BackendJobState::Completed + && attempt.state == BackendJobAttemptState::Completed + { + api.deliver_backend_job_result( + &BackendJobResultAcceptance { + job, + attempt, + replayed: true, + }, + &delivery.target, + ); + } + } + api.recover_backend_jobs()?; + if let Some(orchestrator) = find_online_workspace_orchestrator(&api) { + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } + Ok(api) + }) + .await } pub fn workspace_id(&self) -> &str { @@ -7041,6 +7070,7 @@ fn worker_ticket_assignment_from_api( assignment: server_api::RuntimeWorkerTicketAssignmentRequest, ) -> WorkerTicketAssignmentRequest { WorkerTicketAssignmentRequest { + orchestrator_assignment_id: assignment.orchestrator_assignment_id, ticket_id: assignment.ticket_id, operation_id: assignment.operation_id, } @@ -8712,10 +8742,12 @@ impl server_api::ServerApi for ServerApiContractService { request: server_api::CreateWorkspaceWorkerRequest, ) -> std::result::Result { - create_workspace_worker( - State(self.workspace_api()?.clone()), + create_workspace_worker_authenticated( + self.workspace_api()?.clone(), contract_request_headers(&context)?, - Json(request), + request, + None, + Some(&context), ) .await .map(|Json(response)| response) @@ -8777,11 +8809,12 @@ impl server_api::ServerApi for ServerApiContractService { request: server_api::CreateWorkspaceWorkerRequest, ) -> std::result::Result { - spawn_known_worker( + spawn_known_worker_authenticated( State(self.workspace_api()?.clone()), AxumPath(ScopedWorkspacePath { workspace_id }), current_worker_contract_headers(&context)?, Json(request), + Some(&context), ) .await .map(|Json(response)| response) @@ -8955,11 +8988,14 @@ impl server_api::ServerApi for ServerApiContractService { request: server_api::CreateWorkspaceWorkerRequest, ) -> std::result::Result { - scoped_create_workspace_worker( - State(self.workspace_api()?.clone()), - AxumPath(ScopedWorkspacePath { workspace_id }), + validate_workspace_scope(self.workspace_api()?, &workspace_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + create_workspace_worker_authenticated( + self.workspace_api()?.clone(), contract_request_headers(&context)?, - Json(request), + request, + None, + Some(&context), ) .await .map(|Json(response)| response) @@ -9421,14 +9457,16 @@ impl server_api::ServerApi for ServerApiContractService { async fn runtime_worker_create_alias( &self, + context: server_api::ServerRequestContext, runtime_id: String, request: server_api::RuntimeWorkerSpawnRequest, ) -> std::result::Result { - let Json(response) = create_runtime_worker( + let Json(response) = create_runtime_worker_authenticated( State(self.workspace_api()?.clone()), AxumPath(runtime_id), Json(request), + Some(&context), ) .await .map_err(ApiError::into_repository_api_error)?; @@ -9457,18 +9495,19 @@ impl server_api::ServerApi for ServerApiContractService { async fn runtime_worker_create( &self, + context: server_api::ServerRequestContext, workspace_id: String, runtime_id: String, request: server_api::RuntimeWorkerSpawnRequest, ) -> std::result::Result { - let Json(response) = scoped_create_runtime_worker( + validate_workspace_scope(self.workspace_api()?, &workspace_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + let Json(response) = create_runtime_worker_authenticated( State(self.workspace_api()?.clone()), - AxumPath(ScopedRuntimePath { - workspace_id, - runtime_id, - }), + AxumPath(runtime_id), Json(request), + Some(&context), ) .await .map_err(ApiError::into_repository_api_error)?; @@ -13728,6 +13767,32 @@ async fn start_manual_ticket_worker_assignment( record: &TicketRoleAssignmentRecord, operation_id: &str, selected: &[server_api::TicketWorkerAttachmentBinding], + delegation: Option<&TicketStartDelegation>, +) -> ApiResult { + let api = api.clone(); + let record = record.clone(); + let operation_id = operation_id.to_owned(); + let selected = selected.to_vec(); + let delegation = delegation.cloned(); + run_host_workflow(async move { + start_manual_ticket_worker_assignment_on_host( + &api, + &record, + &operation_id, + &selected, + delegation.as_ref(), + ) + .await + }) + .await +} + +async fn start_manual_ticket_worker_assignment_on_host( + api: &WorkspaceApi, + record: &TicketRoleAssignmentRecord, + operation_id: &str, + selected: &[server_api::TicketWorkerAttachmentBinding], + delegation: Option<&TicketStartDelegation>, ) -> ApiResult { let TicketAssignmentPrincipal::Worker { runtime_id, @@ -13760,6 +13825,20 @@ async fn start_manual_ticket_worker_assignment( .store .get_ticket_assignment_operation(&api.config.workspace_id, operation_id)? { + if delegation.is_some() { + reserve_ticket_start( + api, + operation_id, + &record.ticket_id, + &worker.runtime_id, + Some(&worker.worker_id), + &fingerprint, + &record.assigned_at, + delegation, + )?; + } + api.store + .validate_ticket_assignment_operation_fence(&api.config.workspace_id, operation_id)?; if operation.request_fingerprint.as_deref() != Some(fingerprint.as_str()) { return Err(Error::TicketAssignmentConflict( "manual Worker claim operation was reused with different input".into(), @@ -13855,14 +13934,15 @@ async fn start_manual_ticket_worker_assignment( } let binding = validate_manual_ticket_worker_workdir_binding(api, &worker, &record.ticket_id, &selected)?; - let admitted = api.store.reserve_ticket_assignment_operation( - &api.config.workspace_id, + let admitted = reserve_ticket_start( + api, operation_id, &record.ticket_id, &worker.runtime_id, Some(&worker.worker_id), &fingerprint, &record.assigned_at, + delegation, )?; if !admitted { return Err(Error::TicketAssignmentConflict("manual Worker claim is already admitted; reread its durable result instead of redispatching".into()).into()); @@ -13886,6 +13966,8 @@ async fn start_manual_ticket_worker_assignment( )?; return Err(error.into()); } + api.store + .validate_ticket_assignment_operation_fence(&api.config.workspace_id, operation_id)?; let stopped = api .runtime .stop_worker( @@ -14232,9 +14314,10 @@ async fn apply_set_ticket_assignment( )) }) .await?; - let assignment = match role { + let (assignment, notification) = match role { TicketAssignmentRole::Orchestrator => { let store = api.store.clone(); + let notification_api = api.clone(); let record = record.clone(); let expected = expected_assignment_id.clone(); let operation_id = operation_id.clone(); @@ -14242,7 +14325,7 @@ async fn apply_set_ticket_assignment( .db_executor() .run(move || { let expected_assignment_id = expected; - if let Some(source) = source.as_ref() { + let assignment = if let Some(source) = source.as_ref() { store.set_current_ticket_orchestrator_assignment( &record, expected_assignment_id.as_deref(), @@ -14260,7 +14343,14 @@ async fn apply_set_ticket_assignment( &operation_id, expected_assignment_id.is_some(), ) - } + }?; + // Dispatch from the accepted SQL owner after commit, even + // when its waiter has gone. The response still observes the + // same best-effort notification completion as before offload. + Ok::<_, Error>(( + assignment, + enqueue_orchestrator_assignment_attention(¬ification_api), + )) }) .await? } @@ -14272,8 +14362,21 @@ async fn apply_set_ticket_assignment( ) .into()); } - start_manual_ticket_worker_assignment(&api, &record, &operation_id, &workdir_bindings) - .await? + let delegation = ticket_start_delegation( + source.as_ref(), + request.orchestrator_assignment_id.as_deref(), + )?; + ( + start_manual_ticket_worker_assignment( + &api, + &record, + &operation_id, + &workdir_bindings, + delegation.as_ref(), + ) + .await?, + None, + ) } TicketAssignmentRole::Owner | TicketAssignmentRole::Contributor => { return Err(Error::TicketAssignmentConflict( @@ -14283,6 +14386,9 @@ async fn apply_set_ticket_assignment( .into()); } }; + if let Some(notification) = notification { + let _ = notification.await; + } Ok(Json(server_api::TicketRoleAssignmentMutationResponse { workspace_id, ticket_id: ticket.id, @@ -14482,6 +14588,118 @@ async fn cancel_ticket_worker( Ok(()) } +/// Observation-bound delegation for a new Ticket execution operation. This is +/// distinct from self-reported handling and from the Worker work assignment. +type TicketStartDelegation = (String, ticket::TicketAssignmentTrackingSourceContext); + +fn ticket_start_delegation( + source: Option<&crate::store::TicketOrchestratorAssignmentSourceContext>, + observed_assignment_id: Option<&str>, +) -> Result> { + use crate::store::TicketOrchestratorAssignmentSourceContext; + match source { + Some(TicketOrchestratorAssignmentSourceContext::Runtime { + runtime_id, + worker_id, + agent_key, + authentication_binding_id, + }) => { + let assignment_id = observed_assignment_id.filter(|id| !id.trim().is_empty()) + .ok_or_else(|| Error::TicketAssignmentConflict("Orchestrator Ticket start requires the observed orchestrator_assignment_id".into()))?; + if assignment_id.len() > 1024 || assignment_id.chars().any(char::is_control) { + return Err(Error::InvalidInput( + "orchestrator_assignment_id must be control-free and at most 1024 bytes".into(), + )); + } + Ok(Some(( + assignment_id.to_owned(), + ticket::TicketAssignmentTrackingSourceContext { + actor: format!("worker:{runtime_id}/{worker_id}"), + runtime_id: runtime_id.clone(), + worker_id: worker_id.clone(), + agent_key: agent_key.clone(), + authentication_binding_id: authentication_binding_id.clone(), + }, + ))) + } + _ if observed_assignment_id.is_some() => Err(Error::WorkspacePermissionDenied( + "Only the authenticated current Orchestrator may start under a delegation".into(), + )), + _ => Ok(None), + } +} + +fn ticket_spawn_delegation( + api: &WorkspaceApi, + headers: &HeaderMap, + context: Option<&server_api::ServerRequestContext>, + observed_assignment_id: Option<&str>, +) -> Result> { + if let Some(context) = context { + // Generic authenticated Worker/manual starts are not implicit delegation. + // Only the registered Orchestrator starts under its observed assignment. + let worker = authenticated_ticket_worker_source(headers, context)?; + if let Some(worker) = worker.as_ref() { + api.store + .require_current_worker_singleton_owner(&api.config.workspace_id, worker)?; + if find_workspace_orchestrator(api).is_some_and(|current| current.worker == *worker) { + let (_, source) = authenticated_assignment_source(api, headers, context)?; + return ticket_start_delegation(Some(&source), observed_assignment_id); + } + } + ticket_start_delegation(None, observed_assignment_id) + } else { + // Only direct test adapters omit middleware context. Production routes + // must preserve the binding that actually authenticated the request. + #[cfg(test)] + { + ticket_start_delegation(None, observed_assignment_id) + } + #[cfg(not(test))] + { + Err(Error::WorkspacePermissionDenied( + "Ticket start requires Backend-authenticated context".into(), + )) + } + } +} + +fn reserve_ticket_start( + api: &WorkspaceApi, + operation_id: &str, + ticket_id: &str, + runtime_id: &str, + worker_id: Option<&str>, + fingerprint: &str, + created_at: &str, + delegation: Option<&TicketStartDelegation>, +) -> Result { + if let Some((assignment_id, source)) = delegation { + api.store + .reserve_ticket_assignment_operation_for_orchestrator( + &api.config.workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + fingerprint, + created_at, + assignment_id, + source, + ) + } else { + api.store.reserve_ticket_assignment_operation( + &api.config.workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + fingerprint, + created_at, + ) + } +} + fn validate_ticket_assignment_state( api: &WorkspaceApi, assignment: &WorkerTicketAssignmentRequest, @@ -14494,13 +14712,8 @@ fn validate_ticket_assignment_state( ticket.state.as_str() ))); } - let dependencies = - browser_ticket_backend(api)?.dependency_check(TicketIdOrSlug::Id(ticket.id))?; - if !dependencies.blockers.is_empty() { - return Err(Error::TicketAssignmentConflict( - "Ticket work is blocked by unresolved relations".into(), - )); - } + // Dependencies remain diagnostics for the caller's partial-work decision; + // they are not authority, resource occupancy, or execution reservations. Ok(()) } @@ -17723,55 +17936,6 @@ fn active_orchestrator_assignment( } } -fn orchestrator_interested( - api: &WorkspaceApi, - workspace_id: &str, - ticket_id: &str, - _state: &str, -) -> Result> { - let backend = browser_ticket_backend(api)?; - let ticket = backend.show(ticket_id.into()).map_err(Error::from)?; - if ticket.meta.workflow_state != TicketState::Open { - return Ok(None); - } - let has_unfinished_work = api - .store - .get_active_ticket_worker_assignment(workspace_id, ticket_id)? - .is_some(); - // Transitional routing continues to require an explicit queue request or - // unfinished work. Openness alone is not a routing request. A later case - // decision fences older queue receipts; T-741/742 will replace this path. - let explicitly_queued = ticket - .events - .iter() - .rev() - .find_map(|event| { - if event.attributes.contains_key("queue_root_ticket") { - Some(true) - } else if event.attributes.contains_key("case_action") { - Some(false) - } else { - None - } - }) - .unwrap_or_else(|| { - // Old queued/inprogress cases may predate queue receipts. This frozen - // migration handoff preserves their intent, but any new case decision - // fences it so reopening never revives an old routing request. - ticket.case_history.is_empty() - && ticket - .legacy_workflow_snapshot - .as_ref() - .is_some_and(|snapshot| { - matches!(snapshot.workflow_state.as_str(), "queued" | "inprogress") - }) - }); - if !has_unfinished_work && !explicitly_queued { - return Ok(None); - } - active_orchestrator_assignment(api, workspace_id, ticket_id) -} - fn worker_ticket_source_context( api: &WorkspaceApi, workspace_id: &str, @@ -17930,13 +18094,8 @@ fn notify_ticket_recipients( { recipients.push(assignment.worker.clone()); } - if orchestrator_interested(api, workspace_id, ticket_id, current_state) - .ok() - .flatten() - .is_some() - && let Some(orchestrator) = find_workspace_orchestrator(api) - { - recipients.push(orchestrator.worker.clone()); + if let Some(orchestrator) = find_workspace_orchestrator(api) { + dispatch_orchestrator_assignment_attention(api, &orchestrator.worker); } recipients.sort(); recipients.dedup(); @@ -19448,8 +19607,29 @@ async fn run_orchestrator_turn_end_hook(api: WorkspaceApi) { }; let mut worker_states = HashMap::new(); while let Some(update) = subscription.recv().await { - if !observe_orchestrator_turn_end(&api, &mut worker_states, update) { - return; + let stage_api = api.clone(); + let result = host_io_executor().map(|executor| async move { + executor + .run(move || -> Result<_> { + let keep_running = + observe_orchestrator_turn_end(&stage_api, &mut worker_states, update); + Ok((keep_running, worker_states)) + }) + .await + }); + match result { + Ok(operation) => match operation.await { + Ok((true, states)) => worker_states = states, + Ok((false, _)) => return, + Err(error) => { + tracing::warn!(%error, "Orchestrator hook stopped; durable inputs remain recoverable"); + return; + } + }, + Err(error) => { + tracing::warn!(%error, "Orchestrator hook executor unavailable"); + return; + } } } } @@ -19515,24 +19695,51 @@ fn maybe_dispatch_orchestrator_turn_end( if orchestrator.worker.runtime_id == EMBEDDED_WORKER_RUNTIME_ID && orchestrator.worker.worker_id == worker_id { - dispatch_orchestrator_queue_attention(api, &orchestrator.worker, false); + // Idle can be a reentrant consequence of this very Runtime send. It is + // not evidence of a new input and must not retry a failed send forever. + // Concurrent committed-input notifications independently defer a check. + with_orchestrator_attention_reservation( + api, + &orchestrator.worker, + false, + |state, pending| { + check_orchestrator_assignment_attention(api, &orchestrator.worker, state, pending); + }, + ); } } -fn dispatch_orchestrator_queue_attention( +fn enqueue_orchestrator_assignment_attention( api: &WorkspaceApi, - worker: &RuntimeWorkerRef, - retry_only: bool, -) { - with_orchestrator_attention_reservation(api, worker, retry_only, |state, pending| { - check_orchestrator_queue_attention(api, worker, state, pending); +) -> Option> { + let api = api.clone(); + let (done, completion) = tokio::sync::oneshot::channel(); + match host_io_executor().and_then(|executor| { + executor.dispatch(move || { + if let Some(orchestrator) = find_workspace_orchestrator(&api) { + dispatch_orchestrator_assignment_attention(&api, &orchestrator.worker); + } + let _ = done.send(()); + }) + }) { + Ok(()) => Some(completion), + Err(error) => { + tracing::warn!(%error, "Assign notification was not admitted; input remains durable"); + None + } + } +} + +fn dispatch_orchestrator_assignment_attention(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { + with_orchestrator_attention_reservation(api, worker, true, |state, pending| { + check_orchestrator_assignment_attention(api, worker, state, pending); }); } fn with_orchestrator_attention_reservation( api: &WorkspaceApi, worker: &RuntimeWorkerRef, - mut retry_only: bool, + defer_input_check_if_reserved: bool, mut check: impl FnMut( &Arc>, Option, @@ -19542,7 +19749,10 @@ fn with_orchestrator_attention_reservation( loop { let pending = { let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); - if state.sending || (retry_only && state.pending.is_none()) { + if state.sending { + // A no-op scan may already have read its empty snapshot. Retain + // committed input/startup/Restore triggers until its completion. + state.input_recheck |= defer_input_check_if_reserved; return; } state.sending = true; @@ -19553,74 +19763,78 @@ fn with_orchestrator_attention_reservation( if !reservation.complete() { return; } - // Accepted Restore is an explicit recheck even when the current check - // was a no-op or an already-online retry. Never infer it from Idle. - retry_only = false; + // Drain coalesced input triggers even when the first scan was a no-op. + // No recheck is created by a failed send or its own idle observation. } } -fn check_orchestrator_queue_attention( +fn orchestrator_delivery_target( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> Result { + if !find_online_workspace_orchestrator(api).is_some_and(|current| current.worker == *worker) { + return Err(Error::WorkspacePermissionDenied( + "Delivery target is not the current registered Orchestrator".into(), + )); + } + let binding = api + .config_store + .get_workspace_runtime_binding(&api.config.workspace_id, &worker.runtime_id)? + .filter(|binding| binding.revoked_at.is_none()) + .ok_or_else(|| { + Error::WorkspacePermissionDenied("Delivery target has no active Runtime binding".into()) + })?; + Ok(ticket::TicketAssignmentTrackingSourceContext { + actor: "backend:assignment-delivery".into(), + runtime_id: worker.runtime_id.clone(), + worker_id: worker.worker_id.clone(), + agent_key: "workspace-orchestrator".into(), + authentication_binding_id: binding.binding_id, + }) +} + +fn check_orchestrator_assignment_attention( api: &WorkspaceApi, worker: &RuntimeWorkerRef, state: &Arc>, pending: Option, ) { - // Unknown outcomes keep the original immutable payload and ID, even if the - // backlog or prompt projection changed before this retry. + let Ok(target) = orchestrator_delivery_target(api, worker) else { + return; + }; if let Some(delivery) = pending { - send_orchestrator_attention(api, worker, &state, delivery); - return; - } - let Ok(backend) = browser_ticket_backend(api) else { - return; - }; - let Ok(mut queued) = backend.list(ticket::TicketListQuery::states([ - ticket::TicketListState::Open, - ])) else { - return; - }; - queued.retain(|ticket| { - orchestrator_interested(api, &api.config.workspace_id, &ticket.id, "open") - .ok() - .flatten() - .is_some() - }); - queued.sort_by(|left, right| left.id.cmp(&right.id)); - if queued.is_empty() { - state.lock().unwrap_or_else(|p| p.into_inner()).accepted = None; - return; - } - let fingerprint = queued - .iter() - .map(|ticket| ticket.id.as_str()) - .collect::>() - .join("|"); - if state - .lock() - .unwrap_or_else(|p| p.into_inner()) - .accepted - .as_ref() - .map(|delivery| delivery.fingerprint.as_str()) - == Some(fingerprint.as_str()) - { - return; - } - - let attention_context = match orchestrator_queue_attention_context( - &api.config.workspace_id, - &api.config.workspace_id, - &queued, - ) { - Ok(context) => context, - Err(error) => { - tracing::warn!( - workspace_id = %api.config.workspace_id, - candidate_count = queued.len(), - diagnostic = error, - "orchestrator backlog attention projection rejected" - ); - return; + // Validate current identity before retrying an immutable request. Clear, + // reissue, close and Worker replacement cannot revive an old delivery. + if api + .store + .reserve_ticket_assignment_delivery( + &api.config.workspace_id, + &delivery.input.assignment_id, + &delivery.input.input_event_id, + &target, + ) + .is_ok() + { + send_orchestrator_attention(api, worker, state, delivery); + if state + .lock() + .unwrap_or_else(|p| p.into_inner()) + .pending + .is_some() + { + return; // Unknown/failed acceptance needs an explicit later trigger. + } + } else { + state.lock().unwrap_or_else(|p| p.into_inner()).pending = None; } + // Discarding or acknowledging an old retry must not consume this new- + // input trigger. Continue the common scan in the same reservation. + } + let Ok(inputs) = api + .store + .list_pending_ticket_assignment_deliveries(&api.config.workspace_id, 20) + else { + return; }; let Ok(Some(config_state)) = api .config_store @@ -19637,27 +19851,66 @@ fn check_orchestrator_queue_attention( let Ok(catalog) = worker::PromptCatalog::from_projection(projection.catalog().clone()) else { return; }; - let content = match catalog.orchestrator_queue_attention( - worker::OrchestratorQueueAttentionPrompt::Server, - &attention_context, - ) { - Ok(content) => content, - Err(error) => { - tracing::warn!( - workspace_id = %api.config.workspace_id, - diagnostic = %error, - "orchestrator backlog attention rendering failed" - ); - return; + for input in inputs { + // Reserve from the same pending-input contract, rechecking current + // assignment and target binding under the write lock before any I/O. + let Ok(input) = api.store.reserve_ticket_assignment_delivery( + &api.config.workspace_id, + &input.assignment_id, + &input.input_event_id, + &target, + ) else { + continue; + }; + let Ok(ticket) = api.authority.ticket(&input.ticket_id) else { + continue; + }; + let Ok(projected) = + worker::OrchestratorQueueAttentionTicket::new(ticket.resource_key, ticket.title) + else { + continue; + }; + let context = + worker::OrchestratorQueueAttentionContext::new(vec![projected.with_assignment_input( + &input.assignment_id, + &input.input_event_id, + input.waiting_reason.as_deref(), + input.next_action.as_deref(), + )]); + let Ok(content) = catalog.orchestrator_queue_attention( + worker::OrchestratorQueueAttentionPrompt::Server, + &context, + ) else { + continue; + }; + // The complete rendered request, including title/catalog projection, is + // pinned before dispatch; restart retries never render a new payload + // under an already accepted Runtime request id. + let Ok(content) = api.store.prepare_ticket_assignment_delivery_content( + &api.config.workspace_id, + &input.delivery_id, + &content, + &target, + ) else { + continue; + }; + let delivery = OrchestratorAttentionDelivery { + fingerprint: input.delivery_id.clone(), + request_id: input.request_id.clone(), + content, + input, + }; + state.lock().unwrap_or_else(|p| p.into_inner()).pending = Some(delivery.clone()); + send_orchestrator_attention(api, worker, state, delivery); + if state + .lock() + .unwrap_or_else(|p| p.into_inner()) + .pending + .is_some() + { + break; } - }; - let delivery = OrchestratorAttentionDelivery { - fingerprint, - request_id: uuid::Uuid::now_v7().to_string(), - content, - }; - state.lock().unwrap_or_else(|p| p.into_inner()).pending = Some(delivery.clone()); - send_orchestrator_attention(api, worker, &state, delivery); + } } fn send_orchestrator_attention( @@ -19666,30 +19919,76 @@ fn send_orchestrator_attention( state: &Mutex, delivery: OrchestratorAttentionDelivery, ) { - // Bounded delivery-side retry, not an Idle-snapshot retry. Rejection and - // transport ambiguity leave pending available for the next attempt. + tracing::debug!(delivery_id = %delivery.fingerprint, assignment_id = %delivery.input.assignment_id, "dispatch committed assignment input"); for _ in 0..2 { - let accepted = api - .runtime - .send_input( - worker, - WorkerInputRequest { - kind: WorkerInputKind::Notify, - content: delivery.content.clone(), - submission_request_id: Some(delivery.request_id.clone()), - segments: None, - }, - ) - .is_ok_and(|result| result.state == InternalWorkerOperationState::Accepted); - if accepted { - let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); - state.accepted = Some(delivery); - state.pending = None; + let Ok(target) = orchestrator_delivery_target(api, worker) else { return; + }; + if api + .store + .reserve_ticket_assignment_delivery( + &api.config.workspace_id, + &delivery.input.assignment_id, + &delivery.input.input_event_id, + &target, + ) + .is_err() + { + state.lock().unwrap_or_else(|p| p.into_inner()).pending = None; + return; + } + let result = api.runtime.send_input( + worker, + WorkerInputRequest { + kind: WorkerInputKind::Notify, + content: delivery.content.clone(), + submission_request_id: Some(delivery.request_id.clone()), + segments: None, + }, + ); + if result + .as_ref() + .is_ok_and(|result| result.state == InternalWorkerOperationState::Accepted) + { + if api + .store + .ack_ticket_assignment_delivery( + &api.config.workspace_id, + &delivery.input.delivery_id, + &target, + ) + .is_ok() + { + let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); + state.accepted = Some(delivery); + state.pending = None; + } + return; + } + if let Some(error_category) = ticket_notification_delivery_error_category(&result) { + tracing::warn!( + event = "ticket_assignment_delivery_pending", + workspace_id = %api.config.workspace_id, + assignment_id = %delivery.input.assignment_id, + input_event_id = %delivery.input.input_event_id, + delivery_id = %delivery.input.delivery_id, + recipient_runtime_id = %worker.runtime_id, + recipient_worker_id = %worker.worker_id, + error_category, + "committed delegation input remains pending; retry the saved request" + ); + emit_ticket_notification_delivery_warning(TicketNotificationDeliveryWarning::new( + &api.config.workspace_id, + &delivery.input.ticket_id, + "open", + worker, + error_category, + )); } } } +#[cfg(test)] fn orchestrator_queue_attention_context( expected_workspace_id: &str, candidate_workspace_id: &str, @@ -23402,11 +23701,43 @@ fn scoped_worker_control_operation_id(controller: &RuntimeWorkerRef, operation_i ) } +#[cfg(test)] async fn spawn_known_worker( + api: State, + path: AxumPath, + headers: HeaderMap, + request: Json, +) -> ApiResult> { + spawn_known_worker_authenticated(api, path, headers, request, None).await +} + +async fn spawn_known_worker_authenticated( + State(api): State, + AxumPath(path): AxumPath, + headers: HeaderMap, + Json(request): Json, + context: Option<&server_api::ServerRequestContext>, +) -> ApiResult> { + let context = context.cloned(); + run_host_workflow(async move { + spawn_known_worker_authenticated_on_host( + State(api), + AxumPath(path), + headers, + Json(request), + context.as_ref(), + ) + .await + }) + .await +} + +async fn spawn_known_worker_authenticated_on_host( State(api): State, AxumPath(path): AxumPath, headers: HeaderMap, Json(mut request): Json, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; @@ -23444,9 +23775,14 @@ async fn spawn_known_worker( operation_id: scoped_worker_control_operation_id(&controller, &operation_id), input_fingerprint, }); - let response = - create_workspace_worker_inner(api.clone(), headers, request, resolved_control_operation) - .await?; + let response = create_workspace_worker_authenticated_on_host( + api.clone(), + headers, + request, + resolved_control_operation, + context, + ) + .await?; if let Err(error) = api .store .create_worker_control_grant(&WorkerControlGrantRecord { @@ -23749,7 +24085,8 @@ async fn scoped_start_workspace_orchestrator( AxumPath(path): AxumPath, ) -> ApiResult> { host_io_executor()?.run(move || { - + #[cfg(test)] + tests::seed_test_delivery_binding(&api); validate_workspace_scope(&api, &path.workspace_id)?; let _guard = api.orchestrator_spawn_lock.lock().map_err(|_| { ApiError::with_diagnostics( @@ -23766,8 +24103,9 @@ async fn scoped_start_workspace_orchestrator( if let Some(existing) = find_workspace_orchestrator(&api) { if workspace_orchestrator_is_online(&existing) { drop(_guard); - // Already-online is not a new startup. Only retry outstanding delivery. - dispatch_orchestrator_queue_attention(&api, &existing.worker, true); + // Already-online creation uses the same committed-input scan; neither + // local cache loss nor handled status alone creates new work. + dispatch_orchestrator_assignment_attention(&api, &existing.worker); return Ok(Json(workspace_orchestrator_response(&api, "existing"))); } let attention = orchestrator_attention_state(&api, &existing.worker); @@ -23789,14 +24127,9 @@ async fn scoped_start_workspace_orchestrator( { attention.accepted = None; } - // A reserved caller may already have decided not to send. Carry - // the explicit Restore check through its completion in that case. - if attention.sending { - attention.restore_recheck = true; - } } drop(_guard); - dispatch_orchestrator_queue_attention(&api, &existing.worker, false); + dispatch_orchestrator_assignment_attention(&api, &existing.worker); return Ok(Json(workspace_orchestrator_response(&api, "restored"))); } if !diagnostics_indicate_unrecoverable_pending_workspace_restore(&restored.diagnostics) { @@ -23876,7 +24209,7 @@ async fn scoped_start_workspace_orchestrator( WorkerRegistryDisplayNamePolicy::UseProvided, )?; drop(_guard); - dispatch_orchestrator_queue_attention(&api, &worker.worker, false); + dispatch_orchestrator_assignment_attention(&api, &worker.worker); Ok(Json(workspace_orchestrator_response(&api, disposition))) }).await @@ -23955,6 +24288,7 @@ fn workspace_orchestrator_is_online(worker: &InternalWorkerSummary) -> bool { matches!(worker.state.as_str(), "idle" | "running" | "paused") } +#[cfg(test)] async fn scoped_create_workspace_worker( State(api): State, AxumPath(path): AxumPath, @@ -27242,6 +27576,7 @@ async fn scoped_list_runtime_workers( list_runtime_workers(State(api), AxumPath(path.runtime_id), Query(query)).await } +#[cfg(test)] async fn scoped_create_runtime_worker( State(api): State, AxumPath(path): AxumPath, @@ -29171,6 +29506,7 @@ fn browser_worker_spawn_policy( }, WorkerSpawnAcceptanceRequirement::RunAccepted { expected_segments }, Some(WorkerTicketAssignmentRequest { + orchestrator_assignment_id: assignment.orchestrator_assignment_id, ticket_id, operation_id, }), @@ -29218,6 +29554,7 @@ fn validate_caller_worker_singleton_key(singleton_key: Option<&str>) -> Result<( Ok(()) } +#[cfg(test)] async fn create_workspace_worker( State(api): State, headers: HeaderMap, @@ -29226,11 +29563,44 @@ async fn create_workspace_worker( create_workspace_worker_inner(api, headers, request, None).await } +#[cfg(test)] async fn create_workspace_worker_inner( api: WorkspaceApi, headers: HeaderMap, request: CreateWorkspaceWorkerRequest, resolved_control_operation: Option, +) -> ApiResult> { + create_workspace_worker_authenticated(api, headers, request, resolved_control_operation, None) + .await +} + +async fn create_workspace_worker_authenticated( + api: WorkspaceApi, + headers: HeaderMap, + request: CreateWorkspaceWorkerRequest, + resolved_control_operation: Option, + context: Option<&server_api::ServerRequestContext>, +) -> ApiResult> { + let context = context.cloned(); + run_host_workflow(async move { + create_workspace_worker_authenticated_on_host( + api, + headers, + request, + resolved_control_operation, + context.as_ref(), + ) + .await + }) + .await +} + +async fn create_workspace_worker_authenticated_on_host( + api: WorkspaceApi, + headers: HeaderMap, + request: CreateWorkspaceWorkerRequest, + resolved_control_operation: Option, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let claim_lock = request .ticket_assignment @@ -29251,7 +29621,23 @@ async fn create_workspace_worker_inner( feature_connections, control_operation_id: _, } = request; - let source = optional_worker_mutation_source(&api, &api.config.workspace_id, &headers)?; + let delegation = ticket_assignment + .as_ref() + .map(|assignment| { + ticket_spawn_delegation( + &api, + &headers, + context, + assignment.orchestrator_assignment_id.as_deref(), + ) + }) + .transpose()? + .flatten(); + let source = if let Some(context) = context { + authenticated_ticket_worker_source(&headers, context)? + } else { + optional_worker_mutation_source(&api, &api.config.workspace_id, &headers)? + }; let config_state = api .config_store .load_workspace_config(&api.config.workspace_id)? @@ -29384,14 +29770,15 @@ async fn create_workspace_worker_inner( if let (Some(assignment), Some(fingerprint)) = (assignment.as_ref(), assignment_fingerprint.as_deref()) { - fresh_ticket_claim = api.store.reserve_ticket_assignment_operation( - &api.config.workspace_id, + fresh_ticket_claim = reserve_ticket_start( + &api, &assignment.operation_id, &assignment.ticket_id, &runtime_id, None, fingerprint, &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + delegation.as_ref(), )?; let operation = api .store @@ -29419,6 +29806,12 @@ async fn create_workspace_worker_inner( .into()); } } + if let Some(assignment) = assignment.as_ref() { + api.store.validate_ticket_assignment_operation_fence( + &api.config.workspace_id, + &assignment.operation_id, + )?; + } let result = match api.spawn_workspace_worker_with_claim_admission( &runtime_id, request, @@ -29969,6 +30362,13 @@ fn resume_internal_worker_restore_intent( request: server_api::WorkerRestoreRequest, ) -> ApiResult { let request_id = request.request_id.clone(); + if request_id.starts_with("manual-worker-restore:") { + let operation_id = request_id + .strip_prefix("manual-worker-restore:") + .expect("checked prefix"); + api.store + .validate_ticket_assignment_operation_fence(&api.config.workspace_id, operation_id)?; + } let result = match WorkspaceWorker::resolve(api, &worker.runtime_id, &worker.worker_id)? .restore_internal(request) { @@ -30525,12 +30925,59 @@ fn runtime_diagnostics_message(diagnostics: &[RuntimeDiagnostic]) -> String { } } +#[cfg(test)] async fn create_runtime_worker( + api: State, + runtime_id: AxumPath, + request: Json, +) -> ApiResult> { + create_runtime_worker_authenticated(api, runtime_id, request, None).await +} + +async fn create_runtime_worker_authenticated( State(api): State, AxumPath(runtime_id): AxumPath, Json(request): Json, + context: Option<&server_api::ServerRequestContext>, +) -> ApiResult> { + let context = context.cloned(); + run_host_workflow(async move { + create_runtime_worker_authenticated_on_host( + State(api), + AxumPath(runtime_id), + Json(request), + context.as_ref(), + ) + .await + }) + .await +} + +async fn create_runtime_worker_authenticated_on_host( + State(api): State, + AxumPath(runtime_id): AxumPath, + Json(request): Json, + context: Option<&server_api::ServerRequestContext>, ) -> ApiResult> { let mut request = worker_spawn_request_from_api(request)?; + let headers = context + .map(contract_request_headers) + .transpose() + .map_err(|error| Error::InvalidInput(error.message))? + .unwrap_or_default(); + let delegation = request + .ticket_assignment + .as_ref() + .map(|assignment| { + ticket_spawn_delegation( + &api, + &headers, + context, + assignment.orchestrator_assignment_id.as_deref(), + ) + }) + .transpose()? + .flatten(); let claim_lock = request .ticket_assignment .as_ref() @@ -30597,14 +31044,15 @@ async fn create_runtime_worker( if let (Some(assignment), Some((_, fingerprint))) = (lifecycle_assignment.as_ref(), spawn_idempotency.as_ref()) { - fresh_ticket_claim = match api.store.reserve_ticket_assignment_operation( - &api.config.workspace_id, + fresh_ticket_claim = match reserve_ticket_start( + &api, &assignment.operation_id, &assignment.ticket_id, &runtime_id, None, fingerprint, &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + delegation.as_ref(), ) { Ok(fresh) => fresh, Err(error) => { @@ -30624,6 +31072,12 @@ async fn create_runtime_worker( diagnostics: Vec::new(), })?)); } + if let Some(assignment) = lifecycle_assignment.as_ref() { + api.store.validate_ticket_assignment_operation_fence( + &api.config.workspace_id, + &assignment.operation_id, + )?; + } let result = api.spawn_workspace_worker_with_claim_admission(&runtime_id, request, fresh_ticket_claim)?; if let Some(worker) = result.worker.as_ref() { @@ -34259,6 +34713,13 @@ mod db_execution_tests; #[cfg(test)] mod tests { + mod assignment_delivery_tests { + include!("server_assignment_delivery_tests.rs"); + } + + mod assignment_start_tests { + include!("server_assignment_start_tests.rs"); + } mod assignment_tracking_tests { include!("server_assignment_tracking_tests.rs"); } @@ -38939,6 +39400,7 @@ mod tests { Segment::text("Implement Ticket 00001KZ9E0DBS"), ]; let assignment_request = || CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: "00001KZ9E0DBS".to_string(), operation_id: "worker-spawn:00001KZ9E0DBS:call-1".to_string(), }; @@ -38961,6 +39423,7 @@ mod tests { assert_eq!( assignment, Some(WorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: "00001KZ9E0DBS".to_string(), operation_id: "worker-spawn:00001KZ9E0DBS:call-1".to_string(), }) @@ -38968,6 +39431,7 @@ mod tests { assert!( browser_worker_spawn_policy( Some(CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: " ".to_string(), operation_id: "operation".to_string(), }), @@ -39034,6 +39498,7 @@ mod tests { }, profile: ProfileSelector::Builtin("builtin:coder".to_string()), ticket_assignment: Some(WorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket.id, operation_id: "planning-spawn".to_string(), }), @@ -39134,6 +39599,7 @@ mod tests { singleton_key: None, profile: Some("builtin:coder".to_string()), ticket_assignment: Some(CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket.id.clone(), operation_id: "failed-queued-assignment".to_string(), }), @@ -46589,7 +47055,10 @@ mod tests { protocol::WorkerRunState::Running, )) })); - assert_eq!( + // Cleanup admission precedes its durable claim. The accepted job may + // still be queued, but neither queued nor executing ownership may stop + // this Run before the exact tracked finish marker below is published. + assert!(matches!( api.store .get_backend_job_attempt( TEST_WORKSPACE_ID, @@ -46599,8 +47068,8 @@ mod tests { .unwrap() .unwrap() .worker_cleanup_state, - Some(BackendJobWorkerCleanupState::Executing) - ); + Some(BackendJobWorkerCleanupState::Pending | BackendJobWorkerCleanupState::Executing) + )); execution.publish_worker_state( &worker, @@ -49619,6 +50088,7 @@ mod tests { accept_queued_ticket_after_worker_spawn( &api, &crate::hosts::WorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket.id.clone(), operation_id: "notification-source-assignment-operation".to_string(), }, @@ -49649,27 +50119,37 @@ mod tests { } let inputs = execution.take_inputs(); - let expected_states = [ - ("open", "open"), - ("open", "open"), - ("open", "open"), - ("open", "open"), - ]; - assert_eq!(inputs.len(), expected_states.len()); - for ((recipient, content), (previous_state, current_state)) in - inputs.iter().zip(expected_states) - { + assert_eq!(inputs.len(), 4, "Assign input plus three external events"); + let unique = inputs + .iter() + .map(|(_, content)| content.clone()) + .collect::>(); + assert_eq!( + unique.len(), + inputs.len(), + "each committed input has its own delivery identity" + ); + for (recipient, content) in &inputs { assert_eq!(recipient.worker_id.to_string(), orchestrator.worker_id); - assert!(!content.contains(&ticket.id)); - assert_eq!( - content, - &ticket_notification_content( - ticket.resource_key.as_deref().unwrap(), - previous_state, - current_state, - ) - ); + assert!(content.starts_with("Assigned Tickets have unreceived input:")); + assert!(content.contains(ticket.resource_key.as_deref().unwrap())); + assert!(content.contains("Delegation:")); + assert!(content.contains("input:")); } + let tracking = api + .store + .list_current_ticket_assignment_tracking(TEST_WORKSPACE_ID, &ticket.id) + .unwrap(); + assert_eq!( + tracking[0].tracking.as_ref().unwrap().status, + ticket::TicketAssignmentTrackingStatus::Waiting + ); + assert!( + api.store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) + .unwrap() + .is_empty() + ); } fn case_test_request( @@ -50037,6 +50517,7 @@ mod tests { "orchestrator".to_string(), )), Json(server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, workdir_bindings: Vec::new(), operation_id: "fresh-orchestrator-cycle".to_string(), principal: server_api::TicketAssignmentPrincipal::WorkspaceAgent { @@ -50083,10 +50564,12 @@ mod tests { let inputs = execution.take_inputs(); assert_eq!(inputs.len(), 1); assert_eq!(inputs[0].0.worker_id.to_string(), orchestrator.worker_id); - assert_eq!( - inputs[0].1, - ticket_notification_content(ticket.resource_key.as_deref().unwrap(), "open", "open") + assert!( + inputs[0] + .1 + .starts_with("Assigned Tickets have unreceived input:") ); + assert!(inputs[0].1.contains(&fresh_id)); assert!( api.store .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &ticket.id) @@ -50208,6 +50691,7 @@ mod tests { operation_id: &str, ) -> server_api::SetTicketRoleAssignmentRequest { server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, workdir_bindings: fixture .api .store @@ -50616,6 +51100,7 @@ mod tests { .create(ticket::NewTicket::new("Idempotent assignment")) .unwrap(); let request = || server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, workdir_bindings: Vec::new(), operation_id: "same-role-operation".to_string(), principal: server_api::TicketAssignmentPrincipal::WorkspaceAgent { @@ -50672,6 +51157,7 @@ mod tests { "worker".to_string(), )), Json(server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, workdir_bindings: Vec::new(), operation_id: "invalid-workspace-agent-coder".to_string(), principal: server_api::TicketAssignmentPrincipal::WorkspaceAgent { @@ -50728,7 +51214,7 @@ mod tests { .iter() .any(|blocker| blocker.contains("Orchestrator and manual Worker")) ); - assert!(!detail.action_eligibility.can_assign_orchestrator); + assert!(detail.action_eligibility.can_assign_orchestrator); assert!(!detail.action_eligibility.can_start_manual_worker); let stale = scoped_clear_ticket_assignment( @@ -51640,19 +52126,21 @@ mod tests { assert!(!serialized.contains("Ticket notification:")); let notifications = execution.take_inputs(); - assert_eq!(notifications.len(), 1); + assert_eq!( + notifications.len(), + 2, + "initial delegation plus external comment" + ); assert_eq!( notifications[0].0.worker_id.to_string(), orchestrator.worker_id ); - assert_eq!( - notifications[0].1, - ticket_notification_content( - ticket_ref.resource_key.as_deref().unwrap(), - TicketState::Open.as_str(), - TicketState::Open.as_str() - ) + assert!( + notifications + .iter() + .all(|(_, content)| content.starts_with("Assigned Tickets have unreceived input:")) ); + assert_ne!(notifications[0].1, notifications[1].1); execution.reject_inputs("sensitive fake Runtime transport detail"); TICKET_NOTIFICATION_DELIVERY_WARNING_CAPTURE @@ -51693,7 +52181,11 @@ mod tests { .filter(|warning| warning.ticket_id == ticket_ref.id) .cloned() .collect::>(); - assert_eq!(warnings.len(), 2); + assert_eq!( + warnings.len(), + 3, + "one missing Worker and two bounded Orchestrator delivery attempts" + ); assert!(warnings.iter().any(|warning| { warning.recipient_worker_id == missing_recipient.worker_id && warning.error_category == "unknown_worker" @@ -51707,969 +52199,15 @@ mod tests { assert!(!serialized.contains("Ticket notification:")); assert!(!serialized.contains("sensitive fake Runtime transport detail")); let attempts = execution.take_inputs(); - assert_eq!(attempts.len(), 1); + assert_eq!(attempts.len(), 2); + assert_eq!( + attempts[0].1, attempts[1].1, + "bounded retry retains immutable request payload" + ); assert_eq!(attempts[0].0.worker_id.to_string(), orchestrator.worker_id); } - async fn attention_test_api() -> ( - tempfile::TempDir, - WorkspaceApi, - Arc, - ) { - let dir = tempfile::tempdir().unwrap(); - init_clean_git_workspace(dir.path()); - let (api, execution) = test_api_with_recording_backend(dir.path()).await; - let mut input = ticket::NewTicket::new("Attention backlog"); - input.workflow_state = Some(TicketState::Open); - set_test_ticket_target(&mut input, "test-repository", "HEAD"); - let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); - assign_test_orchestrator(&api, &ticket.id); - browser_ticket_backend(&api) - .unwrap() - .queue_ready(ticket.id.into(), "test-request") - .unwrap(); - (dir, api, execution) - } - - async fn start_attention_test_orchestrator( - api: &WorkspaceApi, - ) -> BrowserWorkspaceOrchestratorResponse { - scoped_start_workspace_orchestrator( - State(api.clone()), - AxumPath(ScopedWorkspacePath { - workspace_id: TEST_WORKSPACE_ID.to_string(), - }), - ) - .await - .unwrap() - .0 - } - - #[tokio::test] - async fn orchestrator_attention_parallel_start_and_idle_observation_send_once() { - let (_dir, api, execution) = attention_test_api().await; - let reentrant_api = api.clone(); - execution.before_input_returns(move |worker| { - use protocol::subscription::SubscriptionWorkerState as S; - // The callback is inside Runtime send_input. Neither initial Idle nor - // a real turn end may deadlock or issue another in-flight delivery. - maybe_dispatch_orchestrator_turn_end( - &reentrant_api, - &worker.worker_id.to_string(), - None, - S::Idle, - ); - maybe_dispatch_orchestrator_turn_end( - &reentrant_api, - &worker.worker_id.to_string(), - Some(S::Running), - S::Idle, - ); - }); - let barrier = Arc::new(std::sync::Barrier::new(8)); - let results = std::thread::scope(|scope| { - let joins = (0..8) - .map(|_| { - let api = api.clone(); - let barrier = barrier.clone(); - scope.spawn(move || { - barrier.wait(); - futures::executor::block_on(start_attention_test_orchestrator(&api)) - }) - }) - .collect::>(); - joins - .into_iter() - .map(|j| j.join().unwrap()) - .collect::>() - }); - assert_eq!( - results - .iter() - .filter(|r| r.disposition == "created") - .count(), - 1 - ); - assert_eq!( - results - .iter() - .filter(|r| r.disposition == "existing") - .count(), - 7 - ); - assert_eq!(execution.take_inputs().len(), 1); - assert_eq!(execution.take_input_request_ids().len(), 1); - assert_eq!( - start_attention_test_orchestrator(&api).await.disposition, - "existing" - ); - assert!(execution.take_inputs().is_empty()); - } - - #[tokio::test] - async fn orchestrator_attention_barrier_callers_share_unsent_reservation() { - let (_dir, api, execution) = attention_test_api().await; - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let state = orchestrator_attention_state(&api, &worker); - state.lock().unwrap().accepted = None; - execution.take_inputs(); - execution.take_input_request_ids(); - let barrier = Arc::new(std::sync::Barrier::new(8)); - let (entered_tx, entered_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel(); - execution.before_input_returns(move |_| { - entered_tx.send(()).unwrap(); - release_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - }); - std::thread::scope(|scope| { - let joins = (0..8) - .map(|_| { - let api = api.clone(); - let worker = worker.clone(); - let barrier = barrier.clone(); - scope.spawn(move || { - barrier.wait(); - dispatch_orchestrator_queue_attention(&api, &worker, false); - }) - }) - .collect::>(); - entered_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - assert!(state.lock().unwrap().sending); - assert!(state.lock().unwrap().accepted.is_none()); - // The success record does not exist yet. A ninth caller must still - // return immediately rather than compare/send twice or wait on Runtime. - dispatch_orchestrator_queue_attention(&api, &worker, false); - release_tx.send(()).unwrap(); - for join in joins { - join.join().unwrap(); - } - }); - assert_eq!(execution.take_inputs().len(), 1); - assert_eq!(execution.take_input_request_ids().len(), 1); - assert!(!state.lock().unwrap().sending); - assert!(state.lock().unwrap().accepted.is_some()); - } - - #[tokio::test] - async fn orchestrator_attention_restore_rechecks_reserved_noop_before_and_after_comparison() { - for pause_before_comparison in [true, false] { - let (_dir, api, execution) = attention_test_api().await; - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let state = orchestrator_attention_state(&api, &worker); - let old_id = state - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .request_id - .clone(); - execution.take_inputs(); - execution.take_input_request_ids(); - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - execution.accept_restores(); - let (entered_tx, entered_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel(); - std::thread::scope(|scope| { - let api_for_caller = api.clone(); - let worker_for_caller = worker.clone(); - let caller = scope.spawn(move || { - let mut first = true; - with_orchestrator_attention_reservation( - &api_for_caller, - &worker_for_caller, - false, - |state, pending| { - if first && pause_before_comparison { - entered_tx.send(()).unwrap(); - release_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - } - check_orchestrator_queue_attention( - &api_for_caller, - &worker_for_caller, - state, - pending, - ); - if first && !pause_before_comparison { - // Old fingerprint matched: check returned without sending, - // but the exact same production reservation still exists. - assert_eq!( - state.lock().unwrap().accepted.as_ref().unwrap().request_id, - old_id - ); - entered_tx.send(()).unwrap(); - release_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - } - first = false; - }, - ); - }); - entered_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - assert!(state.lock().unwrap().sending); - let restored = futures::executor::block_on(start_attention_test_orchestrator(&api)); - assert_eq!(restored.disposition, "restored"); - let deferred = state.lock().unwrap().restore_recheck; - let old_retired = state.lock().unwrap().accepted.is_none(); - let inputs_before_release = execution.take_inputs(); - release_tx.send(()).unwrap(); - caller.join().unwrap(); - assert!(deferred); - assert!(old_retired); - assert!(inputs_before_release.is_empty()); - }); - assert_eq!(execution.take_inputs().len(), 1); - assert_eq!(execution.take_input_request_ids().len(), 1); - let state = state.lock().unwrap(); - assert!(state.accepted.is_some()); - assert!(!state.sending); - assert!(!state.restore_recheck); - } - } - - #[tokio::test] - async fn orchestrator_attention_restore_keeps_pending_unknown_delivery_and_id() { - let (_dir, api, execution) = attention_test_api().await; - execution.reject_inputs("unknown acceptance response"); - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let state = orchestrator_attention_state(&api, &worker); - let pending = state.lock().unwrap().pending.clone().unwrap(); - execution.take_inputs(); - execution.take_input_request_ids(); - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - execution.accept_restores(); - *execution.input_failure.lock().unwrap() = None; - let restored = start_attention_test_orchestrator(&api).await; - assert_eq!(restored.disposition, "restored"); - let attempts = execution.take_inputs(); - assert_eq!(attempts.len(), 1); - assert_eq!(attempts[0].1, pending.content); - assert_eq!( - execution.take_input_request_ids(), - vec![Some(pending.request_id.clone())] - ); - assert!(state.lock().unwrap().pending.is_none()); - assert_eq!( - state.lock().unwrap().accepted.as_ref().unwrap().request_id, - pending.request_id - ); - } - - #[tokio::test] - async fn orchestrator_attention_unknown_response_retry_keeps_id_and_payload() { - let (_dir, api, execution) = attention_test_api().await; - execution.reject_inputs("accepted response lost (unknown transport outcome)"); - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let state = orchestrator_attention_state(&api, &worker); - assert!(state.lock().unwrap().accepted.is_none()); - let pending = state.lock().unwrap().pending.clone().unwrap(); - let attempts = execution.take_inputs(); - let ids = execution.take_input_request_ids(); - assert_eq!(attempts.len(), 2); - assert_eq!(ids, vec![Some(pending.request_id.clone()); 2]); - // Change the backlog before retry: uncertain acceptance must never reuse - // the ID for a newly rendered payload. - let mut next = ticket::NewTicket::new("Changed backlog"); - next.workflow_state = Some(TicketState::Open); - set_test_ticket_target(&mut next, "test-repository", "HEAD"); - let next = browser_ticket_backend(&api).unwrap().create(next).unwrap(); - assign_test_orchestrator(&api, &next.id); - browser_ticket_backend(&api) - .unwrap() - .queue_ready(next.id.clone().into(), "test-request") - .unwrap(); - *execution.input_failure.lock().unwrap() = None; - assert_eq!( - start_attention_test_orchestrator(&api).await.disposition, - "existing" - ); - let retry = execution.take_inputs(); - assert_eq!(retry.len(), 1); - assert_eq!(retry[0].1, pending.content); - assert_eq!( - execution.take_input_request_ids(), - vec![Some(pending.request_id)] - ); - assert!(state.lock().unwrap().pending.is_none()); - assert!(state.lock().unwrap().accepted.is_some()); - use protocol::subscription::SubscriptionWorkerState as S; - maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, Some(S::Running), S::Idle); - let changed = execution.take_inputs(); - assert_eq!(changed.len(), 1); - assert!(changed[0].1.contains("Changed backlog")); - } - - #[tokio::test] - async fn orchestrator_attention_subscription_snapshots_only_establish_baselines() { - use crate::runtime_subscription::BrokerSubscriptionEvent as B; - use protocol::subscription::{ - SubscriptionEventPayload, SubscriptionSnapshot, SubscriptionWorker, - SubscriptionWorkerState as S, - }; - let (_dir, api, execution) = attention_test_api().await; - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let attention = orchestrator_attention_state(&api, &worker); - attention.lock().unwrap().accepted = None; - execution.take_inputs(); - let summary = |state: S| -> SubscriptionWorker { - serde_json::from_value(serde_json::json!({ - "worker_id": worker.worker_id, "state": state, - })) - .unwrap() - }; - let snapshot = |state| B::Snapshot { - snapshot: SubscriptionSnapshot::Workers { - workers: vec![summary(state)], - }, - }; - let event = |state| B::Event { - payload: SubscriptionEventPayload::WorkerUpserted { - worker: summary(state), - }, - }; - let mut states = HashMap::new(); - assert!(observe_orchestrator_turn_end( - &api, - &mut states, - snapshot(S::Idle) - )); - assert!(observe_orchestrator_turn_end( - &api, - &mut states, - event(S::Idle) - )); - observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); - // Neither snapshot itself nor an Idle event after reconnect is turn-end evidence. - observe_orchestrator_turn_end( - &api, - &mut states, - B::Disconnected { - message: "fixture reconnect".into(), - }, - ); - observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); - observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); - observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Running)); - observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); - assert!(attention.lock().unwrap().accepted.is_none()); - assert!(execution.take_inputs().is_empty()); - observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); - observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); - assert_eq!(execution.take_inputs().len(), 1); - } - - // Real Controller and canonical Session storage with isolated local config. - // A closed loopback model endpoint prevents any production-provider access; - // notifications commit at the model boundary before that request can fail. - struct AttentionWorkerFactory { - root: PathBuf, - } - - impl AttentionWorkerFactory { - async fn controller( - &self, - worker_id: String, - restore: bool, - ) -> std::result::Result - { - let name = format!("attention-{worker_id}"); - let root = self.root.join("workers").join(&worker_id); - let store = session_store::CombinedStore::new( - session_store::WorkerSessionStore::new(root.join("session")) - .map_err(|e| e.to_string())?, - session_store::WorkerAggregateStore::new(&root, &name) - .map_err(|e| e.to_string())?, - ); - let manifest = manifest::WorkerManifest::from_toml(&format!( - r#" - [worker] - name = "{name}" - pwd = "./" - [model] - scheme = "anthropic" - model_id = "attention-test" - base_url = "http://127.0.0.1:1" - auth = {{ kind = "none" }} - [engine] - max_tokens = 10 - [scope] - allow = [] - "# - )) - .map_err(|e| e.to_string())?; - let context = worker::WorkerWorkspaceContext::local_filesystem(None); - let workspace_client = context.client_handle(); - let loader = worker::PromptCatalogSource::builtins_only(); - let worker = if restore { - worker::Worker::restore_from_worker_metadata_with_context( - &name, - manifest, - store, - loader, - context, - worker::WorkerFilesystemAuthority::None, - ) - .await - } else { - worker::Worker::from_manifest_with_context( - manifest, - store, - loader, - context, - worker::WorkerFilesystemAuthority::None, - ) - .await - } - .map_err(|e| e.to_string())?; - let started = worker::PreparedWorker::new( - worker.with_notification_coalesce_delay(std::time::Duration::from_millis(10)), - worker::WorkerBootstrapLayout::RuntimeManagedRun { - run_dir: root.join(uuid::Uuid::now_v7().to_string()), - bash_output_dir: root.join("bash-output"), - }, - worker::WorkerControllerTransport::InProcess, - ) - .start() - .await - .map_err(|e| e.to_string())?; - Ok(worker_runtime::worker_backend::RuntimeWorkerController { - handle: started.handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), - controller_task: started.controller_task, - workspace_client, - }) - } - } - - #[async_trait::async_trait] - impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFactory { - async fn spawn_controller( - &self, - request: worker_runtime::execution::WorkerExecutionSpawnRequest, - ) -> std::result::Result - { - self.controller(request.worker_ref.worker_id.to_string(), false) - .await - } - async fn restore_controller( - &self, - request: worker_runtime::execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result - { - self.controller(request.worker_ref.worker_id.to_string(), true) - .await - } - } - - use worker_runtime::execution as attention_execution; - - // Fault injection occurs AFTER the real Controller durably accepted input. - struct AttentionResponseLossBackend { - inner: Arc, - lose_next: std::sync::atomic::AtomicBool, - attempts: Mutex>, - } - - impl attention_execution::WorkerExecutionBackend for AttentionResponseLossBackend { - fn backend_id(&self) -> &str { - self.inner.backend_id() - } - fn spawn_worker( - &self, - r: attention_execution::WorkerExecutionSpawnRequest, - ) -> attention_execution::WorkerExecutionSpawnResult { - self.inner.spawn_worker(r) - } - fn preflight_restore( - &self, - r: &attention_execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result<(), attention_execution::WorkerExecutionResult> { - self.inner.preflight_restore(r) - } - fn reconcile_restore( - &self, - r: attention_execution::WorkerExecutionRestoreRequest, - ) -> attention_execution::WorkerExecutionSpawnResult { - self.inner.reconcile_restore(r) - } - fn restore_worker( - &self, - r: attention_execution::WorkerExecutionRestoreRequest, - ) -> attention_execution::WorkerExecutionSpawnResult { - self.inner.restore_worker(r) - } - fn activate_restored_worker( - &self, - id: attention_execution::WorkerLifecycleOperationId, - worker_ref: &worker_runtime::identity::WorkerRef, - ) -> std::result::Result<(), String> { - self.inner.activate_restored_worker(id, worker_ref) - } - fn dispatch_input( - &self, - worker_ref: &worker_runtime::identity::WorkerRef, - input: worker_runtime::interaction::WorkerInput, - ) -> attention_execution::WorkerExecutionResult { - self.attempts - .lock() - .unwrap() - .push(input.submission_request_id.clone().unwrap()); - let result = self.inner.dispatch_input(worker_ref, input); - assert_eq!( - result.outcome, - attention_execution::WorkerExecutionOutcome::Accepted - ); - if self - .lose_next - .swap(false, std::sync::atomic::Ordering::SeqCst) - { - attention_execution::WorkerExecutionResult::errored( - attention_execution::WorkerExecutionOperation::Input, - "injected loss of accepted response", - ) - } else { - result - } - } - fn dispatch_method( - &self, - worker_ref: &worker_runtime::identity::WorkerRef, - method: protocol::Method, - ) -> attention_execution::WorkerExecutionResult { - self.inner.dispatch_method(worker_ref, method) - } - fn stop_worker_operation( - &self, - r: attention_execution::WorkerExecutionStopRequest, - ) -> attention_execution::WorkerExecutionResult { - self.inner.stop_worker_operation(r) - } - fn worker_snapshot( - &self, - worker_ref: &worker_runtime::identity::WorkerRef, - ) -> Option { - self.inner.worker_snapshot(worker_ref) - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn orchestrator_attention_real_worker_start_restore_accept_and_commit_once_each() { - real_attention_worker_start_restore(false).await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() { - real_attention_worker_start_restore(true).await; - } - - async fn real_attention_worker_start_restore(lose_response: bool) { - use session_store::{LogEntry, Store, WorkerSessionStore}; - let dir = tempfile::tempdir().unwrap(); - init_clean_git_workspace(dir.path()); - let config = test_server_config(dir.path()); - let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); - seed_test_registered_workspace(store.as_ref(), &config) - .await - .unwrap(); - // Real Worker Runtime adapter/controller/storage, isolated from profile - // workspace callbacks and real LLM credentials. - let execution = Arc::new( - worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new( - AttentionWorkerFactory { - root: config.embedded_runtime_store_root.clone(), - }, - ) - .unwrap(), - ); - let execution = Arc::new(AttentionResponseLossBackend { - inner: execution, - lose_next: std::sync::atomic::AtomicBool::new(lose_response), - attempts: Mutex::new(Vec::new()), - }); - let api = WorkspaceApi::new_with_execution_backend(config, store, execution.clone()) - .await - .unwrap(); - let mut input = ticket::NewTicket::new("Real notification backlog"); - input.workflow_state = Some(TicketState::Open); - set_test_ticket_target(&mut input, "test-repository", "HEAD"); - let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); - assign_test_orchestrator(&api, &ticket.id); - browser_ticket_backend(&api) - .unwrap() - .queue_ready(ticket.id.clone().into(), "test-request") - .unwrap(); - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - let session = WorkerSessionStore::open_read_only( - api.config - .embedded_runtime_store_root - .join("workers") - .join(&worker.worker_id) - .join("session"), - ) - .unwrap(); - let entries = || { - let id = session.session_id().unwrap().unwrap(); - session - .list_segments(id) - .unwrap() - .into_iter() - .flat_map(|segment| session.read_all_read_only(id, segment).unwrap()) - .collect::>() - }; - let receipts = || { - entries() - .iter() - .rev() - .find_map(|entry| match entry { - LogEntry::Extension { - domain, payload, .. - } if domain == "worker.pending_activations.v1" => { - Some(payload["notification_receipts"].as_array().unwrap().clone()) - } - LogEntry::AnnotatedSystemItem { extensions, .. } => extensions - .iter() - .find(|e| e.domain == "worker.pending_activations.v1") - .map(|e| { - e.payload["notification_receipts"] - .as_array() - .unwrap() - .clone() - }), - _ => None, - }) - .unwrap() - }; - let commits = || { - entries().iter().filter(|entry| matches!(entry, - LogEntry::AnnotatedSystemItem { entry, .. } - if matches!(&entry.item, session_store::SystemItem::Notification { message, .. } - if message.starts_with("Queued Tickets require attention:")) - )).count() - }; - async fn wait_for_commits usize>(count: F, expected: usize) { - tokio::time::timeout(std::time::Duration::from_secs(5), async { - loop { - let actual = count(); - assert!(actual <= expected, "duplicate canonical notifications"); - if actual == expected { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - }) - .await - .expect("controlled Worker notification commit"); - } - assert_eq!(receipts().len(), 1); - wait_for_commits(&commits, 1).await; - assert_eq!( - execution.attempts.lock().unwrap().len(), - if lose_response { 2 } else { 1 } - ); - let first_id = orchestrator_attention_state(&api, &worker) - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .request_id - .clone(); - use protocol::subscription::SubscriptionWorkerState as S; - for previous in [ - None, - Some(S::Idle), - Some(S::Stopped), - Some(S::Paused), - Some(S::Running), - ] { - maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, previous, S::Idle); - } - assert_eq!( - start_attention_test_orchestrator(&api).await.disposition, - "existing" - ); - assert_eq!(receipts().len(), 1); - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - let restored = start_attention_test_orchestrator(&api).await; - assert_eq!(restored.disposition, "restored"); - let second_id = orchestrator_attention_state(&api, &worker) - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .request_id - .clone(); - assert_ne!(first_id, second_id); - wait_for_commits(&commits, 2).await; - let attempts = execution.attempts.lock().unwrap(); - assert_eq!(attempts.len(), if lose_response { 3 } else { 2 }); - assert_eq!( - attempts.iter().filter(|id| **id == first_id).count(), - if lose_response { 2 } else { 1 } - ); - assert_eq!(attempts.iter().filter(|id| **id == second_id).count(), 1); - drop(attempts); - let receipts = receipts(); - assert_eq!(receipts.len(), 2); - assert_eq!( - receipts - .iter() - .filter(|r| r["notification_request_id"] == first_id) - .count(), - 1 - ); - assert_eq!( - receipts - .iter() - .filter(|r| r["notification_request_id"] == second_id) - .count(), - 1 - ); - // Stop only this isolated test fixture; no live dogfooding Worker is touched. - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - } - - #[tokio::test] - async fn orchestrator_attention_restore_and_replacement_have_first_delivery() { - let (_dir, api, execution) = attention_test_api().await; - let started = start_attention_test_orchestrator(&api).await; - let worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - started.worker.unwrap().worker_id, - ); - assert_eq!(execution.take_inputs().len(), 1); - api.runtime - .stop_worker( - &worker, - WorkerLifecycleRequest { - reason: None, - ticket_assignment: None, - }, - ) - .unwrap(); - execution.accept_restores(); - let restored = start_attention_test_orchestrator(&api).await; - assert_eq!(restored.disposition, "restored"); - assert_eq!(restored.worker.unwrap().worker_id, worker.worker_id); - assert_eq!(execution.take_inputs().len(), 1); - use protocol::subscription::SubscriptionWorkerState as S; - maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, None, S::Idle); - assert_eq!( - start_attention_test_orchestrator(&api).await.disposition, - "existing" - ); - assert!(execution.take_inputs().is_empty()); - // This is a fixture Worker, not the running dogfooding Orchestrator. - api.runtime.delete_worker(&worker).unwrap(); - api.worker_projection - .publish_ordered(|| { - api.store - .delete_worker_registry(TEST_WORKSPACE_ID, &worker) - .map(|commit| ((), commit)) - }) - .unwrap(); - let replacement = start_attention_test_orchestrator(&api).await; - assert_ne!(replacement.worker.unwrap().worker_id, worker.worker_id); - assert_eq!(execution.take_inputs().len(), 1); - } - - #[tokio::test] - async fn orchestrator_running_to_idle_recovers_queued_ticket_without_notification_memory() { - let dir = tempfile::tempdir().unwrap(); - init_clean_git_workspace(dir.path()); - let (api, execution) = test_api_with_recording_backend(dir.path()).await; - let backend = browser_ticket_backend(&api).unwrap(); - let mut input = ticket::NewTicket::new("Recover queued work"); - input.workflow_state = Some(TicketState::Open); - set_test_ticket_target(&mut input, "test-repository", "HEAD"); - let ticket_ref = backend.create(input).unwrap(); - assign_test_orchestrator(&api, &ticket_ref.id); - browser_ticket_backend(&api) - .unwrap() - .queue_ready(ticket_ref.id.clone().into(), "test-request") - .unwrap(); - - let Json(started) = scoped_start_workspace_orchestrator( - State(api.clone()), - AxumPath(ScopedWorkspacePath { - workspace_id: TEST_WORKSPACE_ID.to_string(), - }), - ) - .await - .unwrap(); - assert!(started.online); - let startup_inputs = execution.take_inputs(); - assert_eq!(startup_inputs.len(), 1); - let worker_id = started.worker.as_ref().unwrap().worker_id.clone(); - let worker = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, &worker_id); - let attention = orchestrator_attention_state(&api, &worker); - assert_eq!( - attention - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .fingerprint, - ticket_ref.id - ); - attention.lock().unwrap().accepted = None; - use protocol::subscription::SubscriptionWorkerState as S; - // Snapshot/reconnection and non-turn transitions cannot stand in for startup. - for previous in [None, Some(S::Idle), Some(S::Stopped), Some(S::Paused)] { - maybe_dispatch_orchestrator_turn_end(&api, &worker_id, previous, S::Idle); - assert!(attention.lock().unwrap().accepted.is_none()); - assert!(execution.take_inputs().is_empty()); - } - maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); - assert_eq!( - attention - .lock() - .unwrap() - .accepted - .as_ref() - .unwrap() - .fingerprint, - ticket_ref.id - ); - maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); - let notifications = execution.take_inputs(); - assert_eq!(notifications.len(), 1); - assert_eq!(notifications[0].0.worker_id.to_string(), worker_id); - let content = ¬ifications[0].1; - assert!(content.starts_with("Queued Tickets require attention:")); - assert!( - content.contains(&format!( - "- {} — Recover queued work", - ticket_ref.resource_key.as_deref().unwrap() - )), - "unexpected notification body: {content:?}" - ); - assert!(content.contains("Reread the current Ticket state before acting")); - assert!(!content.contains(ticket_ref.id.as_str())); - assert!(!content.contains(TEST_WORKSPACE_ID)); - assert!(!content.contains("bounded")); - assert!(!content.contains("omitted")); - - let candidates = backend - .list(ticket::TicketListQuery::states([ - ticket::TicketListState::Open, - ])) - .unwrap(); - let mut truncated_candidates = (1..=worker::OrchestratorQueueAttentionContext::MAX_TICKETS - + 1) - .map(|index| { - let mut candidate = candidates[0].clone(); - candidate.id = format!("opaque-{index}"); - candidate.resource_key = Some(format!("T-{index}")); - candidate.title = format!("Queued {index}"); - candidate - }) - .collect::>(); - let truncated = orchestrator_queue_attention_context( - TEST_WORKSPACE_ID, - TEST_WORKSPACE_ID, - &truncated_candidates, - ) - .unwrap(); - let rendered = worker::PromptCatalog::builtins_only() - .unwrap() - .orchestrator_queue_attention( - worker::OrchestratorQueueAttentionPrompt::Server, - &truncated, - ) - .unwrap(); - assert!(rendered.contains("- T-20 — Queued 20")); - assert!(!rendered.contains("T-21")); - assert!(rendered.contains("were omitted from this notice: 1")); - assert!(!rendered.contains("opaque-")); - - truncated_candidates[0].resource_key = None; - assert_eq!( - orchestrator_queue_attention_context( - TEST_WORKSPACE_ID, - TEST_WORKSPACE_ID, - &truncated_candidates - ) - .unwrap_err(), - "missing_ticket_resource_key" - ); - assert_eq!( - orchestrator_queue_attention_context( - TEST_WORKSPACE_ID, - "foreign-workspace", - &truncated_candidates - ) - .unwrap_err(), - "foreign_workspace_ticket_projection" - ); - } + include!("server_assignment_attention_tests.rs"); #[tokio::test] async fn runtime_worker_create_rejects_reserved_singleton_key_squatting_and_authority_forgery() @@ -52768,6 +52306,7 @@ mod tests { }, profile: server_api::RuntimeProfileSelector::Builtin("builtin:intake".to_string()), ticket_assignment: Some(server_api::RuntimeWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket.id.clone(), operation_id: "runtime-explicit-work".to_string(), }), @@ -52858,6 +52397,7 @@ mod tests { ticket_id ); let mut assignment = Some(CreateWorkspaceWorkerTicketAssignmentRequest { + orchestrator_assignment_id: None, ticket_id: ticket_resource_key.clone(), operation_id: "ticket-key-assignment".to_string(), }); @@ -53408,6 +52948,36 @@ mod tests { assert_eq!(stale.into_response().status(), StatusCode::CONFLICT); } + pub(super) fn seed_test_delivery_binding(api: &WorkspaceApi) { + if api + .config_store + .get_workspace_runtime_binding(TEST_WORKSPACE_ID, EMBEDDED_WORKER_RUNTIME_ID) + .unwrap() + .is_some() + { + return; + } + let identity = (*EMBEDDED_RUNTIME_REQUEST_IDENTITY).clone(); + api.config_store + .upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: TEST_WORKSPACE_ID.into(), + runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), + display_name: "Embedded test Runtime".into(), + base_url: "in-process://embedded".into(), + public_key: identity.public_key, + public_key_fingerprint: String::new(), + binding_id: "binding-test".into(), + authentication_mode: StoredRuntimeAuthenticationMode::LegacyServerIssuer, + created_at: TEST_CREATED_AT.into(), + updated_at: TEST_CREATED_AT.into(), + revoked_at: None, + }, + false, + ) + .unwrap(); + } + async fn test_api_with_recording_backend( workspace_root: impl Into, ) -> (WorkspaceApi, Arc) { diff --git a/crates/workspace-server/src/server/tests/ticket_worker_claim_tests.rs b/crates/workspace-server/src/server/tests/ticket_worker_claim_tests.rs index ae3340877..1e55f7564 100644 --- a/crates/workspace-server/src/server/tests/ticket_worker_claim_tests.rs +++ b/crates/workspace-server/src/server/tests/ticket_worker_claim_tests.rs @@ -448,6 +448,7 @@ async fn pending_manual_worker_claim_reconciles_same_runtime_restore_and_commits .unwrap(); let admitted_content_digest = api.authority.ticket(&ticket_id).unwrap().content_digest; let request = server_api::SetTicketRoleAssignmentRequest { + orchestrator_assignment_id: None, operation_id: "claim-runtime-reconciliation".into(), principal: server_api::TicketAssignmentPrincipal::Worker { runtime_id: worker.runtime_id.clone(), diff --git a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs index 906ca566a..64ca15dc3 100644 --- a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs +++ b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs @@ -1287,23 +1287,33 @@ async fn uncertain_launch_replay_after_target_edit_keeps_claim_and_blocks_compet // Direct policy composition is intentional: this protects routing interest, // independently of the transport/Runtime launch assertions above. #[tokio::test(flavor = "multi_thread")] -async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_just_open_state() { +async fn assignment_inputs_replace_queue_interest_without_reviving_released_work() { let fixture = LaunchFixture::new(false).await; let backend = browser_ticket_backend(&fixture.api).unwrap(); let queued_id = fixture.ticket(); assign_test_orchestrator(&fixture.api, &queued_id); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &queued_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_none() + .iter() + .any(|input| input.ticket_id == queued_id) + == true ); backend .queue_ready(queued_id.clone().into(), "fixture") .unwrap(); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &queued_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_some() + .iter() + .any(|input| input.ticket_id == queued_id) + == true ); fixture.case_action( &queued_id, @@ -1313,9 +1323,14 @@ async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_jus }, ); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &queued_id, "closed") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_none() + .iter() + .any(|input| input.ticket_id == queued_id) + == false ); fixture.case_action( &queued_id, @@ -1323,9 +1338,14 @@ async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_jus ticket::TicketCaseAction::Reopen, ); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &queued_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_none() + .iter() + .any(|input| input.ticket_id == queued_id) + == false ); let active_id = fixture.ticket(); @@ -1347,9 +1367,14 @@ async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_jus .all(|event| !event.attributes.contains_key("queue_root_ticket")) ); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &active_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_some() + .iter() + .any(|input| input.ticket_id == active_id) + == true ); fixture.case_action( &active_id, @@ -1380,8 +1405,13 @@ async fn orchestrator_interest_requires_queue_receipt_or_unfinished_work_not_jus .is_none() ); assert!( - orchestrator_interested(&fixture.api, TEST_WORKSPACE_ID, &active_id, "open") + fixture + .api + .store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) .unwrap() - .is_none() + .iter() + .any(|input| input.ticket_id == active_id) + == false ); } diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs new file mode 100644 index 000000000..d72a8e735 --- /dev/null +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -0,0 +1,1074 @@ +async fn attention_test_api() -> ( + tempfile::TempDir, + WorkspaceApi, + Arc, +) { + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + attention_assign_ticket(&api, "Attention backlog"); + (dir, api, execution) +} + +fn attention_assign_ticket(api: &WorkspaceApi, title: &str) -> ticket::TicketRef { + let mut input = ticket::NewTicket::new(title); + input.workflow_state = Some(TicketState::Open); + set_test_ticket_target(&mut input, "test-repository", "HEAD"); + let ticket = browser_ticket_backend(api).unwrap().create(input).unwrap(); + assign_test_orchestrator(api, &ticket.id); + ticket +} + +fn attention_pending(api: &WorkspaceApi) -> Vec { + api.store + .list_pending_ticket_assignment_deliveries(TEST_WORKSPACE_ID, 100) + .unwrap() +} + +async fn start_attention_test_orchestrator( + api: &WorkspaceApi, +) -> BrowserWorkspaceOrchestratorResponse { + scoped_start_workspace_orchestrator( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + ) + .await + .unwrap() + .0 +} + +#[tokio::test] +async fn orchestrator_attention_parallel_start_and_idle_observation_send_once() { + let (_dir, api, execution) = attention_test_api().await; + let reentrant_api = api.clone(); + execution.before_input_returns(move |worker| { + use protocol::subscription::SubscriptionWorkerState as S; + // The callback is inside Runtime send_input. Neither initial Idle nor + // a real turn end may deadlock or issue another in-flight delivery. + maybe_dispatch_orchestrator_turn_end( + &reentrant_api, + &worker.worker_id.to_string(), + None, + S::Idle, + ); + maybe_dispatch_orchestrator_turn_end( + &reentrant_api, + &worker.worker_id.to_string(), + Some(S::Running), + S::Idle, + ); + }); + let barrier = Arc::new(std::sync::Barrier::new(8)); + let results = std::thread::scope(|scope| { + let joins = (0..8) + .map(|_| { + let api = api.clone(); + let barrier = barrier.clone(); + scope.spawn(move || { + barrier.wait(); + futures::executor::block_on(start_attention_test_orchestrator(&api)) + }) + }) + .collect::>(); + joins + .into_iter() + .map(|j| j.join().unwrap()) + .collect::>() + }); + assert_eq!( + results + .iter() + .filter(|r| r.disposition == "created") + .count(), + 1 + ); + assert_eq!( + results + .iter() + .filter(|r| r.disposition == "existing") + .count(), + 7 + ); + assert_eq!(execution.take_inputs().len(), 1); + assert_eq!(execution.take_input_request_ids().len(), 1); + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + assert!(execution.take_inputs().is_empty()); +} + +#[tokio::test] +async fn orchestrator_attention_barrier_callers_share_unsent_reservation() { + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + state.lock().unwrap().accepted = None; + attention_assign_ticket(&api, "Unreceived concurrent input"); + execution.take_inputs(); + execution.take_input_request_ids(); + let barrier = Arc::new(std::sync::Barrier::new(8)); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + execution.before_input_returns(move |_| { + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + }); + std::thread::scope(|scope| { + let joins = (0..8) + .map(|_| { + let api = api.clone(); + let worker = worker.clone(); + let barrier = barrier.clone(); + scope.spawn(move || { + barrier.wait(); + dispatch_orchestrator_assignment_attention(&api, &worker); + }) + }) + .collect::>(); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert!(state.lock().unwrap().sending); + assert!(state.lock().unwrap().accepted.is_none()); + // The success record does not exist yet. A ninth caller must still + // return immediately rather than compare/send twice or wait on Runtime. + dispatch_orchestrator_assignment_attention(&api, &worker); + release_tx.send(()).unwrap(); + for join in joins { + join.join().unwrap(); + } + }); + assert_eq!(execution.take_inputs().len(), 1); + assert_eq!(execution.take_input_request_ids().len(), 1); + assert!(!state.lock().unwrap().sending); + assert!(state.lock().unwrap().accepted.is_some()); +} + +#[tokio::test] +async fn orchestrator_attention_input_rechecks_reserved_noop_before_and_after_comparison() { + for pause_before_comparison in [true, false] { + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + let acknowledged = state.lock().unwrap().accepted.as_ref().unwrap().clone(); + assert!(attention_pending(&api).is_empty()); + execution.take_inputs(); + execution.take_input_request_ids(); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + execution.accept_restores(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + std::thread::scope(|scope| { + let api_for_caller = api.clone(); + let worker_for_caller = worker.clone(); + let caller = scope.spawn(move || { + let mut checks = 0; + with_orchestrator_attention_reservation( + &api_for_caller, + &worker_for_caller, + true, + |state, pending| { + if checks == 0 && pause_before_comparison { + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + check_orchestrator_assignment_attention( + &api_for_caller, + &worker_for_caller, + state, + pending, + ); + if checks == 0 && !pause_before_comparison { + // Durable acknowledgement matched: check returned without sending, + // but the exact same production reservation still exists. + assert_eq!( + state.lock().unwrap().accepted.as_ref().unwrap().request_id, + acknowledged.request_id + ); + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + checks += 1; + assert!(attention_pending(&api_for_caller).is_empty()); + }, + ); + assert_eq!(checks, 2, "restore must recheck the reserved no-op"); + }); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert!(state.lock().unwrap().sending); + let restored = futures::executor::block_on(start_attention_test_orchestrator(&api)); + assert_eq!(restored.disposition, "restored"); + let deferred = state.lock().unwrap().input_recheck; + let old_retired = state.lock().unwrap().accepted.is_none(); + let inputs_before_release = execution.take_inputs(); + release_tx.send(()).unwrap(); + caller.join().unwrap(); + assert!(deferred); + assert!(old_retired); + assert!(inputs_before_release.is_empty()); + }); + assert!(execution.take_inputs().is_empty()); + assert!(execution.take_input_request_ids().is_empty()); + assert!(attention_pending(&api).is_empty()); + let state = state.lock().unwrap(); + assert!(state.accepted.is_none()); + assert!(!state.sending); + assert!(!state.input_recheck); + } +} + +#[tokio::test] +async fn orchestrator_attention_restore_keeps_pending_unknown_delivery_and_id() { + let (_dir, api, execution) = attention_test_api().await; + execution.reject_inputs("unknown acceptance response"); + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + let pending = state.lock().unwrap().pending.clone().unwrap(); + execution.take_inputs(); + execution.take_input_request_ids(); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + execution.accept_restores(); + *execution.input_failure.lock().unwrap() = None; + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + let attempts = execution.take_inputs(); + assert_eq!(attempts.len(), 1); + assert_eq!(attempts[0].1, pending.content); + assert_eq!( + execution.take_input_request_ids(), + vec![Some(pending.request_id.clone())] + ); + assert!(state.lock().unwrap().pending.is_none()); + assert_eq!( + state.lock().unwrap().accepted.as_ref().unwrap().request_id, + pending.request_id + ); +} + +#[tokio::test] +async fn orchestrator_attention_unknown_response_retry_keeps_id_and_payload() { + let (_dir, api, execution) = attention_test_api().await; + execution.reject_inputs("accepted response lost (unknown transport outcome)"); + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + assert!(state.lock().unwrap().accepted.is_none()); + let pending = state.lock().unwrap().pending.clone().unwrap(); + let attempts = execution.take_inputs(); + let ids = execution.take_input_request_ids(); + assert_eq!(attempts.len(), 2); + assert_eq!(ids, vec![Some(pending.request_id.clone()); 2]); + // Change the backlog before retry: uncertain acceptance must never reuse + // the ID for a newly rendered payload. + attention_assign_ticket(&api, "Changed backlog"); + *execution.input_failure.lock().unwrap() = None; + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + let retry = execution.take_inputs(); + assert_eq!(retry.len(), 2, "acknowledged retry must also drain the changed backlog"); + assert_eq!(retry[0].1, pending.content); + assert!(retry[1].1.contains("Changed backlog")); + let ids = execution.take_input_request_ids(); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], Some(pending.request_id)); + assert_ne!(ids[1], ids[0]); + assert!(state.lock().unwrap().pending.is_none()); + assert!(state.lock().unwrap().accepted.is_some()); + assert!(attention_pending(&api).is_empty()); + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, Some(S::Running), S::Idle); + assert!(execution.take_inputs().is_empty()); +} + +#[tokio::test] +async fn orchestrator_attention_subscription_snapshots_only_establish_baselines() { + use crate::runtime_subscription::BrokerSubscriptionEvent as B; + use protocol::subscription::{ + SubscriptionEventPayload, SubscriptionSnapshot, SubscriptionWorker, + SubscriptionWorkerState as S, + }; + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let attention = orchestrator_attention_state(&api, &worker); + attention.lock().unwrap().accepted = None; + execution.take_inputs(); + attention_assign_ticket(&api, "Unreceived after reconnect"); + let summary = |state: S| -> SubscriptionWorker { + serde_json::from_value(serde_json::json!({ + "worker_id": worker.worker_id, "state": state, + })) + .unwrap() + }; + let snapshot = |state| B::Snapshot { + snapshot: SubscriptionSnapshot::Workers { + workers: vec![summary(state)], + }, + }; + let event = |state| B::Event { + payload: SubscriptionEventPayload::WorkerUpserted { + worker: summary(state), + }, + }; + let mut states = HashMap::new(); + assert!(observe_orchestrator_turn_end( + &api, + &mut states, + snapshot(S::Idle) + )); + assert!(observe_orchestrator_turn_end( + &api, + &mut states, + event(S::Idle) + )); + observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); + // Neither snapshot itself nor an Idle event after reconnect is turn-end evidence. + observe_orchestrator_turn_end( + &api, + &mut states, + B::Disconnected { + message: "fixture reconnect".into(), + }, + ); + observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); + observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); + observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Running)); + observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); + assert!(attention.lock().unwrap().accepted.is_none()); + assert!(execution.take_inputs().is_empty()); + observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); + observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); + assert_eq!(execution.take_inputs().len(), 1); +} + +// Real Controller and canonical Session storage with isolated local config. +// A closed loopback model endpoint prevents any production-provider access; +// notifications commit at the model boundary before that request can fail. +struct AttentionWorkerFactory { + root: PathBuf, +} + +impl AttentionWorkerFactory { + async fn controller( + &self, + worker_id: String, + restore: bool, + ) -> std::result::Result { + let name = format!("attention-{worker_id}"); + let root = self.root.join("workers").join(&worker_id); + let store = session_store::CombinedStore::new( + session_store::WorkerSessionStore::new(root.join("session")) + .map_err(|e| e.to_string())?, + session_store::WorkerAggregateStore::new(&root, &name).map_err(|e| e.to_string())?, + ); + let manifest = manifest::WorkerManifest::from_toml(&format!( + r#" + [worker] + name = "{name}" + pwd = "./" + [model] + scheme = "anthropic" + model_id = "attention-test" + base_url = "http://127.0.0.1:1" + auth = {{ kind = "none" }} + [engine] + max_tokens = 10 + [scope] + allow = [] + "# + )) + .map_err(|e| e.to_string())?; + let context = worker::WorkerWorkspaceContext::local_filesystem(None); + let workspace_client = context.client_handle(); + let loader = worker::PromptCatalogSource::builtins_only(); + let worker = if restore { + worker::Worker::restore_from_worker_metadata_with_context( + &name, + manifest, + store, + loader, + context, + worker::WorkerFilesystemAuthority::None, + ) + .await + } else { + worker::Worker::from_manifest_with_context( + manifest, + store, + loader, + context, + worker::WorkerFilesystemAuthority::None, + ) + .await + } + .map_err(|e| e.to_string())?; + let started = worker::PreparedWorker::new( + worker.with_notification_coalesce_delay(std::time::Duration::from_millis(10)), + worker::WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: root.join(uuid::Uuid::now_v7().to_string()), + bash_output_dir: root.join("bash-output"), + }, + worker::WorkerControllerTransport::InProcess, + ) + .start() + .await + .map_err(|e| e.to_string())?; + Ok(worker_runtime::worker_backend::RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + }) + } +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFactory { + async fn spawn_controller( + &self, + request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> std::result::Result { + self.controller(request.worker_ref.worker_id.to_string(), false) + .await + } + async fn restore_controller( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.controller(request.worker_ref.worker_id.to_string(), true) + .await + } +} + +use worker_runtime::execution as attention_execution; + +// Fault injection occurs AFTER the real Controller durably accepted input. +struct AttentionResponseLossBackend { + inner: Arc, + lose_next: std::sync::atomic::AtomicBool, + attempts: Mutex>, +} + +impl attention_execution::WorkerExecutionBackend for AttentionResponseLossBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + fn spawn_worker( + &self, + r: attention_execution::WorkerExecutionSpawnRequest, + ) -> attention_execution::WorkerExecutionSpawnResult { + self.inner.spawn_worker(r) + } + fn preflight_restore( + &self, + r: &attention_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), attention_execution::WorkerExecutionResult> { + self.inner.preflight_restore(r) + } + fn reconcile_restore( + &self, + r: attention_execution::WorkerExecutionRestoreRequest, + ) -> attention_execution::WorkerExecutionSpawnResult { + self.inner.reconcile_restore(r) + } + fn restore_worker( + &self, + r: attention_execution::WorkerExecutionRestoreRequest, + ) -> attention_execution::WorkerExecutionSpawnResult { + self.inner.restore_worker(r) + } + fn activate_restored_worker( + &self, + id: attention_execution::WorkerLifecycleOperationId, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.activate_restored_worker(id, worker_ref) + } + fn dispatch_input( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + input: worker_runtime::interaction::WorkerInput, + ) -> attention_execution::WorkerExecutionResult { + self.attempts + .lock() + .unwrap() + .push(input.submission_request_id.clone().unwrap()); + let result = self.inner.dispatch_input(worker_ref, input); + assert_eq!( + result.outcome, + attention_execution::WorkerExecutionOutcome::Accepted + ); + if self + .lose_next + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + attention_execution::WorkerExecutionResult::errored( + attention_execution::WorkerExecutionOperation::Input, + "injected loss of accepted response", + ) + } else { + result + } + } + fn dispatch_method( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + method: protocol::Method, + ) -> attention_execution::WorkerExecutionResult { + self.inner.dispatch_method(worker_ref, method) + } + fn stop_worker_operation( + &self, + r: attention_execution::WorkerExecutionStopRequest, + ) -> attention_execution::WorkerExecutionResult { + self.inner.stop_worker_operation(r) + } + fn worker_snapshot( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.worker_snapshot(worker_ref) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn orchestrator_attention_real_worker_restore_keeps_receipt_and_new_input_commits_once() { + real_attention_worker_start_restore(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() { + real_attention_worker_start_restore(true).await; +} + +async fn real_attention_worker_start_restore(lose_response: bool) { + use session_store::{LogEntry, Store, WorkerSessionStore}; + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let config = test_server_config(dir.path()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + // Real Worker Runtime adapter/controller/storage, isolated from profile + // workspace callbacks and real LLM credentials. + let execution = Arc::new( + worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new( + AttentionWorkerFactory { + root: config.embedded_runtime_store_root.clone(), + }, + ) + .unwrap(), + ); + let execution = Arc::new(AttentionResponseLossBackend { + inner: execution, + lose_next: std::sync::atomic::AtomicBool::new(lose_response), + attempts: Mutex::new(Vec::new()), + }); + let api = WorkspaceApi::new_with_execution_backend(config, store, execution.clone()) + .await + .unwrap(); + seed_test_delivery_binding(&api); + let mut input = ticket::NewTicket::new("Real notification backlog"); + input.workflow_state = Some(TicketState::Open); + set_test_ticket_target(&mut input, "test-repository", "HEAD"); + let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); + assign_test_orchestrator(&api, &ticket.id); + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let session = WorkerSessionStore::open_read_only( + api.config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id) + .join("session"), + ) + .unwrap(); + let entries = || { + let id = session.session_id().unwrap().unwrap(); + session + .list_segments(id) + .unwrap() + .into_iter() + .flat_map(|segment| session.read_all_read_only(id, segment).unwrap()) + .collect::>() + }; + let receipts = || { + entries() + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => { + Some(payload["notification_receipts"].as_array().unwrap().clone()) + } + LogEntry::AnnotatedSystemItem { extensions, .. } => extensions + .iter() + .find(|e| e.domain == "worker.pending_activations.v1") + .map(|e| { + e.payload["notification_receipts"] + .as_array() + .unwrap() + .clone() + }), + _ => None, + }) + .unwrap() + }; + let commits = || { + entries().iter().filter(|entry| matches!(entry, + LogEntry::AnnotatedSystemItem { entry, .. } + if matches!(&entry.item, session_store::SystemItem::Notification { message, .. } + if message.starts_with("Assigned Tickets have unreceived input:")) + )).count() + }; + async fn wait_for_commits usize>(count: F, expected: usize) { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let actual = count(); + assert!(actual <= expected, "duplicate canonical notifications"); + if actual == expected { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("controlled Worker notification commit"); + } + assert_eq!(receipts().len(), 1); + wait_for_commits(&commits, 1).await; + assert_eq!( + execution.attempts.lock().unwrap().len(), + if lose_response { 2 } else { 1 } + ); + let first = orchestrator_attention_state(&api, &worker) + .lock() + .unwrap() + .accepted + .clone() + .unwrap(); + let first_id = first.request_id.clone(); + assert_eq!(first.fingerprint, first.input.delivery_id); + assert!(attention_pending(&api).is_empty()); + use protocol::subscription::SubscriptionWorkerState as S; + for previous in [ + None, + Some(S::Idle), + Some(S::Stopped), + Some(S::Paused), + Some(S::Running), + ] { + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, previous, S::Idle); + } + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + assert_eq!(receipts().len(), 1); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + assert!( + orchestrator_attention_state(&api, &worker) + .lock() + .unwrap() + .accepted + .is_none() + ); + assert!(attention_pending(&api).is_empty()); + assert_eq!( + execution.attempts.lock().unwrap().len(), + if lose_response { 2 } else { 1 } + ); + assert_eq!(receipts().len(), 1); + assert_eq!(commits(), 1); + // Retry directly at the real Controller boundary: the same Worker's + // persisted receipt survives restore even after server memory is retired. + let retry = api + .runtime + .send_input( + &worker, + WorkerInputRequest { + kind: WorkerInputKind::Notify, + content: first.content.clone(), + submission_request_id: Some(first_id.clone()), + segments: None, + }, + ) + .unwrap(); + assert_eq!(retry.state, InternalWorkerOperationState::Accepted); + assert_eq!(receipts().len(), 1); + assert_eq!(commits(), 1); + // Only a new external input earns another delivery, not restore itself. + browser_ticket_backend(&api) + .unwrap() + .add_event( + ticket.id.clone().into(), + ticket::NewTicketEvent::new(ticket::TicketEventKind::Comment, "New owner answer"), + ) + .unwrap(); + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, Some(S::Running), S::Idle); + let second = orchestrator_attention_state(&api, &worker) + .lock() + .unwrap() + .accepted + .clone() + .unwrap(); + assert_eq!(first.input.assignment_id, second.input.assignment_id); + assert_ne!(first.input.input_event_id, second.input.input_event_id); + let second_id = second.request_id; + assert_ne!(first_id, second_id); + wait_for_commits(&commits, 2).await; + let attempts = execution.attempts.lock().unwrap(); + assert_eq!(attempts.len(), if lose_response { 4 } else { 3 }); + assert_eq!( + attempts.iter().filter(|id| **id == first_id).count(), + if lose_response { 3 } else { 2 } + ); + assert_eq!(attempts.iter().filter(|id| **id == second_id).count(), 1); + drop(attempts); + let receipts = receipts(); + assert_eq!(receipts.len(), 2); + for id in [first_id, second_id] { + assert_eq!( + receipts + .iter() + .filter(|r| r["notification_request_id"] == id) + .count(), + 1 + ); + } + // Stop only this isolated test fixture; no live dogfooding Worker is touched. + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); +} + +#[tokio::test] +async fn orchestrator_attention_restore_and_replacement_do_not_resend_acknowledged_input() { + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + assert_eq!(execution.take_inputs().len(), 1); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + execution.accept_restores(); + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + assert_eq!(restored.worker.unwrap().worker_id, worker.worker_id); + assert!(execution.take_inputs().is_empty()); + assert!(attention_pending(&api).is_empty()); + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, None, S::Idle); + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + assert!(execution.take_inputs().is_empty()); + // This is a fixture Worker, not the running dogfooding Orchestrator. + api.runtime.delete_worker(&worker).unwrap(); + api.worker_projection + .publish_ordered(|| { + api.store + .delete_worker_registry(TEST_WORKSPACE_ID, &worker) + .map(|commit| ((), commit)) + }) + .unwrap(); + let replacement = start_attention_test_orchestrator(&api).await; + let replacement_id = replacement.worker.unwrap().worker_id; + assert_ne!(replacement_id, worker.worker_id); + assert!(execution.take_inputs().is_empty()); + assert!(attention_pending(&api).is_empty()); + attention_assign_ticket(&api, "New assignment for replacement"); + let replacement = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, replacement_id); + dispatch_orchestrator_assignment_attention(&api, &replacement); + let inputs = execution.take_inputs(); + assert_eq!(inputs.len(), 1); + assert!(inputs[0].1.contains("New assignment for replacement")); +} + +#[tokio::test] +async fn orchestrator_attention_close_reopen_does_not_revive_old_input_but_reassignment_delivers() { + for unknown_response in [false, true] { + let (_dir, api, execution) = attention_test_api().await; + if unknown_response { + execution.reject_inputs("unknown acceptance before closure"); + } + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let attention = orchestrator_attention_state(&api, &worker); + let original = { + let state = attention.lock().unwrap(); + if unknown_response { + state.pending.clone().unwrap() + } else { + state.accepted.clone().unwrap() + } + }; + execution.take_inputs(); + execution.take_input_request_ids(); + *execution.input_failure.lock().unwrap() = None; + let backend = browser_ticket_backend(&api).unwrap(); + for (key, action) in [ + ( + "attention-close", + ticket::TicketCaseAction::Close { + reason: ticket::TicketClosureReason::NotPlanned, + }, + ), + ("attention-reopen", ticket::TicketCaseAction::Reopen), + ] { + let observed = backend + .show(original.input.ticket_id.clone().into()) + .unwrap(); + backend + .update_case( + &original.input.ticket_id, + ticket::TicketCaseUpdate { + expected_content_digest: ticket::ticket_content_digest(&observed), + expected_state: observed.meta.workflow_state, + expected_closure_operation_key: observed.closure_operation_key, + operation_key: key.into(), + action, + explanation: "Human case decision".into(), + references: vec![], + author: None, + }, + ) + .unwrap(); + dispatch_orchestrator_assignment_attention(&api, &worker); + assert!(attention_pending(&api).is_empty()); + assert!( + execution.take_inputs().is_empty(), + "case history must not revive old assignment input" + ); + assert!(attention.lock().unwrap().pending.is_none()); + } + // Even an external comment on the reopened case cannot revive the ended assignment. + backend + .add_event( + original.input.ticket_id.clone().into(), + ticket::NewTicketEvent::new( + ticket::TicketEventKind::Comment, + "Historical result annotation", + ), + ) + .unwrap(); + dispatch_orchestrator_assignment_attention(&api, &worker); + assert!(attention_pending(&api).is_empty()); + assert!(execution.take_inputs().is_empty()); + api.store + .set_current_ticket_role_assignment( + &TicketRoleAssignmentRecord { + workspace_id: TEST_WORKSPACE_ID.into(), + ticket_id: original.input.ticket_id.clone(), + assignment_id: "attention-reassigned".into(), + role: TicketAssignmentRole::Orchestrator, + principal: TicketAssignmentPrincipal::WorkspaceAgent { + agent_key: "workspace-orchestrator".into(), + }, + assigned_by: "test-user".into(), + assigned_at: "2026-09-02T00:00:00Z".into(), + }, + Some(&original.input.assignment_id), + "attention-reassign-event", + "attention-reassign-op", + true, + ) + .unwrap(); + dispatch_orchestrator_assignment_attention(&api, &worker); + assert_eq!( + execution.take_inputs().len(), + 1, + "explicit new assignment must deliver" + ); + let accepted = attention.lock().unwrap().accepted.clone().unwrap(); + assert_ne!(original.input.assignment_id, accepted.input.assignment_id); + assert_ne!(original.request_id, accepted.request_id); + assert!(attention_pending(&api).is_empty()); + } +} + +#[tokio::test] +async fn orchestrator_running_to_idle_recovers_unreceived_input_without_notification_memory() { + let (_dir, api, execution) = attention_test_api().await; + let backend = browser_ticket_backend(&api).unwrap(); + let started = start_attention_test_orchestrator(&api).await; + assert!(started.online); + assert_eq!(execution.take_inputs().len(), 1); + let worker_id = started.worker.unwrap().worker_id; + let worker = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, &worker_id); + let attention = orchestrator_attention_state(&api, &worker); + let acknowledged = attention.lock().unwrap().accepted.clone().unwrap(); + assert_eq!(acknowledged.fingerprint, acknowledged.input.delivery_id); + attention.lock().unwrap().accepted = None; + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); + assert!( + execution.take_inputs().is_empty(), + "lost memory cannot resend durable acknowledgement" + ); + assert!(attention.lock().unwrap().accepted.is_none()); + let ticket_ref = attention_assign_ticket(&api, "Recover unreceived work"); + let pending = attention_pending(&api); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].ticket_id, ticket_ref.id); + // Snapshot/reconnection and non-turn transitions cannot stand in for startup. + for previous in [None, Some(S::Idle), Some(S::Stopped), Some(S::Paused)] { + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, previous, S::Idle); + assert!(attention.lock().unwrap().accepted.is_none()); + assert!(execution.take_inputs().is_empty()); + assert_eq!(attention_pending(&api).len(), 1); + } + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); + let accepted = attention.lock().unwrap().accepted.clone().unwrap(); + assert_eq!(accepted.input.assignment_id, pending[0].assignment_id); + assert_eq!(accepted.input.input_event_id, pending[0].input_event_id); + assert_eq!(accepted.fingerprint, accepted.input.delivery_id); + assert!(attention_pending(&api).is_empty()); + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); + let notifications = execution.take_inputs(); + assert_eq!(notifications.len(), 1); + assert_eq!(notifications[0].0.worker_id.to_string(), worker_id); + let content = ¬ifications[0].1; + assert!(content.starts_with("Assigned Tickets have unreceived input:")); + assert!( + content.contains(&format!( + "- {} — Recover unreceived work", + ticket_ref.resource_key.as_deref().unwrap() + )), + "unexpected notification body: {content:?}" + ); + assert!(content.contains("Reread the current Ticket state before acting")); + assert!(!content.contains(TEST_WORKSPACE_ID)); + assert!(!content.contains("bounded")); + assert!(!content.contains("omitted")); + + let candidates = backend + .list(ticket::TicketListQuery::states([ + ticket::TicketListState::Open, + ])) + .unwrap(); + let mut truncated_candidates = (1..=worker::OrchestratorQueueAttentionContext::MAX_TICKETS + 1) + .map(|index| { + let mut candidate = candidates[0].clone(); + candidate.id = format!("opaque-{index}"); + candidate.resource_key = Some(format!("T-{index}")); + candidate.title = format!("Queued {index}"); + candidate + }) + .collect::>(); + let truncated = orchestrator_queue_attention_context( + TEST_WORKSPACE_ID, + TEST_WORKSPACE_ID, + &truncated_candidates, + ) + .unwrap(); + let rendered = worker::PromptCatalog::builtins_only() + .unwrap() + .orchestrator_queue_attention(worker::OrchestratorQueueAttentionPrompt::Server, &truncated) + .unwrap(); + assert!(rendered.contains("- T-20 — Queued 20")); + assert!(!rendered.contains("T-21")); + assert!(rendered.contains("were omitted from this notice: 1")); + assert!(!rendered.contains("opaque-")); + + truncated_candidates[0].resource_key = None; + assert_eq!( + orchestrator_queue_attention_context( + TEST_WORKSPACE_ID, + TEST_WORKSPACE_ID, + &truncated_candidates + ) + .unwrap_err(), + "missing_ticket_resource_key" + ); + assert_eq!( + orchestrator_queue_attention_context( + TEST_WORKSPACE_ID, + "foreign-workspace", + &truncated_candidates + ) + .unwrap_err(), + "foreign_workspace_ticket_projection" + ); +} diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs new file mode 100644 index 000000000..489674c19 --- /dev/null +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -0,0 +1,1058 @@ +// Included as a child module of server::tests. These tests exercise committed +// Assign + HTTP + embedded Runtime boundaries, not the old Queue predicate. +use super::*; +use session_store::{LogEntry, Store, WorkerSessionStore}; +use worker_runtime::execution as delivery_execution; + +fn delivery_assignment_path(ticket: &str) -> String { + format!("/api/w/{TEST_WORKSPACE_ID}/tickets/{ticket}/assignments/orchestrator") +} + +async fn delivery_http( + api: &WorkspaceApi, + method: &str, + path: &str, + body: Option, + expected: StatusCode, +) -> Value { + let token = format!("api-token-{TEST_WORKSPACE_ID}"); + let response = build_router(api.clone()) + .oneshot( + Request::builder() + .method(method) + .uri(path) + .header("authorization", format!("Bearer {token}")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from(body.map(|v| v.to_string()).unwrap_or_default())) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + status, + expected, + "{method} {path}: {}", + String::from_utf8_lossy(&bytes) + ); + if bytes.is_empty() { + Value::Null + } else { + serde_json::from_slice(&bytes).unwrap() + } +} + +async fn delivery_assign(api: &WorkspaceApi, ticket: &str, key: &str) -> Value { + delivery_http( + api, + "PUT", + &delivery_assignment_path(ticket), + Some(json!({ + "operation_id": key, + "principal": {"kind":"workspace_agent", "agent_key":"workspace-orchestrator"} + })), + StatusCode::OK, + ) + .await +} + +fn delivery_ticket(api: &WorkspaceApi, title: &str) -> ticket::TicketRef { + browser_ticket_backend(api) + .unwrap() + .create(ticket::NewTicket::new(title)) + .unwrap() +} + +fn delivery_tracking( + api: &WorkspaceApi, + ticket: &str, + assignment: &str, +) -> ticket::TicketAssignmentTrackingProjection { + api.store + .get_ticket_assignment_tracking(TEST_WORKSPACE_ID, ticket, assignment) + .unwrap() + .unwrap() +} + +fn delivery_worker(response: BrowserWorkspaceOrchestratorResponse) -> RuntimeWorkerRef { + assert!(response.online); + RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + response.worker.unwrap().worker_id, + ) +} + +fn delivery_scan(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { + dispatch_orchestrator_assignment_attention(api, worker); + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(api, &worker.worker_id, Some(S::Running), S::Idle); +} + +fn delivery_stop(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { + api.runtime + .stop_worker( + worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); +} + +async fn delivery_recording_fixture( + root: &Path, +) -> ( + WorkspaceApi, + Arc, + RuntimeWorkerRef, + RuntimeIdentityMaterial, +) { + let (mut api, execution) = test_api_with_recording_backend(root).await; + seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID); + let worker = delivery_worker(start_attention_test_orchestrator(&api).await); + let identity = RuntimeIdentityMaterial::generate(EMBEDDED_WORKER_RUNTIME_ID).unwrap(); + configure_runtime_request_auth(&mut api, &identity, EMBEDDED_WORKER_RUNTIME_ID); + seed_worker_source_member(&api, &worker.runtime_id, &worker.worker_id); + assert!( + execution.take_inputs().is_empty(), + "open tickets are not implicit delegations" + ); + (api, execution, worker, identity) +} + +async fn delivery_self_report( + api: &WorkspaceApi, + identity: &RuntimeIdentityMaterial, + worker: &RuntimeWorkerRef, + current: &ticket::TicketAssignmentTrackingProjection, + status: &str, + key: &str, +) -> Value { + let path = format!( + "/workspaces/{TEST_WORKSPACE_ID}/tickets/{}/assignment-tracking", + current.ticket_id + ); + let body = serde_json::to_vec(&json!({ + "assignment_id":current.assignment_id, + "expected_last_event_id":current.last_event_id, + "operation_key":key, + "tracking": {"status":status, "waiting_reason":"Need the human's choice", + "progress":"Options investigated", "next_action":"Read the answer then decide", + "result": if status == "done" { Some("Returned options without closing") } else { None }} + })).unwrap(); + let response = build_router(api.clone()) + .oneshot(runtime_source_request( + identity, + Some(&worker.worker_id), + "POST", + &path, + body, + )) + .await + .unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + status, + StatusCode::OK, + "{}", + String::from_utf8_lossy(&bytes) + ); + serde_json::from_slice(&bytes).unwrap() +} + +fn delivery_external_comment(api: &WorkspaceApi, ticket: &str, text: &str) { + browser_ticket_backend(api) + .unwrap() + .add_event( + ticket.to_owned().into(), + ticket::NewTicketEvent::new(ticket::TicketEventKind::Comment, text), + ) + .unwrap(); + notify_ticket_recipients(api, TEST_WORKSPACE_ID, ticket, "open", "open", None); +} + +#[tokio::test] +async fn assignment_waiting_self_report_does_not_loop_but_external_answer_is_delivered() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, identity) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "Choose an option"); + let assigned = delivery_assign(&api, &ticket.id, "assign-choice").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + let initial = delivery_tracking(&api, &ticket.id, assignment); + assert_eq!( + initial.tracking.as_ref().unwrap().status, + ticket::TicketAssignmentTrackingStatus::Waiting + ); + let first = execution.take_inputs(); + assert_eq!(first.len(), 1, "Assign alone delivers without Queue"); + let first_ids = execution.take_input_request_ids(); + assert!(first_ids[0].is_some()); + assert!(first[0].1.contains(ticket.resource_key.as_deref().unwrap())); + let reported = + delivery_self_report(&api, &identity, &worker, &initial, "waiting", "wait-human").await; + // Even an explicit notification of one's own mutation must not manufacture + // fresh work, and repeated idle/startup scans must not use waiting as a queue. + notify_ticket_recipients( + &api, + TEST_WORKSPACE_ID, + &ticket.id, + "open", + "open", + Some(worker.clone()), + ); + for _ in 0..3 { + delivery_scan(&api, &worker); + } + assert!( + execution.take_inputs().is_empty(), + "self progress became an input loop" + ); + let waiting = delivery_tracking(&api, &ticket.id, assignment); + assert_eq!(serde_json::to_value(&waiting).unwrap(), reported); + delivery_external_comment(&api, &ticket.id, "Human answer: choose option B"); + let answer = execution.take_inputs(); + assert_eq!( + answer.len(), + 1, + "external answer must wake the existing delegation" + ); + assert_ne!(execution.take_input_request_ids(), first_ids); + assert!( + answer[0] + .1 + .contains(ticket.resource_key.as_deref().unwrap()) + ); + assert!( + answer[0].1.contains("Need the human's choice"), + "waiting context missing: {:?}", + answer[0].1 + ); + assert!(answer[0].1.contains("Read the answer then decide")); + for _ in 0..3 { + delivery_scan(&api, &worker); + } + assert!( + execution.take_inputs().is_empty(), + "same answer was delivered again" + ); + assert_eq!( + delivery_tracking(&api, &ticket.id, assignment), + waiting, + "delivery/idle must not infer handling inprogress or done" + ); + delivery_stop(&api, &worker); +} + +#[tokio::test] +async fn assignment_done_historical_inputs_do_not_restart_work_but_reissue_is_new_delegation() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, identity) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "Return options while open"); + let assigned = delivery_assign(&api, &ticket.id, "assign-return").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + execution.take_inputs(); + execution.take_input_request_ids(); + let initial = delivery_tracking(&api, &ticket.id, assignment); + delivery_self_report(&api, &identity, &worker, &initial, "done", "return-options").await; + let done = delivery_tracking(&api, &ticket.id, assignment); + assert_eq!( + api.authority.ticket(&ticket.id).unwrap().state.as_str(), + "open" + ); + delivery_external_comment( + &api, + &ticket.id, + "Historical result annotation, not a new request", + ); + for _ in 0..3 { + delivery_scan(&api, &worker); + } + assert!( + execution.take_inputs().is_empty(), + "open + done must not execute history" + ); + assert_eq!(delivery_tracking(&api, &ticket.id, assignment), done); + for (key, action) in [ + ( + "close-return", + json!({"action":"close", "reason":"not_planned"}), + ), + ("reopen-return", json!({"action":"reopen"})), + ] { + let observed = api.authority.ticket(&ticket.id).unwrap(); + delivery_http( + &api, + "POST", + &format!("/workspaces/{TEST_WORKSPACE_ID}/tickets/{}/case", ticket.id), + Some(json!({"expected_content_digest":observed.content_digest, + "expected_state":observed.state, + "expected_closure_operation_key":observed.closure_operation_key, + "operation_key":key, "action":action, "explanation":"Human case decision"})), + StatusCode::OK, + ) + .await; + delivery_scan(&api, &worker); + assert!( + execution.take_inputs().is_empty(), + "case history must not revive done delegation" + ); + assert_eq!(delivery_tracking(&api, &ticket.id, assignment), done); + } + let reassigned = delivery_assign(&api, &ticket.id, "reissue-return").await; + let new_assignment = reassigned["assignment"]["assignment_id"].as_str().unwrap(); + assert_ne!(new_assignment, assignment); + assert_eq!( + execution.take_inputs().len(), + 1, + "explicit reissue must be reconsidered" + ); + let retained = delivery_tracking(&api, &ticket.id, assignment); + assert!(!retained.current); + assert_eq!( + retained.tracking, done.tracking, + "new request overwrote the old result" + ); + assert_eq!( + delivery_tracking(&api, &ticket.id, new_assignment) + .tracking + .unwrap() + .status, + ticket::TicketAssignmentTrackingStatus::Waiting + ); + // An exact old Assign retry returns its receipt, never revives old work. + assert_eq!( + delivery_assign(&api, &ticket.id, "assign-return").await, + assigned + ); + delivery_scan(&api, &worker); + assert!(execution.take_inputs().is_empty()); + delivery_stop(&api, &worker); +} + +#[tokio::test] +async fn assignment_clear_and_reassign_discard_old_unknown_delivery_without_stopping_worker() { + for clear_first in [true, false] { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, _) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "Fence old delivery"); + execution.reject_inputs("unknown acceptance response"); + let assigned = delivery_assign(&api, &ticket.id, "assign-old").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + let old_attempts = execution.take_inputs(); + let old_ids = execution.take_input_request_ids(); + assert!(!old_attempts.is_empty()); + assert!(old_ids.iter().all(|id| id == &old_ids[0])); + if clear_first { + delivery_http( + &api, + "DELETE", + &format!( + "{}?assignment_id={assignment}&operation_id=clear-old", + delivery_assignment_path(&ticket.id) + ), + None, + StatusCode::OK, + ) + .await; + } + *execution.input_failure.lock().unwrap() = None; + // No extra scan, old-operation retry, or synthetic turn end between + // clearing/reissuing the failed input and this committed new Assign. + let reassigned = delivery_assign(&api, &ticket.id, "assign-new").await; + let new_assignment = reassigned["assignment"]["assignment_id"].as_str().unwrap(); + assert_ne!(new_assignment, assignment); + let inputs = execution.take_inputs(); + assert_eq!(inputs.len(), 1, "new Assign must not need another scan"); + assert!(inputs[0].1.contains(new_assignment)); + let ids = execution.take_input_request_ids(); + assert_eq!(ids.len(), 1); + assert_ne!(ids[0], old_ids[0], "new assignment reused stale input identity"); + assert_eq!( + delivery_assign(&api, &ticket.id, "assign-old").await, + assigned + ); + assert!(execution.take_inputs().is_empty(), "old retry revived work"); + let old = delivery_tracking(&api, &ticket.id, assignment); + assert!(!old.current); + assert_eq!( + old.tracking.unwrap().status, + ticket::TicketAssignmentTrackingStatus::Waiting, + "clear/reissue must not rewrite the old status to done" + ); + assert_eq!( + find_workspace_orchestrator(&api).unwrap().worker, + worker, + "clear/reassign are not Worker stop or replacement" + ); + delivery_stop(&api, &worker); + } +} + +#[tokio::test] +async fn assignment_or_answer_committed_during_empty_scan_is_delivered_without_another_trigger() { + for answer in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, identity) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "Input racing an empty scan"); + if answer { + let assigned = delivery_assign(&api, &ticket.id, "initial-assign").await; + let current = delivery_tracking( + &api, + &ticket.id, + assigned["assignment"]["assignment_id"].as_str().unwrap(), + ); + delivery_self_report(&api, &identity, &worker, ¤t, "waiting", "wait-answer").await; + execution.take_inputs(); + execution.take_input_request_ids(); + } + assert!(attention_pending(&api).is_empty()); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + std::thread::scope(|scope| { + let caller_api = api.clone(); + let caller_worker = worker.clone(); + let caller = scope.spawn(move || { + let mut checks = 0; + with_orchestrator_attention_reservation( + &caller_api, + &caller_worker, + true, + |state, pending| { + check_orchestrator_assignment_attention( + &caller_api, &caller_worker, state, pending, + ); + if checks == 0 { + // The production scan has read an empty snapshot but + // still owns its reservation. Race a real HTTP commit. + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); + } + checks += 1; + }, + ); + assert_eq!(checks, 2, "committed input must defer a scan"); + }); + entered_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); + if answer { + futures::executor::block_on(delivery_http( + &api, + "POST", + &format!("/api/w/{TEST_WORKSPACE_ID}/tickets/{}/thread-events", ticket.id), + Some(serde_json::to_value(NewTicketEvent::new( + TicketEventKind::Comment, "Human answer: go ahead", + )).unwrap()), + StatusCode::NO_CONTENT, + )); + } else { + futures::executor::block_on(delivery_assign(&api, &ticket.id, "racing-assign")); + } + // No manual dispatch, Restore, or synthetic Running -> Idle. + assert!(execution.take_inputs().is_empty()); + release_tx.send(()).unwrap(); + caller.join().unwrap(); + }); + let inputs = execution.take_inputs(); + assert_eq!(inputs.len(), 1, "idle Orchestrator was stranded"); + assert!(inputs[0].1.contains(ticket.resource_key.as_deref().unwrap())); + if answer { + assert!(inputs[0].1.contains("Need the human's choice")); + } + assert_eq!(execution.take_input_request_ids().len(), 1); + assert!(attention_pending(&api).is_empty()); + delivery_stop(&api, &worker); + } +} + +#[tokio::test] +async fn assignment_failed_delivery_and_its_reentrant_idle_do_not_schedule_retry_loop() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, worker, _) = delivery_recording_fixture(dir.path()).await; + let ticket = delivery_ticket(&api, "No failed-delivery busy loop"); + execution.reject_inputs("Runtime currently cannot accept notifications"); + let callback_api = api.clone(); + execution.before_input_returns(move |worker| { + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end( + &callback_api, &worker.worker_id.to_string(), Some(S::Running), S::Idle, + ); + }); + delivery_assign(&api, &ticket.id, "assign-failed").await; + assert_eq!(execution.take_inputs().len(), 2, "idle must not add retries beyond the bounded send"); + let ids = execution.take_input_request_ids(); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + assert_eq!(attention_pending(&api).len(), 1, "failure must retain durable input"); + delivery_stop(&api, &worker); +} + +// The model boundary is scripted, but Controller, notification acceptance, +// canonical session commits, Runtime management and HTTP routing are real. +#[derive(Clone, Default)] +struct DeliveryScriptedClient { + requests: Arc>>, +} + +#[async_trait::async_trait] +impl agen::llm_client::LlmClient for DeliveryScriptedClient { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + async fn stream( + &self, + request: agen::llm_client::Request, + ) -> std::result::Result + { + self.requests.lock().unwrap().push(request); + use agen::llm_client::event::Event as LlmEvent; + Ok(Box::pin(futures::stream::iter( + [ + LlmEvent::text_block_start(0), + LlmEvent::text_delta( + 0, + "Read the delegation; returning without starting a Coder.", + ), + LlmEvent::text_block_stop(0, None), + ] + .into_iter() + .map(Ok), + ))) + } +} + +struct DeliveryWorkerFactory { + root: PathBuf, + client: DeliveryScriptedClient, +} + +impl DeliveryWorkerFactory { + async fn controller( + &self, + worker_id: String, + restore: bool, + ) -> std::result::Result { + let name = format!("delivery-{worker_id}"); + let root = self.root.join("workers").join(&worker_id); + let store = session_store::CombinedStore::new( + WorkerSessionStore::new(root.join("session")).map_err(|e| e.to_string())?, + session_store::WorkerAggregateStore::new(&root, &name).map_err(|e| e.to_string())?, + ); + let manifest = manifest::WorkerManifest::from_toml(&format!( + r#" + [worker] + name = "{name}" + pwd = "./" + [model] + scheme = "anthropic" + model_id = "scripted-delivery" + auth = {{ kind = "none" }} + [engine] + max_tokens = 64 + [scope] + allow = [] + "# + )) + .map_err(|e| e.to_string())?; + let context = worker::WorkerWorkspaceContext::local_filesystem(None); + let workspace_client = context.client_handle(); + let bootstrap = worker::WorkerBootstrap::new( + manifest, + store, + worker::PromptCatalogSource::builtins_only(), + context, + worker::WorkerFilesystemAuthority::None, + worker::WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: root.join(uuid::Uuid::now_v7().to_string()), + bash_output_dir: root.join("bash-output"), + }, + worker::WorkerControllerTransport::InProcess, + ) + .with_model_client(self.client.clone()); + let prepared = if restore { + bootstrap.prepare_restored(&name).await + } else { + bootstrap.prepare().await + } + .map_err(|e| e.to_string())?; + let started = prepared.start().await.map_err(|e| e.to_string())?; + Ok(worker_runtime::worker_backend::RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + }) + } +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFactory { + async fn spawn_controller( + &self, + request: delivery_execution::WorkerExecutionSpawnRequest, + ) -> std::result::Result { + self.controller(request.worker_ref.worker_id.to_string(), false) + .await + } + async fn restore_controller( + &self, + request: delivery_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.controller(request.worker_ref.worker_id.to_string(), true) + .await + } +} + +// Lose the response only AFTER the real Worker accepts the exact input. Unlike +// a pre-send rejection, this exercises durable Worker receipt deduplication. +struct DeliveryResponseLossBackend { + inner: Arc, + lose_responses: std::sync::atomic::AtomicBool, + attempts: Mutex>, +} + +impl delivery_execution::WorkerExecutionBackend for DeliveryResponseLossBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + fn spawn_worker( + &self, + r: delivery_execution::WorkerExecutionSpawnRequest, + ) -> delivery_execution::WorkerExecutionSpawnResult { + self.inner.spawn_worker(r) + } + fn preflight_restore( + &self, + r: &delivery_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), delivery_execution::WorkerExecutionResult> { + self.inner.preflight_restore(r) + } + fn restore_worker( + &self, + r: delivery_execution::WorkerExecutionRestoreRequest, + ) -> delivery_execution::WorkerExecutionSpawnResult { + self.inner.restore_worker(r) + } + fn reconcile_restore( + &self, + r: delivery_execution::WorkerExecutionRestoreRequest, + ) -> delivery_execution::WorkerExecutionSpawnResult { + self.inner.reconcile_restore(r) + } + fn activate_restored_worker( + &self, + id: delivery_execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.activate_restored_worker(id, worker) + } + fn dispatch_input( + &self, + worker: &worker_runtime::identity::WorkerRef, + input: worker_runtime::interaction::WorkerInput, + ) -> delivery_execution::WorkerExecutionResult { + self.attempts.lock().unwrap().push(( + input.submission_request_id.clone().unwrap(), + input.content.clone(), + )); + let result = self.inner.dispatch_input(worker, input); + if result.is_accepted() + && self + .lose_responses + .load(std::sync::atomic::Ordering::SeqCst) + { + delivery_execution::WorkerExecutionResult::errored( + delivery_execution::WorkerExecutionOperation::Input, + "accepted response lost", + ) + } else { + result + } + } + fn dispatch_method( + &self, + worker: &worker_runtime::identity::WorkerRef, + method: protocol::Method, + ) -> delivery_execution::WorkerExecutionResult { + self.inner.dispatch_method(worker, method) + } + fn stop_worker_operation( + &self, + r: delivery_execution::WorkerExecutionStopRequest, + ) -> delivery_execution::WorkerExecutionResult { + self.inner.stop_worker_operation(r) + } + fn worker_snapshot( + &self, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.worker_snapshot(worker) + } +} + +async fn delivery_real_fixture( + root: &Path, + lose_responses: bool, +) -> ( + WorkspaceApi, + Arc, + DeliveryScriptedClient, +) { + let config = test_server_config(root); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + let client = DeliveryScriptedClient::default(); + let inner = Arc::new( + worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new(DeliveryWorkerFactory { + root: config.embedded_runtime_store_root.clone(), + client: client.clone(), + }) + .unwrap(), + ); + let execution = Arc::new(DeliveryResponseLossBackend { + inner, + lose_responses: std::sync::atomic::AtomicBool::new(lose_responses), + attempts: Mutex::new(Vec::new()), + }); + let mut api = WorkspaceApi::new_with_execution_backend(config, store, execution.clone()) + .await + .unwrap(); + // The injectable constructor does not install production Runtime trust. + // Delivery resolves the registered singleton plus its active binding. + let identity = RuntimeIdentityMaterial::generate(EMBEDDED_WORKER_RUNTIME_ID).unwrap(); + configure_runtime_request_auth(&mut api, &identity, EMBEDDED_WORKER_RUNTIME_ID); + seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID); + (api, execution, client) +} + +fn delivery_session_entries(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Vec { + let session = WorkerSessionStore::open_read_only( + api.config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id) + .join("session"), + ) + .unwrap(); + let id = session.session_id().unwrap().unwrap(); + session + .list_segments(id) + .unwrap() + .into_iter() + .flat_map(|segment| session.read_all_read_only(id, segment).unwrap()) + .collect() +} + +fn delivery_receipts(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Vec { + delivery_session_entries(api, worker) + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => { + Some(payload["notification_receipts"].as_array().unwrap().clone()) + } + LogEntry::AnnotatedSystemItem { extensions, .. } => extensions + .iter() + .find(|e| e.domain == "worker.pending_activations.v1") + .map(|e| { + e.payload["notification_receipts"] + .as_array() + .unwrap() + .clone() + }), + _ => None, + }) + .unwrap_or_default() +} + +fn delivery_commits(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Vec { + delivery_session_entries(api, worker) + .into_iter() + .filter_map(|entry| match entry { + LogEntry::AnnotatedSystemItem { entry, .. } => match entry.item { + session_store::SystemItem::Notification { message, .. } => Some(message), + _ => None, + }, + _ => None, + }) + .collect() +} + +async fn delivery_wait_idle( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + expected_commits: usize, +) { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let commits = delivery_commits(api, worker).len(); + assert!( + commits <= expected_commits, + "duplicate canonical delivery: {:?}", + delivery_commits(api, worker) + ); + let summary = api.runtime.worker(worker).unwrap(); + if commits == expected_commits + && summary + .worker_state + .as_ref() + .is_some_and(|state| state.state == protocol::WorkerState::Idle) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap_or_else(|_| panic!("scripted Worker did not finish: expected_commits={expected_commits}, commits={:?}, receipts={:?}, worker={:?}", + delivery_commits(api, worker), delivery_receipts(api, worker), api.runtime.worker(worker))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn assignment_without_queue_reaches_real_worker_and_stopped_assign_survives_restore() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, client) = delivery_real_fixture(dir.path(), false).await; + let first = delivery_ticket(&api, "Assign alone is the delegation"); + let assigned = delivery_assign(&api, &first.id, "assign-before-start").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + let initial = delivery_tracking(&api, &first.id, assignment); + assert!( + execution.attempts.lock().unwrap().is_empty(), + "Assign must not auto-restore a stopped Orchestrator" + ); + let worker = delivery_worker(start_attention_test_orchestrator(&api).await); + delivery_wait_idle(&api, &worker, 1).await; + assert_eq!(delivery_receipts(&api, &worker).len(), 1); + assert!(delivery_commits(&api, &worker)[0].contains(first.resource_key.as_deref().unwrap())); + assert_eq!( + client.requests.lock().unwrap().len(), + 1, + "real scripted model did not read the request" + ); + assert_eq!(delivery_tracking(&api, &first.id, assignment), initial); + delivery_stop(&api, &worker); + let second = delivery_ticket(&api, "Assigned while stopped"); + let second_assigned = delivery_assign(&api, &second.id, "assign-while-stopped").await; + let second_id = second_assigned["assignment"]["assignment_id"] + .as_str() + .unwrap(); + let second_initial = delivery_tracking(&api, &second.id, second_id); + assert_eq!(execution.attempts.lock().unwrap().len(), 1); + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + assert_eq!(delivery_worker(restored), worker); + delivery_wait_idle(&api, &worker, 2).await; + assert_eq!(delivery_receipts(&api, &worker).len(), 2); + assert!(delivery_commits(&api, &worker)[1].contains(second.resource_key.as_deref().unwrap())); + assert_eq!( + delivery_tracking(&api, &second.id, second_id), + second_initial + ); + let accepted = execution.attempts.lock().unwrap().clone(); + delivery_scan(&api, &worker); + delivery_stop(&api, &worker); + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "restored" + ); + delivery_scan(&api, &worker); + assert_eq!( + execution.attempts.lock().unwrap().as_slice(), + accepted.as_slice(), + "restoring the same Worker must retain durable acceptance, not generate new work" + ); + assert_eq!(delivery_commits(&api, &worker).len(), 2); + assert_eq!(delivery_receipts(&api, &worker).len(), 2); + delivery_stop(&api, &worker); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commit_once() { + let dir = tempfile::tempdir().unwrap(); + let (mut api, execution, _) = delivery_real_fixture(dir.path(), true).await; + let identity = RuntimeIdentityMaterial::generate(EMBEDDED_WORKER_RUNTIME_ID).unwrap(); + configure_runtime_request_auth(&mut api, &identity, EMBEDDED_WORKER_RUNTIME_ID); + let ticket = delivery_ticket(&api, "Immutable lost-response payload"); + let assigned = delivery_assign(&api, &ticket.id, "assign-lost-response").await; + let assignment = assigned["assignment"]["assignment_id"].as_str().unwrap(); + let initial = delivery_tracking(&api, &ticket.id, assignment); + let worker = delivery_worker(start_attention_test_orchestrator(&api).await); + delivery_wait_idle(&api, &worker, 1).await; + let attempts = execution.attempts.lock().unwrap().clone(); + assert!(!attempts.is_empty()); + assert!( + attempts.iter().all(|attempt| attempt == &attempts[0]), + "uncertain delivery changed request id or payload: {attempts:?}" + ); + assert_eq!(delivery_receipts(&api, &worker).len(), 1); + // Change mutable rendering context while the acknowledgement is unknown. + // A new self-report is not a new input and cannot change the frozen retry. + seed_worker_source_member(&api, &worker.runtime_id, &worker.worker_id); + delivery_self_report( + &api, + &identity, + &worker, + &initial, + "waiting", + "waiting-during-response-loss", + ) + .await; + let waiting = delivery_tracking(&api, &ticket.id, assignment); + delivery_stop(&api, &worker); + execution + .lose_responses + .store(false, std::sync::atomic::Ordering::SeqCst); + browser_ticket_backend(&api) + .unwrap() + .edit_item( + ticket.id.clone().into(), + ticket::TicketItemEdit { + title: Some("Title changed while acceptance was unknown".into()), + ..Default::default() + }, + ) + .unwrap(); + // Reopen while acceptance is still unknown. This intentionally discards the + // old API's cached pending text, but not the durable input/Worker receipts. + let config = api.config.clone(); + drop(api); + tokio::task::yield_now().await; + let reopened = WorkspaceApi::new_with_execution_backend( + config.clone(), + Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()), + execution.clone(), + ) + .await + .unwrap(); + let restored = start_attention_test_orchestrator(&reopened).await; + assert_eq!(restored.disposition, "restored"); + assert_eq!(delivery_worker(restored), worker); + { + let after_restart = execution.attempts.lock().unwrap(); + let same_request = after_restart + .iter() + .filter(|attempt| attempt.0 == attempts[0].0) + .collect::>(); + assert!( + same_request.len() > attempts.len(), + "unknown outcome was not retried after restore" + ); + assert!( + same_request.iter().all(|attempt| *attempt == &attempts[0]), + "same request id changed payload after DB reopen/title/self-report change: {after_restart:?}" + ); + } + delivery_wait_idle(&reopened, &worker, 2).await; + assert_eq!(delivery_receipts(&reopened, &worker).len(), 2); + assert_eq!( + delivery_commits(&reopened, &worker) + .iter() + .filter(|message| *message == &attempts[0].1) + .count(), + 1, + "accepted retry committed the old notification twice" + ); + assert_eq!( + delivery_tracking(&reopened, &ticket.id, assignment), + waiting + ); + let before = execution.attempts.lock().unwrap().len(); + browser_ticket_backend(&reopened) + .unwrap() + .add_event( + ticket.id.clone().into(), + ticket::NewTicketEvent::new( + ticket::TicketEventKind::Comment, + "External answer after server restart", + ), + ) + .unwrap(); + let barrier = Arc::new(std::sync::Barrier::new(8)); + std::thread::scope(|scope| { + let joins = (0..8) + .map(|_| { + let api = reopened.clone(); + let worker = worker.clone(); + let barrier = barrier.clone(); + scope.spawn(move || { + barrier.wait(); + dispatch_orchestrator_assignment_attention(&api, &worker); + }) + }) + .collect::>(); + for join in joins { + join.join().unwrap(); + } + }); + delivery_wait_idle(&reopened, &worker, 3).await; + assert_eq!( + execution.attempts.lock().unwrap().len(), + before + 1, + "concurrent scans must share the committed reservation" + ); + assert_eq!(delivery_receipts(&reopened, &worker).len(), 3); + delivery_scan(&reopened, &worker); + assert_eq!(execution.attempts.lock().unwrap().len(), before + 1); + assert_eq!( + delivery_tracking(&reopened, &ticket.id, assignment), + waiting + ); + delivery_stop(&reopened, &worker); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn assignment_worker_replacement_receives_pending_input_without_recreating_delegation() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution, _) = delivery_real_fixture(dir.path(), false).await; + let handled = delivery_ticket(&api, "Already received before replacement"); + let saved = delivery_assign(&api, &handled.id, "assign-before-replacement").await; + let assignment = saved["assignment"]["assignment_id"].as_str().unwrap(); + let initial = delivery_tracking(&api, &handled.id, assignment); + let old_worker = delivery_worker(start_attention_test_orchestrator(&api).await); + delivery_wait_idle(&api, &old_worker, 1).await; + delivery_stop(&api, &old_worker); + // Delete only this isolated fixture's stopped identity. A new live Worker + // has a new session; it is not a restore and cannot reuse old session receipts. + api.runtime.delete_worker(&old_worker).unwrap(); + api.worker_projection + .publish_ordered(|| { + api.store + .delete_worker_registry(TEST_WORKSPACE_ID, &old_worker) + .map(|commit| ((), commit)) + }) + .unwrap(); + let pending = delivery_ticket(&api, "Pending input for replacement"); + let new_assign = delivery_assign(&api, &pending.id, "assign-pending-replacement").await; + let pending_assignment = new_assign["assignment"]["assignment_id"].as_str().unwrap(); + let pending_initial = delivery_tracking(&api, &pending.id, pending_assignment); + let started = start_attention_test_orchestrator(&api).await; + assert_eq!(started.disposition, "created"); + let replacement = delivery_worker(started); + assert_ne!(replacement, old_worker); + delivery_wait_idle(&api, &replacement, 1).await; + assert_eq!(delivery_receipts(&api, &replacement).len(), 1); + let commits = delivery_commits(&api, &replacement); + assert!(commits[0].contains(pending.resource_key.as_deref().unwrap())); + assert!( + !commits[0].contains(handled.resource_key.as_deref().unwrap()), + "replacement must not execute acknowledged history" + ); + assert_eq!(execution.attempts.lock().unwrap().len(), 2); + assert_eq!( + delivery_tracking(&api, &handled.id, assignment), + initial, + "Worker replacement must not reassign the stable principal" + ); + assert_eq!( + delivery_tracking(&api, &pending.id, pending_assignment), + pending_initial + ); + delivery_scan(&api, &replacement); + assert_eq!(execution.attempts.lock().unwrap().len(), 2); + delivery_stop(&api, &replacement); +} diff --git a/crates/workspace-server/src/server_assignment_start_tests.rs b/crates/workspace-server/src/server_assignment_start_tests.rs new file mode 100644 index 000000000..11e0cdee6 --- /dev/null +++ b/crates/workspace-server/src/server_assignment_start_tests.rs @@ -0,0 +1,752 @@ +// Include from server.rs's `tests` module as a child module: +// mod assignment_start_tests { include!("server_assignment_start_tests.rs"); } +// These exercise the authenticated router with the existing in-process Runtime +// adapter; they do not start product processes or claim model/E2E coverage. +use super::*; + +struct StartSource { + identity: RuntimeIdentityMaterial, + worker: RuntimeWorkerRef, +} + +async fn start_source(api: &mut WorkspaceApi, runtime: &WorkdirlessFixtureRuntime) -> StartSource { + let runtime_id = WorkdirlessFixtureRuntime::RUNTIME_ID; + let identity = RuntimeIdentityMaterial::generate(runtime_id).unwrap(); + configure_runtime_request_auth(api, &identity, runtime_id); + let memory = api + .config_store + .get_workspace_memory_settings(TEST_WORKSPACE_ID) + .unwrap(); + let reservation = api + .config_store + .reserve_worker_create( + TEST_WORKSPACE_ID, + runtime_id, + "start-orchestrator", + "sha256:start-orchestrator", + Some(crate::hosts::WORKSPACE_ORCHESTRATOR_SINGLETON_KEY), + &memory, + ) + .unwrap(); + let mut summary = WorkdirlessFixtureRuntime::worker_summary( + WorkerCreateBinding { + worker_id: reservation.worker_id, + create_fingerprint: reservation.create_fingerprint, + }, + &WorkerSpawnRequest { + requested_worker_name: Some("Start Orchestrator".into()), + singleton_key: Some(crate::hosts::WORKSPACE_ORCHESTRATOR_SINGLETON_KEY.into()), + intent: WorkerSpawnIntent::WorkspaceCompanion, + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 0, + }, + profile: ProfileSelector::Builtin("builtin:orchestrator".into()), + ticket_assignment: None, + initial_submit: Vec::new(), + workdir_attachment_requests: Vec::new(), + resolved_workdir_attachment_requests: Vec::new(), + resolved_workdir_attachments: Vec::new(), + resolved_config_bundle: None, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_control_operation: None, + }, + ); + summary.tags = vec!["orchestrator".into(), "singleton".into()]; + let worker = summary.worker.clone(); + runtime.workers.lock().unwrap().push(summary); + seed_worker_source_member(api, runtime_id, &worker.worker_id); + api.config_store + .complete_worker_create_reservation(TEST_WORKSPACE_ID, worker.worker_id.parse().unwrap()) + .unwrap(); + assert_eq!(find_workspace_orchestrator(api).unwrap().worker, worker); + StartSource { identity, worker } +} + +fn add_unfinished_start_dependency(api: &WorkspaceApi, ticket_id: &str) -> String { + let backend = browser_ticket_backend(api).unwrap(); + let dependency = backend + .create(ticket::NewTicket::new("Unfinished prerequisite")) + .unwrap(); + backend + .add_ticket_relation( + ticket_id.into(), + ticket::NewTicketRelation { + kind: ticket::TicketRelationKind::DependsOn, + target: dependency.id.clone(), + note: Some("Partial work is intentional".into()), + author: Some("owner".into()), + }, + ) + .unwrap(); + dependency.id +} + +fn start_http_request( + source: Option<&StartSource>, + token: &str, + method: &str, + path: &str, + body: Value, +) -> Request { + let body = serde_json::to_vec(&body).unwrap(); + match source { + Some(source) => runtime_source_request( + &source.identity, + Some(&source.worker.worker_id), + method, + path, + body, + ), + None => Request::builder() + .method(method) + .uri(path) + .header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from(body)) + .unwrap(), + } +} + +async fn start_http_response( + api: &WorkspaceApi, + request: Request, + expected: StatusCode, +) -> Value { + let response = build_router(api.clone()).oneshot(request).await.unwrap(); + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&body)); + serde_json::from_slice(&body) + .unwrap_or_else(|_| serde_json::json!({"message": String::from_utf8_lossy(&body)})) +} + +fn observed_start_delegation( + api: &WorkspaceApi, + ticket_id: &str, + source: Option<&StartSource>, +) -> Option { + source.map(|_| { + api.store + .get_current_ticket_role_assignment( + TEST_WORKSPACE_ID, + ticket_id, + TicketAssignmentRole::Orchestrator, + ) + .unwrap() + .unwrap() + .assignment_id + }) +} + +fn start_worker_body( + ticket_id: Option<&str>, + attachments: Value, + delegation: Option<&str>, +) -> Value { + let mut body = serde_json::json!({ + "runtime_id": WorkdirlessFixtureRuntime::RUNTIME_ID, + "display_name": "Partial investigation", + "profile": "builtin:intake", + "initial_submit": [], + "workdir_attachments": attachments, + "feature_connections": {}, + "control_operation_id": "partial-control-start" + }); + if let Some(ticket_id) = ticket_id { + body["ticket_assignment"] = serde_json::json!({ + "ticket_id": ticket_id, + "operation_id": "partial-start" + }); + if let Some(delegation) = delegation { + body["ticket_assignment"]["orchestrator_assignment_id"] = + Value::String(delegation.into()); + } + } + body +} + +fn assert_start_dependency_diagnostic_only(api: &WorkspaceApi, ticket: &str, dependency: &str) { + let Json(detail) = browser_ticket_detail(api, ticket).unwrap(); + assert_eq!(detail.relations.blockers.len(), 1); + assert_eq!(detail.relations.blockers[0].blocking_ticket, dependency); + assert!(!detail.action_eligibility.can_start_manual_worker); + assert!(detail.current_worker.is_some()); + let Json(prerequisite) = browser_ticket_detail(api, dependency).unwrap(); + assert_eq!(prerequisite.state.as_str(), "open"); + assert!(prerequisite.assignments.is_empty()); + assert!(prerequisite.assignment_tracking.is_empty()); +} + +#[tokio::test] +async fn unfinished_dependencies_allow_manual_and_orchestrator_resource_free_start_and_assign() { + for orchestrator in [false, true] { + for existing in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let mut api = test_api(directory.path()).await; + register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; + let runtime = WorkdirlessFixtureRuntime::default(); + api.runtime.register_or_replace(runtime.clone()); + let source = if orchestrator { + Some(start_source(&mut api, &runtime).await) + } else { + None + }; + let token = seed_test_api_token(api.store.as_ref(), "partial-start-owner"); + let ticket = browser_ticket_backend(&api) + .unwrap() + .create(ticket::NewTicket::new( + "Partial resource-free investigation", + )) + .unwrap(); + let dependency = add_unfinished_start_dependency(&api, &ticket.id); + if orchestrator { + assign_test_orchestrator(&api, &ticket.id); + } + let delegation = observed_start_delegation(&api, &ticket.id, source.as_ref()); + assert!( + browser_ticket_detail(&api, &ticket.id) + .unwrap() + .0 + .action_eligibility + .can_start_manual_worker + ); + let create_path = if orchestrator { + format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers") + } else { + format!("/api/w/{TEST_WORKSPACE_ID}/workers") + }; + let created = start_http_response( + &api, + start_http_request( + source.as_ref(), + &token, + "POST", + &create_path, + start_worker_body( + (!existing).then_some(ticket.id.as_str()), + serde_json::json!([]), + delegation.as_deref(), + ), + ), + StatusCode::OK, + ) + .await; + let worker_id = created["worker_id"].as_str().unwrap(); + assert_eq!(created["worker"]["profile"], "builtin:intake"); + if existing { + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + ticket.id + ); + let mut body = serde_json::json!({ + "operation_id": "partial-existing-assign", + "principal": { + "kind": "worker", + "runtime_id": WorkdirlessFixtureRuntime::RUNTIME_ID, + "worker_id": worker_id + }, + "workdir_bindings": [] + }); + if let Some(delegation) = &delegation { + body["orchestrator_assignment_id"] = Value::String(delegation.clone()); + } + let assigned = start_http_response( + &api, + start_http_request(source.as_ref(), &token, "PUT", &path, body.clone()), + StatusCode::OK, + ) + .await; + let replay = start_http_response( + &api, + start_http_request(source.as_ref(), &token, "PUT", &path, body), + StatusCode::OK, + ) + .await; + assert_eq!(assigned, replay); + } + let worker = RuntimeWorkerRef::new(WorkdirlessFixtureRuntime::RUNTIME_ID, worker_id); + assert!( + api.store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &worker) + .unwrap() + .is_empty() + ); + assert_start_dependency_diagnostic_only(&api, &ticket.id, &dependency); + // Partial work never permits a second unfinished Worker. + let mut competing = start_worker_body( + Some(&ticket.id), + serde_json::json!([]), + delegation.as_deref(), + ); + competing["ticket_assignment"]["operation_id"] = + Value::String("competing-start".into()); + competing["control_operation_id"] = Value::String("competing-control-start".into()); + let spawn_count = runtime.spawn_requests().len(); + start_http_response( + &api, + start_http_request(source.as_ref(), &token, "POST", &create_path, competing), + StatusCode::CONFLICT, + ) + .await; + assert_eq!(runtime.spawn_requests().len(), spawn_count); + if !orchestrator { + assert!( + api.store + .get_current_ticket_role_assignment( + TEST_WORKSPACE_ID, + &ticket.id, + TicketAssignmentRole::Orchestrator + ) + .unwrap() + .is_none() + ); + } + } + } +} + +#[tokio::test] +async fn unfinished_dependencies_allow_read_only_start_and_assign_without_write_or_command_grants() +{ + for orchestrator in [false, true] { + for existing in [false, true] { + let mut fixture = manual_worker_assignment_fixture().await; + let source = if orchestrator { + Some(start_source(&mut fixture.api, &fixture.runtime).await) + } else { + None + }; + let api = &fixture.api; + // Spawn finalization observes Runtime Workdir identity; do not use + // the adapter's unrelated default Repository id for this checkout. + fixture.runtime.set_workdir_repository( + &fixture.docs_workdir_id, + &test_repository_id_by_key(api, "docs"), + ); + let backend = browser_ticket_backend(api).unwrap(); + // Keep only the reference target. Partial investigation needs no writable Repository. + backend + .edit_item( + fixture.ticket_id.clone().into(), + ticket::TicketItemEdit { + targets: Some(ticket::TicketTargetsEdit::Set { + targets: vec![test_ticket_target( + "docs", + "develop", + TicketTargetAccess::ReadOnly, + )], + }), + ..Default::default() + }, + ) + .unwrap(); + let dependency = add_unfinished_start_dependency(api, &fixture.ticket_id); + if orchestrator { + assign_test_orchestrator(api, &fixture.ticket_id); + } + let delegation = observed_start_delegation(api, &fixture.ticket_id, source.as_ref()); + let token = seed_test_api_token(api.store.as_ref(), "read-only-start-owner"); + let worker = if existing { + // Drop the unrelated checkout so the entire attachment set is reference-only. + let main = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .into_iter() + .find(|link| link.workdir_id == fixture.main_workdir_id) + .unwrap(); + api.store + .detach_worker_workdir_connection( + TEST_WORKSPACE_ID, + &fixture.worker, + &main.alias, + &main.connection_id, + &now_registry_timestamp(), + ) + .unwrap(); + sync_runtime_worker_workdir_attachments(api, &fixture.worker).unwrap(); + let mut body = serde_json::to_value(manual_worker_assignment_request( + &fixture, + "read-only-existing", + )) + .unwrap(); + if let Some(delegation) = &delegation { + body["orchestrator_assignment_id"] = Value::String(delegation.clone()); + } + body["workdir_bindings"] = serde_json::json!([{ + "alias": "docs", "working_directory_id": fixture.docs_workdir_id, + "connection_id": api.store.list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker).unwrap().into_iter().find(|link| link.workdir_id == fixture.docs_workdir_id && link.unlinked_at.is_none()).unwrap().connection_id + }]); + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + fixture.ticket_id + ); + start_http_response( + api, + start_http_request(source.as_ref(), &token, "PUT", &path, body), + StatusCode::OK, + ) + .await; + fixture.worker.clone() + } else { + // This matrix tests read-only start, not sharing an occupied + // Workdir. Release the fixture candidate's reference attachment. + let docs = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .into_iter() + .find(|link| { + link.workdir_id == fixture.docs_workdir_id && link.unlinked_at.is_none() + }) + .unwrap(); + api.store + .detach_worker_workdir_connection( + TEST_WORKSPACE_ID, + &fixture.worker, + &docs.alias, + &docs.connection_id, + &now_registry_timestamp(), + ) + .unwrap(); + sync_runtime_worker_workdir_attachments(api, &fixture.worker).unwrap(); + let path = if orchestrator { + format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers") + } else { + format!("/api/w/{TEST_WORKSPACE_ID}/workers") + }; + let body = start_worker_body( + Some(&fixture.ticket_id), + serde_json::json!([{ + "alias": "docs", "working_directory_id": fixture.docs_workdir_id + }]), + delegation.as_deref(), + ); + let created = start_http_response( + api, + start_http_request(source.as_ref(), &token, "POST", &path, body), + StatusCode::OK, + ) + .await; + assert_eq!(created["worker"]["profile"], "builtin:intake"); + RuntimeWorkerRef::new( + WorkdirlessFixtureRuntime::RUNTIME_ID, + created["worker_id"].as_str().unwrap(), + ) + }; + let links = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &worker) + .unwrap() + .into_iter() + .filter(|link| link.unlinked_at.is_none()) + .collect::>(); + assert_eq!(links.len(), 1); + assert_eq!(links[0].workdir_id, fixture.docs_workdir_id); + assert_eq!( + links[0].capabilities, + workdir::WorkdirSessionCapabilities::READ_ONLY + ); + if existing { + let attachments = fixture.runtime.logical_attachments(&worker.worker_id); + assert_eq!(attachments.len(), 1); + assert_eq!( + attachments[0].capabilities, + workdir::WorkdirSessionCapabilities::READ_ONLY + ); + } else { + // This adapter records spawn inputs separately from the + // replacement-attachment calls used by existing-Worker assign. + let request = fixture.runtime.spawn_requests().last().unwrap().clone(); + assert_eq!(request.resolved_workdir_attachments.len(), 1); + assert_eq!( + request.resolved_workdir_attachments[0].working_directory_id, + fixture.docs_workdir_id + ); + assert_eq!( + request.resolved_workdir_attachments[0].capabilities, + workdir::WorkdirSessionCapabilities::READ_ONLY + ); + } + assert_start_dependency_diagnostic_only(api, &fixture.ticket_id, &dependency); + } + } +} + +#[tokio::test] +async fn unfinished_dependencies_do_not_bypass_http_auth_closed_or_resource_binding_guards() { + let fixture = manual_worker_assignment_fixture().await; + let api = &fixture.api; + add_unfinished_start_dependency(api, &fixture.ticket_id); + let token = seed_test_api_token(api.store.as_ref(), "guard-start-owner"); + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + fixture.ticket_id + ); + let valid = + serde_json::to_value(manual_worker_assignment_request(&fixture, "guarded-assign")).unwrap(); + let denied = start_http_response( + api, + start_http_request(None, "invalid-token", "PUT", &path, valid.clone()), + StatusCode::UNAUTHORIZED, + ) + .await; + assert!(denied["message"].is_string()); + let mut stale = valid.clone(); + stale["workdir_bindings"][0]["connection_id"] = Value::String("stale-connection".into()); + let denied = start_http_response( + api, + start_http_request(None, &token, "PUT", &path, stale), + StatusCode::CONFLICT, + ) + .await; + assert!(denied["message"].as_str().unwrap().contains("connection")); + assert!( + api.store + .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &fixture.ticket_id) + .unwrap() + .is_none() + ); + // The existing case/work-release trigger is separate from relation diagnostics. + rusqlite::Connection::open(&api.config.database_path).unwrap().execute( + "UPDATE typed_tickets SET workflow_state='closed' WHERE workspace_id=?1 AND ticket_id=?2", + rusqlite::params![TEST_WORKSPACE_ID, fixture.ticket_id], + ).unwrap(); + let denied = start_http_response( + api, + start_http_request(None, &token, "PUT", &path, valid), + StatusCode::CONFLICT, + ) + .await; + assert!( + denied["message"].as_str().unwrap().contains("open") + || denied["message"].as_str().unwrap().contains("closed") + ); + assert!( + api.store + .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &fixture.ticket_id) + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn orchestrator_start_and_assign_reject_missing_stale_cleared_or_reissued_delegation() { + for existing in [false, true] { + for invalidation in ["missing", "stale", "clear", "reissue"] { + let directory = tempfile::tempdir().unwrap(); + let mut api = test_api(directory.path()).await; + register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; + let runtime = WorkdirlessFixtureRuntime::default(); + api.runtime.register_or_replace(runtime.clone()); + let source = start_source(&mut api, &runtime).await; + let token = seed_test_api_token(api.store.as_ref(), "delegation-owner"); + let ticket = browser_ticket_backend(&api) + .unwrap() + .create(ticket::NewTicket::new("Observed delegation start")) + .unwrap(); + assign_test_orchestrator(&api, &ticket.id); + let observed = observed_start_delegation(&api, &ticket.id, Some(&source)).unwrap(); + let worker = if existing { + Some( + start_http_response( + &api, + start_http_request( + None, + &token, + "POST", + &format!("/api/w/{TEST_WORKSPACE_ID}/workers"), + start_worker_body(None, serde_json::json!([]), None), + ), + StatusCode::OK, + ) + .await, + ) + } else { + None + }; + if matches!(invalidation, "clear" | "reissue") { + let clear = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/orchestrator?assignment_id={observed}&operation_id=clear-observed", + ticket.id + ); + start_http_response( + &api, + start_http_request(None, &token, "DELETE", &clear, Value::Null), + StatusCode::OK, + ) + .await; + if invalidation == "reissue" { + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/orchestrator", + ticket.id + ); + start_http_response(&api, start_http_request(None, &token, "PUT", &path, + serde_json::json!({"operation_id":"reissue-observed", "principal":{"kind":"workspace_agent", "agent_key":"workspace-orchestrator"}})), StatusCode::OK).await; + assert_ne!( + observed_start_delegation(&api, &ticket.id, Some(&source)).unwrap(), + observed + ); + } + } + let submitted = match invalidation { + "missing" => None, + "stale" => Some("unrelated-assignment"), + _ => Some(observed.as_str()), + }; + let (method, path, body) = if let Some(worker) = worker { + let mut body = serde_json::json!({ + "operation_id":"invalid-existing-start", + "principal":{"kind":"worker", "runtime_id":WorkdirlessFixtureRuntime::RUNTIME_ID, "worker_id":worker["worker_id"]}, + "workdir_bindings":[] + }); + if let Some(submitted) = submitted { + body["orchestrator_assignment_id"] = Value::String(submitted.into()); + } + ( + "PUT", + format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + ticket.id + ), + body, + ) + } else { + ( + "POST", + format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers"), + start_worker_body(Some(&ticket.id), serde_json::json!([]), submitted), + ) + }; + let spawns = runtime.spawn_requests().len(); + let denied = start_http_response( + &api, + start_http_request(Some(&source), &token, method, &path, body), + StatusCode::CONFLICT, + ) + .await; + assert!( + denied["message"].as_str().unwrap().contains("assignment") + || denied["message"].as_str().unwrap().contains("delegation"), + "{invalidation}: {denied}" + ); + assert_eq!(runtime.spawn_requests().len(), spawns); + assert!( + api.store + .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &ticket.id) + .unwrap() + .is_none() + ); + let operation = if existing { + "invalid-existing-start" + } else { + "partial-start" + }; + assert!( + api.store + .get_ticket_assignment_operation(TEST_WORKSPACE_ID, operation) + .unwrap() + .is_none() + ); + } + } +} + +#[tokio::test] +async fn replaced_orchestrator_cannot_start_or_assign_under_retained_current_delegation() { + for existing in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let mut api = test_api(directory.path()).await; + register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; + let runtime = WorkdirlessFixtureRuntime::default(); + api.runtime.register_or_replace(runtime.clone()); + let source = start_source(&mut api, &runtime).await; + let token = seed_test_api_token(api.store.as_ref(), "obsolete-worker-owner"); + let ticket = browser_ticket_backend(&api) + .unwrap() + .create(ticket::NewTicket::new( + "Retained delegation after Worker replacement", + )) + .unwrap(); + assign_test_orchestrator(&api, &ticket.id); + let delegation = observed_start_delegation(&api, &ticket.id, Some(&source)).unwrap(); + let worker = if existing { + Some( + start_http_response( + &api, + start_http_request( + None, + &token, + "POST", + &format!("/api/w/{TEST_WORKSPACE_ID}/workers"), + start_worker_body(None, serde_json::json!([]), None), + ), + StatusCode::OK, + ) + .await, + ) + } else { + None + }; + // Backend ownership changes independently of the retained assignment row. + let replacement = + RuntimeWorkerRef::new(&source.worker.runtime_id, WorkerId::now_v7().to_string()); + seed_worker_source_member(&api, &replacement.runtime_id, &replacement.worker_id); + let mut summary = runtime.workers.lock().unwrap()[0].clone(); + summary.worker = replacement.clone(); + runtime.workers.lock().unwrap().push(summary); + rusqlite::Connection::open(&api.config.database_path).unwrap().execute( + "UPDATE worker_singleton_owners SET worker_id=?1,generation=generation+1 WHERE workspace_id=?2 AND singleton_key=?3", + rusqlite::params![replacement.worker_id, TEST_WORKSPACE_ID, crate::hosts::WORKSPACE_ORCHESTRATOR_SINGLETON_KEY], + ).unwrap(); + assert_eq!( + find_workspace_orchestrator(&api).unwrap().worker, + replacement + ); + assert_eq!( + observed_start_delegation(&api, &ticket.id, Some(&source)).unwrap(), + delegation + ); + let (method, path, body) = if let Some(worker) = worker { + ( + "PUT", + format!( + "/api/w/{TEST_WORKSPACE_ID}/tickets/{}/assignments/worker", + ticket.id + ), + serde_json::json!({ + "operation_id":"obsolete-existing-start", "orchestrator_assignment_id":delegation, + "principal":{"kind":"worker", "runtime_id":WorkdirlessFixtureRuntime::RUNTIME_ID, "worker_id":worker["worker_id"]}, + "workdir_bindings":[] + }), + ) + } else { + ( + "POST", + format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers"), + start_worker_body(Some(&ticket.id), serde_json::json!([]), Some(&delegation)), + ) + }; + let spawns = runtime.spawn_requests().len(); + let denied = start_http_response( + &api, + start_http_request(Some(&source), &token, method, &path, body), + // The authenticated Worker-source middleware fences the replaced + // singleton before the start/assign handler is admitted. + StatusCode::UNAUTHORIZED, + ) + .await; + assert!(denied["message"].is_string()); + assert_eq!(runtime.spawn_requests().len(), spawns); + assert!( + api.store + .get_active_ticket_worker_assignment(TEST_WORKSPACE_ID, &ticket.id) + .unwrap() + .is_none() + ); + } +} diff --git a/crates/workspace-server/src/server_assignment_tracking_tests.rs b/crates/workspace-server/src/server_assignment_tracking_tests.rs index 138fae02b..7ad464549 100644 --- a/crates/workspace-server/src/server_assignment_tracking_tests.rs +++ b/crates/workspace-server/src/server_assignment_tracking_tests.rs @@ -68,7 +68,9 @@ async fn runtime_and_account_assign_retries_return_saved_result_after_clear_and_ })).unwrap(); tracking_response( app.clone(), - request("PUT", &worker_path, worker_input), + Request::builder().method("PUT").uri(&worker_path).header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from(worker_input)).unwrap(), StatusCode::OK, ) .await; @@ -117,15 +119,15 @@ async fn runtime_and_account_assign_retries_return_saved_result_after_clear_and_ .await, saved ); - // Only a genuinely new operation is refused; a receipt never restores - // the cleared current pointer, initializes tracking, or starts work. - tracking_response( + // Receipt replay does not recreate a delegation or transfer manual work. + assert_eq!(counts(), before); + let fresh = tracking_response( app, request("PUT", &path, input("new-assign")), - StatusCode::CONFLICT, + StatusCode::OK, ) .await; - assert_eq!(counts(), before); + assert_ne!(fresh["assignment"]["assignment_id"].as_str(), Some(assignment)); assert_eq!( api.store .get_ticket_assignment_tracking(TEST_WORKSPACE_ID, &ticket, assignment) @@ -144,16 +146,9 @@ async fn runtime_and_account_assign_retries_return_saved_result_after_clear_and_ .unwrap(), active ); - assert!( - api.store - .get_current_ticket_role_assignment( - TEST_WORKSPACE_ID, - &ticket, - TicketAssignmentRole::Orchestrator - ) - .unwrap() - .is_none() - ); + assert_eq!(api.store.get_current_ticket_role_assignment(TEST_WORKSPACE_ID, &ticket, + TicketAssignmentRole::Orchestrator).unwrap().unwrap().assignment_id, + fresh["assignment"]["assignment_id"].as_str().unwrap()); } } diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index b5cfd5cec..1517c10eb 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -27,8 +27,11 @@ use crate::backend_job::{ use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; +#[path = "assignment_delivery.rs"] +mod assignment_delivery; #[path = "assignment_tracking.rs"] mod assignment_tracking; +pub use assignment_delivery::TicketAssignmentDelivery; pub use assignment_tracking::TicketOrchestratorAssignmentSourceContext; pub use ticket::{ TicketAssignmentTracking, TicketAssignmentTrackingHistoryEntry, @@ -37,7 +40,7 @@ pub use ticket::{ }; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 92; +const LATEST_SCHEMA_VERSION: i64 = 93; const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants"; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; @@ -305,6 +308,11 @@ const MIGRATIONS: &[Migration] = &[ name: "Ticket-owned assignment tracking", apply: migrate_ticket_tracking_v91_to_v92, }, + Migration { + version: 93, + name: "Durable assignment deliveries and source-bound startup fences", + apply: migrate_assignment_delivery_v92_to_v93, + }, ]; #[derive(Clone, Copy)] @@ -1808,6 +1816,52 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { request_fingerprint: &str, created_at: &str, ) -> Result; + /// Common scan for committed current delegation inputs, not handling status. + fn list_pending_ticket_assignment_deliveries( + &self, + workspace_id: &str, + limit: usize, + ) -> Result>; + /// Frozen payload/request identity for exact response-loss retry. + fn reserve_ticket_assignment_delivery( + &self, + workspace_id: &str, + assignment_id: &str, + input_event_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result; + /// Freeze exact rendered Runtime request bytes before dispatch. + fn prepare_ticket_assignment_delivery_content( + &self, + workspace_id: &str, + delivery_id: &str, + content: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result; + fn ack_ticket_assignment_delivery( + &self, + workspace_id: &str, + delivery_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result<()>; + fn reserve_ticket_assignment_operation_for_orchestrator( + &self, + workspace_id: &str, + operation_id: &str, + ticket_id: &str, + runtime_id: &str, + worker_id: Option<&str>, + request_fingerprint: &str, + created_at: &str, + orchestrator_assignment_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result; + /// Recovery/pre-dispatch check; write boundaries also revalidate this fence. + fn validate_ticket_assignment_operation_fence( + &self, + workspace_id: &str, + operation_id: &str, + ) -> Result<()>; fn bind_ticket_assignment_operation_worker( &self, workspace_id: &str, @@ -7841,92 +7895,90 @@ impl ControlPlaneStore for SqliteWorkspaceStore { request_fingerprint: &str, created_at: &str, ) -> Result { - validate_non_empty("operation_id", operation_id)?; - validate_non_empty("request_fingerprint", request_fingerprint)?; + self.reserve_ticket_assignment_operation_with_fence( + workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + request_fingerprint, + created_at, + None, + ) + } + fn reserve_ticket_assignment_operation_for_orchestrator( + &self, + workspace_id: &str, + operation_id: &str, + ticket_id: &str, + runtime_id: &str, + worker_id: Option<&str>, + request_fingerprint: &str, + created_at: &str, + orchestrator_assignment_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result { + self.reserve_ticket_assignment_operation_with_fence( + workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + request_fingerprint, + created_at, + Some((orchestrator_assignment_id, source)), + ) + } + fn validate_ticket_assignment_operation_fence( + &self, + workspace_id: &str, + operation_id: &str, + ) -> Result<()> { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - if let Some(existing) = read_assignment_operation(&tx, workspace_id, operation_id)? { - if existing.action == "assign" - && existing.ticket_id == ticket_id - && existing.runtime_id.as_deref() == Some(runtime_id) - && (worker_id.is_none() - || existing - .worker - .as_ref() - .map(|worker| worker.worker_id.as_str()) - == worker_id) - && existing.expected_assignment_id.is_none() - && existing.request_fingerprint.as_deref() == Some(request_fingerprint) - { - tx.commit()?; - return Ok(false); - } - return Err(Error::TicketAssignmentConflict(format!( - "assignment operation {operation_id} was already used with different input" - ))); - } - let worker = worker_id.map(|id| RuntimeWorkerRef::new(runtime_id, id)); - ensure_no_competing_ticket_claim( - &tx, - workspace_id, - ticket_id, - worker.as_ref(), - operation_id, - )?; - if let Some(worker) = &worker { - ensure_worker_assignment_available(&tx, workspace_id, worker)?; - let occupied: bool = tx.query_row( - "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments - WHERE workspace_id=?1 AND role='worker' AND runtime_id=?2 AND worker_id=?3)", - params![workspace_id, worker.runtime_id, worker.worker_id], - |row| row.get(0), - )?; - if occupied { - return Err(Error::TicketAssignmentConflict( - "Worker has unfinished Ticket work".into(), - )); - } - } - let state: String = tx.query_row( - "SELECT workflow_state FROM typed_tickets WHERE workspace_id=?1 AND ticket_id=?2", - params![workspace_id, ticket_id], - |row| row.get(0), - )?; - if state != "open" { - return Err(Error::TicketAssignmentConflict(format!( - "Ticket `{ticket_id}` cannot claim work in state `{state}`" - ))); - } - let occupied: bool = tx.query_row( - "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments - WHERE workspace_id=?1 AND ticket_id=?2 AND role='worker')", - params![workspace_id, ticket_id], - |row| row.get(0), - )?; - if occupied { - return Err(Error::TicketAssignmentConflict( - "Ticket has unfinished Worker work".into(), - )); - } - tx.execute( - "INSERT INTO ticket_assignment_operations ( - workspace_id,operation_id,action,ticket_id,runtime_id,worker_id,assignment_id, - expected_assignment_id,created_at,request_fingerprint,claim_state) - VALUES(?1,?2,'assign',?3,?4,?5,NULL,NULL,?6,?7,'pending')", - params![ - workspace_id, - operation_id, - ticket_id, - runtime_id, - worker_id, - created_at, - request_fingerprint - ], - )?; + assignment_delivery::operation_guard(&tx, workspace_id, operation_id)?; tx.commit()?; - Ok(true) + Ok(()) }) } + fn list_pending_ticket_assignment_deliveries( + &self, + workspace_id: &str, + limit: usize, + ) -> Result> { + if !(1..=1000).contains(&limit) { + return Err(Error::InvalidInput( + "Assignment delivery scan limit must be 1..=1000".into(), + )); + } + self.with_conn(|conn| assignment_delivery::list_in_tx(conn, workspace_id, limit)) + } + fn reserve_ticket_assignment_delivery( + &self, + workspace_id: &str, + assignment_id: &str, + input_event_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result { + assignment_delivery::reserve(self, workspace_id, assignment_id, input_event_id, source) + } + fn prepare_ticket_assignment_delivery_content( + &self, + workspace_id: &str, + delivery_id: &str, + content: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result { + assignment_delivery::prepare_content(self, workspace_id, delivery_id, content, source) + } + fn ack_ticket_assignment_delivery( + &self, + workspace_id: &str, + delivery_id: &str, + source: &TicketAssignmentTrackingSourceContext, + ) -> Result<()> { + assignment_delivery::ack(self, workspace_id, delivery_id, source) + } fn bind_ticket_assignment_operation_worker( &self, @@ -7936,6 +7988,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result<()> { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,workspace_id,operation_id)?; let receipt = read_assignment_operation(&tx, workspace_id, operation_id)? .ok_or_else(|| Error::TicketAssignmentConflict(format!("missing Ticket claim `{operation_id}`")))?; // Exact binding replay is safe even after assignment finalization. @@ -8279,6 +8332,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,&record.workspace_id,operation_id)?; if let Some(worker) = record.principal.worker() { ensure_worker_assignment_available(&tx, &record.workspace_id, &worker)?; } @@ -8334,29 +8388,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { "manual Worker assignment requires an open Ticket; current state is `{state}`" ))); } - let unresolved_blockers: i64 = tx.query_row( - "SELECT COUNT(*) - FROM typed_ticket_relations AS relation - JOIN typed_tickets AS related - ON related.workspace_id = relation.workspace_id - AND related.ticket_id = CASE - WHEN relation.ticket_id = ?2 THEN relation.target - ELSE relation.ticket_id - END - WHERE relation.workspace_id = ?1 - AND ( - (relation.ticket_id = ?2 AND relation.kind = 'depends_on') - OR (relation.target = ?2 AND relation.kind = 'blocks') - ) - AND related.workflow_state = 'open'", - params![record.workspace_id, record.ticket_id], - |row| row.get(0), - )?; - if unresolved_blockers != 0 { - return Err(Error::TicketAssignmentConflict( - "manual Worker assignment is blocked by unresolved Ticket relations".to_string(), - )); - } + // Relations remain diagnostics; they do not reserve work or block partial starts. let conflicting: i64 = tx.query_row( "SELECT COUNT(*) FROM ticket_active_worker_assignments WHERE workspace_id = ?1 AND ticket_id = ?2 @@ -8578,6 +8610,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,&record.workspace_id,operation_id)?; let removal_blocks_assignment: bool = tx.query_row( "SELECT EXISTS( SELECT 1 FROM worker_removal_operations @@ -11603,6 +11636,119 @@ fn ensure_worker_assignment_available( } impl SqliteWorkspaceStore { + fn reserve_ticket_assignment_operation_with_fence( + &self, + workspace_id: &str, + operation_id: &str, + ticket_id: &str, + runtime_id: &str, + worker_id: Option<&str>, + request_fingerprint: &str, + created_at: &str, + fence: Option<(&str, &TicketAssignmentTrackingSourceContext)>, + ) -> Result { + validate_non_empty("operation_id", operation_id)?; + validate_non_empty("request_fingerprint", request_fingerprint)?; + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,workspace_id,operation_id)?; + if fence.is_none() { + let fenced:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_assignment_operation_fences WHERE workspace_id=?1 AND operation_id=?2)",params![workspace_id,operation_id],|r|r.get(0))?; + if fenced {return Err(Error::TicketAssignmentConflict("Source-bound reservation requires authenticated Orchestrator context".into()));} + } + if let Some(existing) = read_assignment_operation(&tx, workspace_id, operation_id)? { + if existing.action == "assign" + && existing.ticket_id == ticket_id + && existing.runtime_id.as_deref() == Some(runtime_id) + && (worker_id.is_none() + || existing + .worker + .as_ref() + .map(|worker| worker.worker_id.as_str()) + == worker_id) + && existing.expected_assignment_id.is_none() + && existing.request_fingerprint.as_deref() == Some(request_fingerprint) + { + if let Some((assignment_id,source))=fence { + assignment_delivery::save_operation_fence(&tx,workspace_id,operation_id,ticket_id,assignment_id,source,true)?; + } + tx.commit()?; + return Ok(false); + } + return Err(Error::TicketAssignmentConflict(format!( + "assignment operation {operation_id} was already used with different input" + ))); + } + if let Some((assignment_id,source))=fence { + assignment_tracking::require_runtime_source(&tx,workspace_id,&source.runtime_id,&source.worker_id,&source.agent_key,&source.authentication_binding_id)?; + assignment_delivery::require_current(&tx,workspace_id,ticket_id,assignment_id)?; + } + let worker = worker_id.map(|id| RuntimeWorkerRef::new(runtime_id, id)); + ensure_no_competing_ticket_claim( + &tx, + workspace_id, + ticket_id, + worker.as_ref(), + operation_id, + )?; + if let Some(worker) = &worker { + ensure_worker_assignment_available(&tx, workspace_id, worker)?; + let occupied: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments + WHERE workspace_id=?1 AND role='worker' AND runtime_id=?2 AND worker_id=?3)", + params![workspace_id, worker.runtime_id, worker.worker_id], + |row| row.get(0), + )?; + if occupied { + return Err(Error::TicketAssignmentConflict( + "Worker has unfinished Ticket work".into(), + )); + } + } + let state: String = tx.query_row( + "SELECT workflow_state FROM typed_tickets WHERE workspace_id=?1 AND ticket_id=?2", + params![workspace_id, ticket_id], + |row| row.get(0), + )?; + if state != "open" { + return Err(Error::TicketAssignmentConflict(format!( + "Ticket `{ticket_id}` cannot claim work in state `{state}`" + ))); + } + let occupied: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments + WHERE workspace_id=?1 AND ticket_id=?2 AND role='worker')", + params![workspace_id, ticket_id], + |row| row.get(0), + )?; + if occupied { + return Err(Error::TicketAssignmentConflict( + "Ticket has unfinished Worker work".into(), + )); + } + tx.execute( + "INSERT INTO ticket_assignment_operations ( + workspace_id,operation_id,action,ticket_id,runtime_id,worker_id,assignment_id, + expected_assignment_id,created_at,request_fingerprint,claim_state) + VALUES(?1,?2,'assign',?3,?4,?5,NULL,NULL,?6,?7,'pending')", + params![ + workspace_id, + operation_id, + ticket_id, + runtime_id, + worker_id, + created_at, + request_fingerprint + ], + )?; + if let Some((assignment_id,source))=fence { + assignment_delivery::save_operation_fence(&tx,workspace_id,operation_id,ticket_id,assignment_id,source,false)?; + } + tx.commit()?; + Ok(true) + }) + } + fn set_ticket_role_assignment_with_source( &self, record: &TicketRoleAssignmentRecord, @@ -11664,6 +11810,7 @@ impl SqliteWorkspaceStore { }; self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + assignment_delivery::operation_guard(&tx,&record.workspace_id,operation_id)?; if let Some(source)=source {assignment_tracking::require_assignment_source(&tx,&record.workspace_id,source)?;} if let Some(worker) = record.principal.worker() { ensure_worker_assignment_available(&tx, &record.workspace_id, &worker)?; @@ -11699,24 +11846,13 @@ impl SqliteWorkspaceStore { return Ok(persisted); } - // This is a constraint on a new delegation, not on a saved result. - // Authenticate the source and resolve its full-input receipt first; - // work that began after the original Assign must not block replay. - if record.role == TicketAssignmentRole::Orchestrator { - let active_worker: bool = tx.query_row( - "SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments - WHERE workspace_id=?1 AND ticket_id=?2 AND role='worker')", - params![record.workspace_id, record.ticket_id], - |row| row.get(0), - )?; - if active_worker { - return Err(Error::TicketAssignmentConflict( - "Orchestrator assignment conflicts with an active Worker assignment".into(), - )); - } + // Delegation has no Runtime/resource occupancy. It can invalidate + // an in-flight startup fence; existing actual Worker/claim resource + // exclusion is checked independently at the next work admission. + if record.role != TicketAssignmentRole::Orchestrator { + ensure_no_competing_ticket_claim(&tx, &record.workspace_id, &record.ticket_id, + record.principal.worker().as_ref(), operation_id)?; } - ensure_no_competing_ticket_claim(&tx, &record.workspace_id, &record.ticket_id, - record.principal.worker().as_ref(), operation_id)?; let current_sql = ticket_role_assignment_select_sql( "WHERE current.workspace_id = ?1 AND current.ticket_id = ?2 AND current.role = ?3 \ ORDER BY a.assigned_at, a.assignment_id", @@ -12059,13 +12195,15 @@ fn clear_current_ticket_role_assignment_with_source_in_tx( } return Ok(true); } - ensure_no_competing_ticket_claim( - tx, - workspace_id, - ticket_id, - current.principal.worker().as_ref(), - operation_id, - )?; + if role != TicketAssignmentRole::Orchestrator { + ensure_no_competing_ticket_claim( + tx, + workspace_id, + ticket_id, + current.principal.worker().as_ref(), + operation_id, + )?; + } let is_current:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_current_worker_assignments WHERE workspace_id=?1 AND ticket_id=?2 AND role=?3 AND assignment_id=?4)",params![workspace_id,ticket_id,role.as_str(),assignment_id],|row|row.get(0))?; if !is_current { return Ok(false); @@ -13700,8 +13838,22 @@ fn migrate_ticket_tracking_v91_to_v92(conn: &Connection) -> Result<()> { Ok(()) } +fn migrate_assignment_delivery_v92_to_v93(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 92 { + return Err(Error::Store( + "Assignment delivery migration requires schema 92".into(), + )); + } + let tx = conn.unchecked_transaction()?; + assignment_delivery::install(&tx)?; + tx.execute("INSERT INTO __yoi_schema_migrations(version,name) VALUES(93,'Durable assignment deliveries and source-bound startup fences')",[])?; + tx.commit()?; + Ok(()) +} + fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { create_workspace_schema_structure(conn)?; + assignment_delivery::install(conn)?; ticket::verify_sqlite_ticket_schema(conn).map_err(|error| { Error::Store(format!( "fresh Workspace Ticket schema verification failed: {error}" @@ -13895,6 +14047,37 @@ fn verify_workspace_resource_constraints(conn: &Connection) -> Result<()> { } } } + if current_schema_version(conn)? >= 93 { + for (kind, name) in [ + ("table", "ticket_assignment_delivery_origins"), + ("table", "ticket_assignment_deliveries"), + ("table", "ticket_assignment_delivery_requests"), + ("table", "ticket_assignment_delivery_acknowledgements"), + ("table", "ticket_assignment_operation_fences"), + ("trigger", "ticket_assignment_delivery_origin_insert"), + ("trigger", "ticket_assignment_delivery_origin_no_update"), + ("trigger", "ticket_assignment_delivery_origin_no_delete"), + ("trigger", "ticket_assignment_delivery_payload_no_update"), + ("trigger", "ticket_assignment_delivery_payload_no_delete"), + ("trigger", "ticket_assignment_delivery_request_no_update"), + ("trigger", "ticket_assignment_delivery_request_no_delete"), + ("trigger", "ticket_assignment_delivery_ack_no_update"), + ("trigger", "ticket_assignment_delivery_ack_no_delete"), + ("trigger", "ticket_assignment_operation_fence_no_update"), + ("trigger", "ticket_assignment_operation_fence_no_delete"), + ] { + let present: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type=?1 AND name=?2)", + params![kind, name], + |row| row.get(0), + )?; + if !present { + return Err(Error::Store(format!( + "Assignment delivery constraint {name} is missing" + ))); + } + } + } Ok(()) } @@ -14648,6 +14831,7 @@ fn table_columns(conn: &Connection, table_name: &str) -> Result> { #[cfg(test)] mod tests { include!("assignment_tracking_tests.rs"); + include!("assignment_delivery_tests.rs"); include!("store_workspace_config_tests.rs"); fn drive_grant_fixture(store: &SqliteWorkspaceStore) -> server_api::DriveGrantResponse { @@ -16857,6 +17041,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx name: "Ticket open/closed cases and retained routing evidence".to_string() }, WorkspaceSchemaMigrationStep {version:92, name:"Ticket-owned assignment tracking".into()}, + WorkspaceSchemaMigrationStep {version:93, name:"Durable assignment deliveries and source-bound startup fences".into()}, ] ); @@ -16959,6 +17144,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx (90, "Runtime binding connection configuration and historical verification archive".to_string()), (91, "Ticket open/closed cases and retained routing evidence".to_string()), (92, "Ticket-owned assignment tracking".to_string()), +(93, "Durable assignment deliveries and source-bound startup fences".to_string()), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -19556,9 +19742,9 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx .map(|step| step.version) .collect::>(), if version == 89 { - vec![90, 91, 92] + vec![90, 91, 92, 93] } else { - vec![91, 92] + vec![91, 92, 93] } ); drop(conn); @@ -19573,7 +19759,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx assert_eq!(workspace_schema_migration_history(conn)?.into_iter().filter(|(version, _)| *version >= 90).collect::>(), vec![(90, "Runtime binding connection configuration and historical verification archive".into()), (91, "Ticket open/closed cases and retained routing evidence".into()), - (92, "Ticket-owned assignment tracking".into())]); + (92, "Ticket-owned assignment tracking".into()), + (93, "Durable assignment deliveries and source-bound startup fences".into())]); for (table, rows) in unchanged.iter().zip(before.iter()) { assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}"); } @@ -20291,7 +20478,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx } #[tokio::test] - async fn pending_claims_fence_all_assignment_writers_and_worker_binding_until_owner_commits() { + async fn pending_claims_fence_work_writers_but_allow_delegation_clear() { let (_dir, store) = manual_worker_start_store().await; seed_manual_start_ticket(&store, "first", "open"); seed_manual_start_ticket(&store, "second", "open"); @@ -20343,7 +20530,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx "2", None ) - .is_err() + .unwrap() ); assert!( store diff --git a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs index 0f1e65591..0ea9c64e7 100644 --- a/crates/workspace-server/src/ticket_tag_upgrade_tests.rs +++ b/crates/workspace-server/src/ticket_tag_upgrade_tests.rs @@ -265,7 +265,7 @@ fn assert_retained_classification_upgrade(workspace_version: i64, component: i64 drop(conn); let store = SqliteWorkspaceStore::open(&path).unwrap(); store.with_conn(|conn| { - assert_eq!(current_schema_version(conn)?,92); + assert_eq!(current_schema_version(conn)?,93); assert_eq!(conn.query_row("SELECT MAX(version) FROM ticket_schema_migrations",[],|row|row.get::<_,i64>(0))?,11); assert_eq!(fixture_auth_rows(conn),auth_before); for (table, rows) in preserved_tables.iter().zip(&preserved) { @@ -493,7 +493,7 @@ fn workspace90_cases9_add_classification_without_reinterpreting_reasoned_history let store = SqliteWorkspaceStore::open(&path).unwrap(); store .with_conn(|conn| { - assert_eq!(current_schema_version(conn)?, 92); + assert_eq!(current_schema_version(conn)?,93); assert_eq!( conn.query_row( "SELECT MAX(version) FROM ticket_schema_migrations", diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index f81403e7e..e241f8aee 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -45,6 +45,11 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ "repository_ssh_host_trust_keys", "repository_ssh_host_trusts", "server_secret_objects", + "ticket_assignment_delivery_acknowledgements", + "ticket_assignment_delivery_requests", + "ticket_assignment_deliveries", + "ticket_assignment_delivery_origins", + "ticket_assignment_operation_fences", "ticket_assignment_operations", "ticket_assignment_work_releases", "ticket_assignment_ticket_tombstones", @@ -479,6 +484,11 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore { ("ticket_worker_assignments", "typed_ticket_assignment_identity_no_delete"), ("ticket_worker_assignment_events", "typed_ticket_assignment_events_no_delete"), ("ticket_assignment_operations", "typed_ticket_assignment_receipt_no_delete"), + ("ticket_assignment_deliveries", "ticket_assignment_delivery_payload_no_delete"), + ("ticket_assignment_delivery_requests", "ticket_assignment_delivery_request_no_delete"), + ("ticket_assignment_delivery_acknowledgements", "ticket_assignment_delivery_ack_no_delete"), + ("ticket_assignment_delivery_origins", "ticket_assignment_delivery_origin_no_delete"), + ("ticket_assignment_operation_fences", "ticket_assignment_operation_fence_no_delete"), ("typed_ticket_assignment_tracking_history", "typed_ticket_assignment_tracking_history_no_delete"), ("typed_ticket_assignment_tracking_operations", "typed_ticket_assignment_tracking_operations_no_delete"), ("typed_ticket_case_history", "typed_ticket_case_history_no_delete"), diff --git a/docs/development/backend-db-execution.md b/docs/development/backend-db-execution.md index dd8f1241e..40baa3c3c 100644 --- a/docs/development/backend-db-execution.md +++ b/docs/development/backend-db-execution.md @@ -45,8 +45,10 @@ worker using the original Tokio handle; do not build a replacement runtime for cached services and do not submit to and await the Host lane from inside itself. Control/session guards needed by an accepted mutation belong to that actual job. -Backend Job cleanup is admitted before its durable cleanup claim. Rejection leaves -Pending/Failed ownership recoverable instead of abandoning a Running claim. Ticket +Backend Job cleanup has a third finite lane (two workers, 16 queued jobs), so a +long Run teardown wait cannot starve ordinary Host control/stop requests. It is +admitted before its durable cleanup claim. Rejection leaves Pending/Failed +ownership recoverable instead of abandoning an Executing claim. Ticket checker dispatch and timeout processing use bounded admission; overload leaves the saved reservation/deadline diagnosable. They are not automatically retried after an unknown DB outcome. Runtime teardown is still its own lifecycle fence, not proof diff --git a/docs/development/ticket-assignment-delivery.md b/docs/development/ticket-assignment-delivery.md new file mode 100644 index 000000000..59fa87214 --- /dev/null +++ b/docs/development/ticket-assignment-delivery.md @@ -0,0 +1,60 @@ +# Assign-driven Orchestrator delivery and start admission (T-742) + +## Separate authorities + +[Ticket assignment tracking](ticket-assignment-tracking.md) remains the sole delegation/handling authority (Ticket component schema 11). Workspace schema 93 adds **operational** input origins, delivery reservations, exact rendered requests, acceptance acknowledgements, and assignment/source-bound start fences. These tables do not create another assignment identity or handling status. + +Assign commits the current pointer, initial waiting, assignment event and source-bound operation receipt together. Its committed tracking-history insertion records an initial input origin in that same transaction. Delivery runs only after commit. Runtime acceptance, running/idle/stop, MR changes, and delivery acknowledgement never update `waiting`/`inprogress`/`done`. + +A delivery acknowledgement means the Runtime accepted the immutable Notify request. It is not an assertion that the model considered the input, satisfied the Ticket, or started execution. Runtime notification/activation receipts provide the corresponding durable session boundary. Self-reported progress/result remains a separate explicit mutation. + +## Common input scan and delivery + +Mutation notification, Backend startup, Orchestrator create (including already-online), accepted restore, replacement, and running→idle all use `list_pending_ticket_assignment_deliveries`. There is no parallel Queue/open-state routing predicate. Openness is a new-work safety boundary, not delegation intent. + +The scan selects current `workspace_agent:workspace-orchestrator` assignments with actionable handling and an explicit committed input origin. It excludes released identities, unknown legacy relationships, done/history and closed cases. External content/answer events after the origin are new inputs; handling self-reports and case/routing history are not. Waiting alone is never a resend reason. Close releases operational work; reopen alone does not revive the old delegation. Explicit reissue creates a new assignment and initial input. + +Schema 92→93 preserves original assignment/event/Queue evidence. Native Assign history establishes its input origin; migrated tracking contributes an input only where its already-reconciled assignment-specific Queue fence is present. Ambiguous/historical rows remain diagnostic data, not fresh requests. Earlier migrations and Ticket schema 11 are unchanged. + +Delivery order: + +1. Resolve the currently registered online Worker and active Runtime binding from Backend authority (a stopped Worker is not automatically restored). +2. Reserve the exact `assignment_id`/`input_event_id` under IMMEDIATE, rechecking current delegation, singleton and binding. +3. Render using the reservation's metadata, including waiting reason and next action. Persist the **entire rendered content**, not just its metadata or a title fingerprint. +4. Send the saved content with the saved `request_id`, outside the transaction. Response loss retains the same request/payload. Concurrent reservations converge on the same identity, and Runtime request receipts deduplicate actual acceptance. +5. Acknowledge accepted delivery under another IMMEDIATE transaction after rechecking current authority. The acknowledgement never changes handling. + +Process-local attention locks prevent concurrent/reentrant send loops; they are not backlog authority. A committed-input/startup/Restore trigger arriving during a reserved scan is coalesced into an input recheck and drained on reservation release, including when that scan read an empty snapshot. Discarding or acknowledging a pending retry continues the common scan in the same reservation; a successful retry cannot consume the trigger for newer input. Reentrant idle observations caused by a send do not schedule another retry of a failed send. These liveness repairs change no DB transaction or public API boundary. Losing that cache does not lose a delegation or resend acknowledged input. Replacement does not create assignment rows; only pending, unacknowledged inputs are recovered. Accepted history is not replayed into new work merely because a Worker was exchanged. + +## Public start/action handoff for T-743 + +Existing canonical tracking query/history/update endpoints, Tools/WIP, Ticket query/detail, and retained assignment projections remain available; see the tracking document. Model-visible notices include the committed delegation/input identity and bounded waiting/next-action context. The Server attention template is updated here; final Web/TUI/all-Prompt operation alignment belongs to T-743. + +Start request contracts now carry optional `orchestrator_assignment_id`: + +- `CreateWorkspaceWorkerRequest.ticket_assignment` (including Worker control spawn). +- `RuntimeWorkerSpawnRequest.ticket_assignment`. +- `SetTicketRoleAssignmentRequest` for an existing Worker. +- `SpawnTicketWorker` and generic `WorkerSpawn` tool input/lifecycle forwarding. + +An authenticated current Orchestrator **must** supply its observed delegation identity. Backend checks it with the actual `ServerRequestContext.runtime_source.authentication_binding_id`; it never derives the source from model role text or reacquires another binding as authentication proof. Missing/stale/cleared/reissued identities and obsolete Workers cannot admit work. Manual/generic authorized starts omit the delegation field and do not implicitly assign the Orchestrator. New starts and existing Worker assignments share the same reservation/start fence. + +The fenced reservation extends the existing Ticket operation receipt and Worker allocation/recovery identity. Reserve/retry, bind and finalization validate the saved assignment/source under IMMEDIATE. Recovery/pre-dispatch checks use the same fence. Clear/reassign can invalidate a reservation but do not stop, transfer or clean up already-running work. Unknown external outcomes keep their reservation/resource fences for explicit reconciliation rather than permitting a second start. Runtime I/O is never inside a database transaction. + +Unfinished dependencies remain query/relation diagnostics and do not disable manual/Orchestrator new starts or existing Worker assignment. They are never implicitly started. Case closure, authenticated permissions, actual unfinished work/claims, resource/grant conflicts, target containment and recovery remain independent guards. Repository/Workdir-free work, read-only work and caller-selected published Profiles are preserved. + +Action projection permits delegation independently of existing Worker occupancy; actual Worker start eligibility still excludes unfinished Worker work and closed cases. `can_start_manual_worker` no longer includes an unresolved-dependency predicate. `can_assign_orchestrator` permits an explicit fresh Assign/reissue operation even when a current row exists; callers can supply the observed expected assignment for CAS. `can_unassign_orchestrator` uses the retained current pointer, not active-work or case-state filtering. These authoring actions are not Worker transfer. Final browser controls must use these Backend decisions rather than recreating old phase/dependency gates. + +Delivery diagnostics are available through the typed Store scan/reservation/ack records and structured `ticket_assignment_delivery_pending` warning (`assignment_id`, `input_event_id`, `delivery_id`, recipient and error category, never content). Existing `ticket_notification_delivery_failed` remains for recipient failures. No browser acknowledgement/status-edit endpoint is introduced: successful delivery is not a user handling update. T-743 can consume the operational query independently of handling without inventing a third status in tracking. + +## DB caller/transaction handoff for T-748 and protocol boundaries + +Store callers are in `store.rs`; operational SQL/mutations are in `assignment_delivery.rs`. The public Store methods are `list_pending_ticket_assignment_deliveries`, `reserve_ticket_assignment_delivery`, `prepare_ticket_assignment_delivery_content`, `ack_ticket_assignment_delivery`, `reserve_ticket_assignment_operation_for_orchestrator`, and `validate_ticket_assignment_operation_fence`. Existing bind/finalization writes call the saved-operation guard inside their own IMMEDIATE transaction. + +Operational tables: `ticket_assignment_delivery_origins`, `ticket_assignment_deliveries`, `ticket_assignment_delivery_requests`, `ticket_assignment_delivery_acknowledgements`, `ticket_assignment_operation_fences`. Workspace explicit purge inventory includes their scoped rows and temporarily suspends/restores their immutable-delete guards in the existing guarded purge transaction. Ordinary operations cannot delete or rewrite receipts. + +T-741 set/clear/update/history remain unchanged in transaction ownership: actual source binding/singleton checks, receipt comparison, current/event CAS, audit and result must stay together. A future DB blocking adapter must move the whole synchronous transaction as one unit, not split its queries around awaits. Server Runtime method admission and `ServerRuntimeMutationSourceContext.authentication_binding_id` are not changed. T-739 close/reopen and T-715 identity-specific work releases remain execution boundaries. + +## Validation limits + +Local Store tests, authenticated HTTP tests, real embedded Controller/session tests, and loopback remote Runtime HTTP tests prove persistence, concurrency, source/assignment fencing, payload replay, resource contracts and activation dedupe. They do not prove model judgment quality or final Web/TUI/all-Prompt behavior (T-743/T-744). No live DB migration, environment update, target integration, Ticket conclusion or Runtime Worker/Workdir cleanup is part of this handoff. diff --git a/openapi/server-api.json b/openapi/server-api.json index 78819a744..a8b846fa9 100644 --- a/openapi/server-api.json +++ b/openapi/server-api.json @@ -1606,6 +1606,13 @@ "operation_id": { "type": "string" }, + "orchestrator_assignment_id": { + "description": "Observed delegation required for Orchestrator Ticket starts; manual starts omit it.", + "type": [ + "string", + "null" + ] + }, "ticket_id": { "type": "string" } @@ -8953,6 +8960,13 @@ "operation_id": { "type": "string" }, + "orchestrator_assignment_id": { + "description": "Observed delegation required for Orchestrator Ticket starts; manual starts omit it.", + "type": [ + "string", + "null" + ] + }, "ticket_id": { "type": "string" } @@ -9304,6 +9318,13 @@ "operation_id": { "type": "string" }, + "orchestrator_assignment_id": { + "description": "Observed delegation required for Orchestrator Ticket starts; manual starts omit it.", + "type": [ + "string", + "null" + ] + }, "principal": { "$ref": "#/components/schemas/TicketAssignmentPrincipal" }, @@ -16813,7 +16834,7 @@ "info": { "title": "Yoi Server API", "version": "0.1.0", - "x-yoi-source-digest": "sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774" + "x-yoi-source-digest": "sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b" }, "openapi": "3.1.0", "paths": { diff --git a/resources/prompts/internal/workspace_orchestrator_queue_attention.md b/resources/prompts/internal/workspace_orchestrator_queue_attention.md index eee42a88e..87c914e12 100644 --- a/resources/prompts/internal/workspace_orchestrator_queue_attention.md +++ b/resources/prompts/internal/workspace_orchestrator_queue_attention.md @@ -1,8 +1,17 @@ -Queued Tickets require attention: +Assigned Tickets have unreceived input: {% for ticket in tickets -%} - {{ ticket.resource_key }} {{ separator }} {{ ticket.title }} +{% if ticket.assignment_id -%} + Delegation: {{ ticket.assignment_id }}; input: {{ ticket.input_event_id }}. +{% endif -%} +{% if ticket.waiting_reason -%} + Waiting reason: {{ ticket.waiting_reason }} +{% endif -%} +{% if ticket.next_action -%} + Next action: {{ ticket.next_action }} +{% endif -%} {% endfor -%} {% if omitted_ticket_count > 0 -%} -Additional queued Tickets were omitted from this notice: {{ omitted_ticket_count }}. Re-query current Ticket authority for the complete set. +Additional assigned Tickets were omitted from this notice: {{ omitted_ticket_count }}. Re-query current Ticket authority for the complete set. {% endif -%} -Reread the current Ticket state before acting. Preserve the human queue gate and current assignment, dependency, Worker, and Workdir authority; do not create duplicate work. +Reread the current Ticket state before acting, including current assignment tracking and new input. Assign is delegation; no ready/Queue step is required. Handling status is your self-report, not delivery or Worker state. Waiting alone and done history are not new work. Dependencies are diagnostics for partial work, not a blanket start gate; never implicitly start dependencies. Preserve current assignment identity, authenticated Worker, unfinished work, and Workdir/resource authority; do not create duplicate work. diff --git a/web/workspace/src/lib/generated/companion-api.ts b/web/workspace/src/lib/generated/companion-api.ts index 88c4da3bb..d40e0565a 100644 --- a/web/workspace/src/lib/generated/companion-api.ts +++ b/web/workspace/src/lib/generated/companion-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-companion-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/companion-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check diff --git a/web/workspace/src/lib/generated/repository-api.ts b/web/workspace/src/lib/generated/repository-api.ts index a3f8fc78e..307f65277 100644 --- a/web/workspace/src/lib/generated/repository-api.ts +++ b/web/workspace/src/lib/generated/repository-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-repository-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output // Output: web/workspace/src/lib/generated/repository-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check diff --git a/web/workspace/src/lib/generated/runtime-api.ts b/web/workspace/src/lib/generated/runtime-api.ts index 7a78b1505..516e54f07 100644 --- a/web/workspace/src/lib/generated/runtime-api.ts +++ b/web/workspace/src/lib/generated/runtime-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-runtime-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/runtime-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check @@ -151,7 +151,7 @@ export type RuntimeWorkerSpawnWorkingDirectoryRequest = { repository_key: string export type RuntimeWorkerSummary = { diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: unknown; workspace: WorkerWorkspaceSummary; }; -export type RuntimeWorkerTicketAssignmentRequest = { operation_id: string; ticket_id: string; }; +export type RuntimeWorkerTicketAssignmentRequest = { operation_id: string; orchestrator_assignment_id?: string | null; ticket_id: string; }; export type Segment = { content: string; kind: "text"; } | { chars: number; content: string; id: number; kind: "paste"; lines: number; } | { artifact: PasteArtifactRef; kind: "paste_artifact"; } | { file: UploadedFileRef; kind: "uploaded_file"; } | { kind: "file_ref"; path: string; } | { invocation: FeatureInvocation; kind: "feature_invoke"; } | { kind: "flow"; selector: string; } | { kind: "unknown"; }; diff --git a/web/workspace/src/lib/generated/ticket-api.ts b/web/workspace/src/lib/generated/ticket-api.ts index 60e817c7a..7c9829198 100644 --- a/web/workspace/src/lib/generated/ticket-api.ts +++ b/web/workspace/src/lib/generated/ticket-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/ticket-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check @@ -83,7 +83,7 @@ export type ReviewDecision = "approve" | "request_changes"; export type ReviewFinding = { body: string; code?: string | null; line?: number | null; path?: string | null; severity: FindingSeverity; }; -export type SetTicketRoleAssignmentRequest = { expected_assignment_id?: string | null; operation_id: string; principal: TicketAssignmentPrincipal; workdir_bindings?: Array; }; +export type SetTicketRoleAssignmentRequest = { expected_assignment_id?: string | null; operation_id: string; orchestrator_assignment_id?: string | null; principal: TicketAssignmentPrincipal; workdir_bindings?: Array; }; export type TicketActionEligibility = { blockers: Array; can_assign_orchestrator: boolean; can_queue: boolean; can_start_manual_worker: boolean; can_unassign_orchestrator: boolean; queue_tickets: Array; }; diff --git a/web/workspace/src/lib/generated/worker-launch-api.ts b/web/workspace/src/lib/generated/worker-launch-api.ts index 8db0ab85e..3868137c5 100644 --- a/web/workspace/src/lib/generated/worker-launch-api.ts +++ b/web/workspace/src/lib/generated/worker-launch-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-worker-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:982cc7edc8fd8655a14e943b9d0022df56e941107def02251186d8ddebfd6774) +// Input contract: 0.1.0 (sha256:8c8254cbe4ca8675f3dfffdf6e0eeecfda1c9213b285f571704648774efa497b) // Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/worker-launch-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check @@ -13,7 +13,7 @@ export type BrowserWorkspaceOrchestratorResponse = { diagnostics: Array; profile?: string | null; runtime_id: string; singleton_key?: string | null; ticket_assignment?: CreateWorkspaceWorkerTicketAssignmentRequest | null; workdir_attachments?: Array; }; -export type CreateWorkspaceWorkerTicketAssignmentRequest = { operation_id: string; ticket_id: string; }; +export type CreateWorkspaceWorkerTicketAssignmentRequest = { operation_id: string; orchestrator_assignment_id?: string | null; ticket_id: string; }; export type Diagnostic = { code: string; message: string; severity: DiagnosticSeverity; };