runtime: route notifications through worker inbox
This commit is contained in:
@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum WorkerInputKind {
|
pub enum WorkerInputKind {
|
||||||
User,
|
User,
|
||||||
System,
|
Notify,
|
||||||
Compact,
|
Compact,
|
||||||
ListRewindTargets,
|
ListRewindTargets,
|
||||||
RegisterPeer,
|
RegisterPeer,
|
||||||
@@ -38,15 +38,35 @@ impl WorkerInput {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn system(content: impl Into<String>) -> Self {
|
pub fn notify(content: impl Into<String>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
kind: WorkerInputKind::System,
|
kind: WorkerInputKind::Notify,
|
||||||
content: content.into(),
|
content: content.into(),
|
||||||
segments: None,
|
segments: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::WorkerInput;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn notify_is_an_operation_and_legacy_system_kind_is_rejected() {
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(WorkerInput::notify("message")).unwrap(),
|
||||||
|
serde_json::json!({ "kind": "notify", "content": "message" })
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<WorkerInput>(serde_json::json!({
|
||||||
|
"kind": "system",
|
||||||
|
"content": "message"
|
||||||
|
}))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Acknowledgement returned after input is accepted into the Worker.
|
/// Acknowledgement returned after input is accepted into the Worker.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct WorkerInteractionAck {
|
pub struct WorkerInteractionAck {
|
||||||
|
|||||||
@@ -2392,9 +2392,9 @@ fn input_protocol_event(input: &WorkerInput) -> protocol::Event {
|
|||||||
}]
|
}]
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
WorkerInputKind::System => protocol::Event::SystemItem {
|
WorkerInputKind::Notify => protocol::Event::SystemItem {
|
||||||
item: serde_json::json!({
|
item: serde_json::json!({
|
||||||
"kind": "embedded_worker_system_input",
|
"kind": "embedded_worker_notification",
|
||||||
"content": input.content.clone(),
|
"content": input.content.clone(),
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
@@ -3148,7 +3148,7 @@ mod tests {
|
|||||||
fn create_worker_rejects_system_initial_input_without_persisting_worker() {
|
fn create_worker_rejects_system_initial_input_without_persisting_worker() {
|
||||||
let runtime = runtime_with_backend();
|
let runtime = runtime_with_backend();
|
||||||
let mut request = task_request("system initial input");
|
let mut request = task_request("system initial input");
|
||||||
request.initial_input = Some(WorkerInput::system("role/system belongs in config bundle"));
|
request.initial_input = Some(WorkerInput::notify("role/system belongs in config bundle"));
|
||||||
|
|
||||||
let error = runtime.create_worker(request).unwrap_err();
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
@@ -3390,7 +3390,7 @@ mod tests {
|
|||||||
.send_input(&detail.worker_ref, WorkerInput::user("hello"))
|
.send_input(&detail.worker_ref, WorkerInput::user("hello"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime
|
runtime
|
||||||
.send_input(&detail.worker_ref, WorkerInput::system("note"))
|
.send_input(&detail.worker_ref, WorkerInput::notify("note"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let observations = runtime
|
let observations = runtime
|
||||||
@@ -3550,7 +3550,7 @@ mod tests {
|
|||||||
.send_input(&worker.worker_ref, WorkerInput::user("first"))
|
.send_input(&worker.worker_ref, WorkerInput::user("first"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime
|
runtime
|
||||||
.send_input(&worker.worker_ref, WorkerInput::system("second"))
|
.send_input(&worker.worker_ref, WorkerInput::notify("second"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime
|
runtime
|
||||||
.stop_worker(&worker.worker_ref, Some("finished".to_string()))
|
.stop_worker(&worker.worker_ref, Some("finished".to_string()))
|
||||||
|
|||||||
@@ -793,6 +793,14 @@ fn method_starts_turn(method: &Method) -> bool {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExecutionRunState {
|
||||||
|
match status {
|
||||||
|
WorkerStatus::Running => WorkerExecutionRunState::Busy,
|
||||||
|
WorkerStatus::Idle if auto_run => WorkerExecutionRunState::Busy,
|
||||||
|
WorkerStatus::Idle | WorkerStatus::Paused => WorkerExecutionRunState::Idle,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
|
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
|
||||||
match method {
|
match method {
|
||||||
Method::Run { .. }
|
Method::Run { .. }
|
||||||
@@ -1098,6 +1106,29 @@ where
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if input.kind == WorkerInputKind::Notify {
|
||||||
|
let status = worker.shared_state.get_status();
|
||||||
|
let accepted_run_state = accepted_notify_run_state(status, true);
|
||||||
|
let claimed_here = status == WorkerStatus::Idle
|
||||||
|
&& busy
|
||||||
|
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||||
|
.is_ok();
|
||||||
|
let result = self.send_method(
|
||||||
|
WorkerExecutionOperation::Input,
|
||||||
|
worker,
|
||||||
|
Method::Notify {
|
||||||
|
message: input.content,
|
||||||
|
auto_run: true,
|
||||||
|
},
|
||||||
|
accepted_run_state,
|
||||||
|
);
|
||||||
|
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
|
||||||
|
{
|
||||||
|
busy.store(false, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
if worker.shared_state.get_status() != WorkerStatus::Idle
|
if worker.shared_state.get_status() != WorkerStatus::Idle
|
||||||
|| busy
|
|| busy
|
||||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||||
@@ -1115,10 +1146,9 @@ where
|
|||||||
.segments
|
.segments
|
||||||
.unwrap_or_else(|| vec![Segment::text(input.content.trim().to_string())]),
|
.unwrap_or_else(|| vec![Segment::text(input.content.trim().to_string())]),
|
||||||
},
|
},
|
||||||
WorkerInputKind::System => Method::Notify {
|
WorkerInputKind::Notify => {
|
||||||
message: input.content,
|
unreachable!("Notify input is dispatched before the turn-start busy guard")
|
||||||
auto_run: true,
|
}
|
||||||
},
|
|
||||||
WorkerInputKind::Compact => Method::Compact,
|
WorkerInputKind::Compact => Method::Compact,
|
||||||
WorkerInputKind::ListRewindTargets => Method::ListRewindTargets,
|
WorkerInputKind::ListRewindTargets => Method::ListRewindTargets,
|
||||||
WorkerInputKind::RegisterPeer => Method::RegisterPeer {
|
WorkerInputKind::RegisterPeer => Method::RegisterPeer {
|
||||||
@@ -1159,6 +1189,28 @@ where
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if let Method::Notify { auto_run, .. } = &method {
|
||||||
|
let auto_run = *auto_run;
|
||||||
|
let status = worker.shared_state.get_status();
|
||||||
|
let accepted_run_state = accepted_notify_run_state(status, auto_run);
|
||||||
|
let claimed_here = status == WorkerStatus::Idle
|
||||||
|
&& auto_run
|
||||||
|
&& busy
|
||||||
|
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||||
|
.is_ok();
|
||||||
|
let result = self.send_method(
|
||||||
|
WorkerExecutionOperation::ProtocolMethod,
|
||||||
|
worker,
|
||||||
|
method,
|
||||||
|
accepted_run_state,
|
||||||
|
);
|
||||||
|
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
|
||||||
|
{
|
||||||
|
busy.store(false, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
let starts_turn = method_starts_turn(&method);
|
let starts_turn = method_starts_turn(&method);
|
||||||
if starts_turn
|
if starts_turn
|
||||||
&& (worker.shared_state.get_status() != WorkerStatus::Idle
|
&& (worker.shared_state.get_status() != WorkerStatus::Idle
|
||||||
@@ -1298,6 +1350,26 @@ mod tests {
|
|||||||
use manifest::{Scope, WorkerManifest};
|
use manifest::{Scope, WorkerManifest};
|
||||||
use session_store::WorkerMetadataStore;
|
use session_store::WorkerMetadataStore;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn notify_run_state_allows_running_worker_inbox_delivery() {
|
||||||
|
assert_eq!(
|
||||||
|
accepted_notify_run_state(WorkerStatus::Running, true),
|
||||||
|
WorkerExecutionRunState::Busy
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
accepted_notify_run_state(WorkerStatus::Idle, true),
|
||||||
|
WorkerExecutionRunState::Busy
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
accepted_notify_run_state(WorkerStatus::Idle, false),
|
||||||
|
WorkerExecutionRunState::Idle
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
accepted_notify_run_state(WorkerStatus::Paused, true),
|
||||||
|
WorkerExecutionRunState::Idle
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct MockClient {
|
struct MockClient {
|
||||||
responses: Arc<Vec<Vec<LlmEvent>>>,
|
responses: Arc<Vec<Vec<LlmEvent>>>,
|
||||||
|
|||||||
@@ -285,6 +285,7 @@ impl WorkerController {
|
|||||||
worker.push_notify(
|
worker.push_notify(
|
||||||
"Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
"Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
||||||
.to_string(),
|
.to_string(),
|
||||||
|
false,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -883,7 +884,7 @@ async fn controller_loop<C, St>(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let parent_originated = run.is_parent_originated();
|
let parent_originated = run.is_parent_originated();
|
||||||
let (new_status, shutdown) = match run {
|
let (mut new_status, shutdown) = match run {
|
||||||
PendingRun::Run(input) => {
|
PendingRun::Run(input) => {
|
||||||
drive_turn(
|
drive_turn(
|
||||||
worker.run(input),
|
worker.run(input),
|
||||||
@@ -930,6 +931,11 @@ async fn controller_loop<C, St>(
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if !shutdown && new_status == WorkerStatus::Idle && notify_buffer.has_auto_run_pending()
|
||||||
|
{
|
||||||
|
pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify));
|
||||||
|
new_status = WorkerStatus::Running;
|
||||||
|
}
|
||||||
finish_controller_run(
|
finish_controller_run(
|
||||||
&mut worker,
|
&mut worker,
|
||||||
&shared_state,
|
&shared_state,
|
||||||
@@ -985,13 +991,13 @@ async fn controller_loop<C, St>(
|
|||||||
// `LogEntry::SystemItem` entry — drained out of the
|
// `LogEntry::SystemItem` entry — drained out of the
|
||||||
// notify buffer + broadcast through the sink. No
|
// notify buffer + broadcast through the sink. No
|
||||||
// separate echo here.
|
// separate echo here.
|
||||||
worker.push_notify(message);
|
worker.push_notify(message, auto_run);
|
||||||
// RUNNING / Paused: the buffer push is the entire
|
// RUNNING: the in-flight turn drains the buffer at its next
|
||||||
// operation; an in-flight turn (or the next
|
// pending_history_appends; if an auto-run notification remains
|
||||||
// Resume/Run) will drain it at its next
|
// at turn end, the Controller stages a follow-up notification
|
||||||
// pending_history_appends. IDLE: only `auto_run`
|
// turn. Paused notifications remain queued until Resume/Run.
|
||||||
// notifications stage RunForNotification; weak progress
|
// IDLE: `auto_run` notifications stage RunForNotification;
|
||||||
// notices stay queued until an explicit run/resume.
|
// weak progress notices stay queued until an explicit run.
|
||||||
if should_auto_run_notification(shared_state.get_status(), auto_run) {
|
if should_auto_run_notification(shared_state.get_status(), auto_run) {
|
||||||
pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify));
|
pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify));
|
||||||
}
|
}
|
||||||
@@ -1385,11 +1391,11 @@ where
|
|||||||
.into(),
|
.into(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Some(Method::Notify { message, .. }) => {
|
Some(Method::Notify { message, auto_run }) => {
|
||||||
// Live echo arrives via `Event::SystemItem` once
|
// Live echo arrives via `Event::SystemItem` once
|
||||||
// the in-flight turn's next `pending_history_appends`
|
// the in-flight turn's next `pending_history_appends`
|
||||||
// drains this entry through the interceptor.
|
// drains this entry through the interceptor.
|
||||||
notify_buffer.push_notify(message);
|
notify_buffer.push_notify(message, auto_run);
|
||||||
}
|
}
|
||||||
Some(Method::ListCompletions { .. }) => {}
|
Some(Method::ListCompletions { .. }) => {}
|
||||||
Some(Method::ListWorkers | Method::RestoreWorker { .. } | Method::RegisterPeer { .. }) => {
|
Some(Method::ListWorkers | Method::RestoreWorker { .. } | Method::RegisterPeer { .. }) => {
|
||||||
@@ -1904,6 +1910,41 @@ mod tests {
|
|||||||
assert_eq!(env.notify_buffer.len(), 1);
|
assert_eq!(env.notify_buffer.len(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn running_auto_run_notify_remains_staged_for_followup_turn() {
|
||||||
|
let mut env = make_env().await;
|
||||||
|
env._method_tx
|
||||||
|
.send(Method::Notify {
|
||||||
|
message: "continue".into(),
|
||||||
|
auto_run: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("send notify");
|
||||||
|
|
||||||
|
let worker_future = async {
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
Ok::<_, WorkerError>(WorkerRunResult::Finished)
|
||||||
|
};
|
||||||
|
let (status, shutdown) = drive_turn(
|
||||||
|
worker_future,
|
||||||
|
&mut env.method_rx,
|
||||||
|
&env.event_tx,
|
||||||
|
&env.cancel_tx,
|
||||||
|
&env.shared_state,
|
||||||
|
&env.notify_buffer,
|
||||||
|
Some(&env.parent_socket_path),
|
||||||
|
"parent",
|
||||||
|
&env.spawned_registry,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(status, WorkerStatus::Idle);
|
||||||
|
assert!(!shutdown);
|
||||||
|
assert_eq!(env.notify_buffer.len(), 1);
|
||||||
|
assert!(env.notify_buffer.has_auto_run_pending());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn compact_method_is_rejected_while_running() {
|
async fn compact_method_is_rejected_while_running() {
|
||||||
let mut env = make_env().await;
|
let mut env = make_env().await;
|
||||||
|
|||||||
@@ -220,7 +220,9 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
// simply be skipped from the SystemItem batch.
|
// simply be skipped from the SystemItem batch.
|
||||||
warn!(error = %e, "failed to render notify_wrapper; using raw message");
|
warn!(error = %e, "failed to render notify_wrapper; using raw message");
|
||||||
let fallback = match &entry {
|
let fallback = match &entry {
|
||||||
super::notify_buffer::PendingNotify::Notify { message } => message.clone(),
|
super::notify_buffer::PendingNotify::Notify { message, .. } => {
|
||||||
|
message.clone()
|
||||||
|
}
|
||||||
super::notify_buffer::PendingNotify::WorkerEvent { event } => {
|
super::notify_buffer::PendingNotify::WorkerEvent { event } => {
|
||||||
session_store::render_worker_event(event)
|
session_store::render_worker_event(event)
|
||||||
}
|
}
|
||||||
@@ -1019,8 +1021,8 @@ mod tests {
|
|||||||
async fn pending_history_appends_drains_buffer_into_items() {
|
async fn pending_history_appends_drains_buffer_into_items() {
|
||||||
let registry = Arc::new(HookRegistryBuilder::new().build());
|
let registry = Arc::new(HookRegistryBuilder::new().build());
|
||||||
let buffer = NotifyBuffer::new();
|
let buffer = NotifyBuffer::new();
|
||||||
buffer.push_notify("first".into());
|
buffer.push_notify("first".into(), false);
|
||||||
buffer.push_notify("second".into());
|
buffer.push_notify("second".into(), false);
|
||||||
|
|
||||||
let interceptor = WorkerInterceptor::new(
|
let interceptor = WorkerInterceptor::new(
|
||||||
registry,
|
registry,
|
||||||
@@ -1057,7 +1059,7 @@ mod tests {
|
|||||||
// anything itself.
|
// anything itself.
|
||||||
let registry = Arc::new(HookRegistryBuilder::new().build());
|
let registry = Arc::new(HookRegistryBuilder::new().build());
|
||||||
let buffer = NotifyBuffer::new();
|
let buffer = NotifyBuffer::new();
|
||||||
buffer.push_notify("msg".into());
|
buffer.push_notify("msg".into(), false);
|
||||||
|
|
||||||
let interceptor = WorkerInterceptor::new(
|
let interceptor = WorkerInterceptor::new(
|
||||||
registry,
|
registry,
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ const CAPACITY: usize = 128;
|
|||||||
/// is available.
|
/// is available.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub enum PendingNotify {
|
pub enum PendingNotify {
|
||||||
Notify { message: String },
|
Notify { message: String, auto_run: bool },
|
||||||
WorkerEvent { event: WorkerEvent },
|
WorkerEvent { event: WorkerEvent },
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,8 +61,8 @@ impl NotifyBuffer {
|
|||||||
/// Push a notify entry onto the queue. If the queue is full, the
|
/// Push a notify entry onto the queue. If the queue is full, the
|
||||||
/// oldest entry is dropped and a `tracing::warn` is emitted — the
|
/// oldest entry is dropped and a `tracing::warn` is emitted — the
|
||||||
/// caller should never hit this in normal operation.
|
/// caller should never hit this in normal operation.
|
||||||
pub fn push_notify(&self, message: String) {
|
pub fn push_notify(&self, message: String, auto_run: bool) {
|
||||||
self.push_entry(PendingNotify::Notify { message });
|
self.push_entry(PendingNotify::Notify { message, auto_run });
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Push a typed worker-event entry onto the queue.
|
/// Push a typed worker-event entry onto the queue.
|
||||||
@@ -89,6 +89,15 @@ impl NotifyBuffer {
|
|||||||
q.drain(..).collect()
|
q.drain(..).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether an undrained `Method::Notify { auto_run: true }` remains.
|
||||||
|
pub fn has_auto_run_pending(&self) -> bool {
|
||||||
|
self.inner
|
||||||
|
.lock()
|
||||||
|
.expect("notify buffer poisoned")
|
||||||
|
.iter()
|
||||||
|
.any(|entry| matches!(entry, PendingNotify::Notify { auto_run: true, .. }))
|
||||||
|
}
|
||||||
|
|
||||||
/// Number of pending entries. Primarily for tests.
|
/// Number of pending entries. Primarily for tests.
|
||||||
pub fn len(&self) -> usize {
|
pub fn len(&self) -> usize {
|
||||||
self.inner.lock().expect("notify buffer poisoned").len()
|
self.inner.lock().expect("notify buffer poisoned").len()
|
||||||
@@ -107,7 +116,7 @@ pub(crate) fn build_system_item(
|
|||||||
prompts: &PromptCatalog,
|
prompts: &PromptCatalog,
|
||||||
) -> Result<SystemItem, CatalogError> {
|
) -> Result<SystemItem, CatalogError> {
|
||||||
match entry {
|
match entry {
|
||||||
PendingNotify::Notify { message } => {
|
PendingNotify::Notify { message, .. } => {
|
||||||
let body = prompts.notify_wrapper(message)?;
|
let body = prompts.notify_wrapper(message)?;
|
||||||
Ok(SystemItem::Notification {
|
Ok(SystemItem::Notification {
|
||||||
message: message.clone(),
|
message: message.clone(),
|
||||||
@@ -132,12 +141,15 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn push_then_drain_preserves_order() {
|
fn push_then_drain_preserves_order() {
|
||||||
let buf = NotifyBuffer::new();
|
let buf = NotifyBuffer::new();
|
||||||
buf.push_notify("one".into());
|
buf.push_notify("one".into(), false);
|
||||||
buf.push_notify("two".into());
|
assert!(!buf.has_auto_run_pending());
|
||||||
|
buf.push_notify("two".into(), true);
|
||||||
|
assert!(buf.has_auto_run_pending());
|
||||||
let drained = buf.drain();
|
let drained = buf.drain();
|
||||||
|
assert!(!buf.has_auto_run_pending());
|
||||||
assert_eq!(drained.len(), 2);
|
assert_eq!(drained.len(), 2);
|
||||||
match &drained[0] {
|
match &drained[0] {
|
||||||
PendingNotify::Notify { message } => assert_eq!(message, "one"),
|
PendingNotify::Notify { message, .. } => assert_eq!(message, "one"),
|
||||||
other => panic!("unexpected: {other:?}"),
|
other => panic!("unexpected: {other:?}"),
|
||||||
}
|
}
|
||||||
assert!(buf.is_empty());
|
assert!(buf.is_empty());
|
||||||
@@ -147,12 +159,12 @@ mod tests {
|
|||||||
fn capacity_drops_oldest() {
|
fn capacity_drops_oldest() {
|
||||||
let buf = NotifyBuffer::new();
|
let buf = NotifyBuffer::new();
|
||||||
for i in 0..(CAPACITY + 5) {
|
for i in 0..(CAPACITY + 5) {
|
||||||
buf.push_notify(format!("msg{i}"));
|
buf.push_notify(format!("msg{i}"), false);
|
||||||
}
|
}
|
||||||
let drained = buf.drain();
|
let drained = buf.drain();
|
||||||
assert_eq!(drained.len(), CAPACITY);
|
assert_eq!(drained.len(), CAPACITY);
|
||||||
match &drained[0] {
|
match &drained[0] {
|
||||||
PendingNotify::Notify { message } => assert_eq!(message, "msg5"),
|
PendingNotify::Notify { message, .. } => assert_eq!(message, "msg5"),
|
||||||
other => panic!("unexpected: {other:?}"),
|
other => panic!("unexpected: {other:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -161,6 +173,7 @@ mod tests {
|
|||||||
fn build_system_item_for_notify_carries_wrapper_body() {
|
fn build_system_item_for_notify_carries_wrapper_body() {
|
||||||
let entry = PendingNotify::Notify {
|
let entry = PendingNotify::Notify {
|
||||||
message: "hello".into(),
|
message: "hello".into(),
|
||||||
|
auto_run: false,
|
||||||
};
|
};
|
||||||
let catalog = PromptCatalog::builtins_only().unwrap();
|
let catalog = PromptCatalog::builtins_only().unwrap();
|
||||||
let item = build_system_item(&entry, &catalog).unwrap();
|
let item = build_system_item(&entry, &catalog).unwrap();
|
||||||
|
|||||||
@@ -1597,8 +1597,8 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
/// `Item::system_message` just before the next LLM request, via
|
/// `Item::system_message` just before the next LLM request, via
|
||||||
/// `WorkerInterceptor::pending_history_appends`. See [`NotifyBuffer`]
|
/// `WorkerInterceptor::pending_history_appends`. See [`NotifyBuffer`]
|
||||||
/// for overflow behaviour and the lane-of-record rationale.
|
/// for overflow behaviour and the lane-of-record rationale.
|
||||||
pub fn push_notify(&self, message: String) {
|
pub fn push_notify(&self, message: String, auto_run: bool) {
|
||||||
self.pending_notifies.push_notify(message);
|
self.pending_notifies.push_notify(message, auto_run);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Push an agent-visible typed `WorkerEvent` entry onto the pending buffer.
|
/// Push an agent-visible typed `WorkerEvent` entry onto the pending buffer.
|
||||||
@@ -4364,6 +4364,7 @@ where
|
|||||||
self.push_notify(
|
self.push_notify(
|
||||||
"Restored Worker state contained missing or unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
"Restored Worker state contained missing or unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
||||||
.to_string(),
|
.to_string(),
|
||||||
|
false,
|
||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -496,7 +496,7 @@ pub struct WorkerLifecycleResult {
|
|||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum WorkerInputKind {
|
pub enum WorkerInputKind {
|
||||||
User,
|
User,
|
||||||
System,
|
Notify,
|
||||||
Compact,
|
Compact,
|
||||||
ListRewindTargets,
|
ListRewindTargets,
|
||||||
RegisterPeer,
|
RegisterPeer,
|
||||||
@@ -2115,7 +2115,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
let input = EmbeddedWorkerInput {
|
let input = EmbeddedWorkerInput {
|
||||||
kind: match request.kind {
|
kind: match request.kind {
|
||||||
WorkerInputKind::User => EmbeddedWorkerInputKind::User,
|
WorkerInputKind::User => EmbeddedWorkerInputKind::User,
|
||||||
WorkerInputKind::System => EmbeddedWorkerInputKind::System,
|
WorkerInputKind::Notify => EmbeddedWorkerInputKind::Notify,
|
||||||
WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact,
|
WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact,
|
||||||
WorkerInputKind::ListRewindTargets => EmbeddedWorkerInputKind::ListRewindTargets,
|
WorkerInputKind::ListRewindTargets => EmbeddedWorkerInputKind::ListRewindTargets,
|
||||||
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
||||||
@@ -3086,7 +3086,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
let input = EmbeddedWorkerInput {
|
let input = EmbeddedWorkerInput {
|
||||||
kind: match request.kind {
|
kind: match request.kind {
|
||||||
WorkerInputKind::User => EmbeddedWorkerInputKind::User,
|
WorkerInputKind::User => EmbeddedWorkerInputKind::User,
|
||||||
WorkerInputKind::System => EmbeddedWorkerInputKind::System,
|
WorkerInputKind::Notify => EmbeddedWorkerInputKind::Notify,
|
||||||
WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact,
|
WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact,
|
||||||
WorkerInputKind::ListRewindTargets => EmbeddedWorkerInputKind::ListRewindTargets,
|
WorkerInputKind::ListRewindTargets => EmbeddedWorkerInputKind::ListRewindTargets,
|
||||||
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
||||||
@@ -4538,7 +4538,7 @@ mod tests {
|
|||||||
.expect("test backend should connect");
|
.expect("test backend should connect");
|
||||||
let mut request = embedded_spawn_request();
|
let mut request = embedded_spawn_request();
|
||||||
request.initial_input = Some(EmbeddedWorkerInput {
|
request.initial_input = Some(EmbeddedWorkerInput {
|
||||||
kind: EmbeddedWorkerInputKind::System,
|
kind: EmbeddedWorkerInputKind::Notify,
|
||||||
content: "system/role instruction belongs in profile".to_string(),
|
content: "system/role instruction belongs in profile".to_string(),
|
||||||
segments: None,
|
segments: None,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||||
use std::path::{Component, Path, PathBuf};
|
use std::path::{Component, Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use axum::extract::ws::{Message as WsMessage, WebSocket, WebSocketUpgrade};
|
use axum::extract::ws::{Message as WsMessage, WebSocket, WebSocketUpgrade};
|
||||||
use axum::extract::{Path as AxumPath, Query, State};
|
use axum::extract::{Path as AxumPath, Query, State};
|
||||||
@@ -17,7 +17,6 @@ use memory::backend::{
|
|||||||
MemoryConsolidationOutput,
|
MemoryConsolidationOutput,
|
||||||
};
|
};
|
||||||
use protocol::stream::{decode_method, encode_event};
|
use protocol::stream::{decode_method, encode_event};
|
||||||
use rusqlite::OptionalExtension;
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use ticket::{
|
use ticket::{
|
||||||
@@ -240,6 +239,12 @@ impl ServerConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ORCHESTRATOR_ATTENTION_TICKET_LIMIT: usize = 20;
|
||||||
|
const ORCHESTRATOR_ATTENTION_PROMPT: &str = include_str!(concat!(
|
||||||
|
env!("CARGO_MANIFEST_DIR"),
|
||||||
|
"/../../resources/prompts/internal/workspace_orchestrator_queue_attention.md"
|
||||||
|
));
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct WorkspaceApi {
|
pub struct WorkspaceApi {
|
||||||
config: ServerConfig,
|
config: ServerConfig,
|
||||||
@@ -248,6 +253,7 @@ pub struct WorkspaceApi {
|
|||||||
runtime: Arc<RuntimeRegistry>,
|
runtime: Arc<RuntimeRegistry>,
|
||||||
companion: Arc<CompanionConsole>,
|
companion: Arc<CompanionConsole>,
|
||||||
orchestrator_spawn_lock: Arc<std::sync::Mutex<()>>,
|
orchestrator_spawn_lock: Arc<std::sync::Mutex<()>>,
|
||||||
|
orchestrator_attention_fingerprint: Arc<Mutex<Option<String>>>,
|
||||||
observation_proxy: BackendObservationProxy,
|
observation_proxy: BackendObservationProxy,
|
||||||
runtime_subscription_broker: RuntimeSubscriptionBroker,
|
runtime_subscription_broker: RuntimeSubscriptionBroker,
|
||||||
resource_broker: BackendResourceBroker,
|
resource_broker: BackendResourceBroker,
|
||||||
@@ -348,6 +354,7 @@ impl WorkspaceApi {
|
|||||||
runtime,
|
runtime,
|
||||||
companion,
|
companion,
|
||||||
orchestrator_spawn_lock: Arc::new(std::sync::Mutex::new(())),
|
orchestrator_spawn_lock: Arc::new(std::sync::Mutex::new(())),
|
||||||
|
orchestrator_attention_fingerprint: Arc::new(Mutex::new(None)),
|
||||||
observation_proxy,
|
observation_proxy,
|
||||||
runtime_subscription_broker,
|
runtime_subscription_broker,
|
||||||
resource_broker,
|
resource_broker,
|
||||||
@@ -963,21 +970,9 @@ pub async fn serve(
|
|||||||
listener: TcpListener,
|
listener: TcpListener,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let api = WorkspaceApi::new(config, store).await?;
|
let api = WorkspaceApi::new(config, store).await?;
|
||||||
let dispatcher_api = api.clone();
|
let orchestrator_hook = tokio::spawn(run_orchestrator_turn_end_hook(api.clone()));
|
||||||
let dispatcher_workspace_id = dispatcher_api.config.workspace_id.clone();
|
|
||||||
let dispatcher = tokio::spawn(async move {
|
|
||||||
loop {
|
|
||||||
let api = dispatcher_api.clone();
|
|
||||||
let workspace_id = dispatcher_workspace_id.clone();
|
|
||||||
let _ = tokio::task::spawn_blocking(move || {
|
|
||||||
dispatch_pending_ticket_notifications(&api, &workspace_id)
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let result = axum::serve(listener, build_router(api)).await;
|
let result = axum::serve(listener, build_router(api)).await;
|
||||||
dispatcher.abort();
|
orchestrator_hook.abort();
|
||||||
result?;
|
result?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -2169,7 +2164,27 @@ async fn scoped_queue_ticket(
|
|||||||
browser_ticket_backend(&api)?
|
browser_ticket_backend(&api)?
|
||||||
.queue_ready(TicketIdOrSlug::Id(path.id.clone()), queued_by)
|
.queue_ready(TicketIdOrSlug::Id(path.id.clone()), queued_by)
|
||||||
.map_err(Error::from)?;
|
.map_err(Error::from)?;
|
||||||
browser_ticket_detail(&api, &path.id)
|
let Json(ticket) = browser_ticket_detail(&api, &path.id)?;
|
||||||
|
notify_ticket_recipients(
|
||||||
|
&api,
|
||||||
|
&path.workspace_id,
|
||||||
|
&path.id,
|
||||||
|
ticket
|
||||||
|
.events
|
||||||
|
.last()
|
||||||
|
.map(|event| event.sequence as i64)
|
||||||
|
.unwrap_or_default(),
|
||||||
|
ticket
|
||||||
|
.events
|
||||||
|
.last()
|
||||||
|
.map(|event| event.kind.as_str())
|
||||||
|
.unwrap_or("state_changed"),
|
||||||
|
"queue_ready",
|
||||||
|
TicketWorkflowState::Ready.as_str(),
|
||||||
|
ticket.state.as_str(),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
Ok(Json(ticket))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn scoped_close_ticket(
|
async fn scoped_close_ticket(
|
||||||
@@ -2204,44 +2219,36 @@ async fn execute_worker_ticket_rest_operation(
|
|||||||
let operation_kind = ticket_mutation_operation_kind(&operation);
|
let operation_kind = ticket_mutation_operation_kind(&operation);
|
||||||
let is_mutation = operation_kind != "read";
|
let is_mutation = operation_kind != "read";
|
||||||
let target = ticket_mutation_target(&operation).cloned();
|
let target = ticket_mutation_target(&operation).cloned();
|
||||||
let read_target = ticket_read_target(&operation).cloned();
|
|
||||||
let source = authenticate_worker_mutation_source(api, workspace_id, &headers)?;
|
let source = authenticate_worker_mutation_source(api, workspace_id, &headers)?;
|
||||||
let before = target.as_ref().and_then(|id| backend.show(id.clone()).ok());
|
let before = target.as_ref().and_then(|id| backend.show(id.clone()).ok());
|
||||||
bind_worker_ticket_operation_source(&source, &mut operation);
|
let previous_state = before
|
||||||
let source_context = worker_ticket_source_context(api, workspace_id, &source, before.as_ref());
|
|
||||||
backend = backend
|
|
||||||
.with_event_attributes(source_context.attributes(operation_kind))
|
|
||||||
.with_mutation_hook(build_ticket_notification_hook(
|
|
||||||
api,
|
|
||||||
source_context,
|
|
||||||
operation_kind,
|
|
||||||
before
|
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|ticket| ticket.meta.workflow_state.as_str().to_string())
|
.map(|ticket| ticket.meta.workflow_state.as_str().to_string())
|
||||||
.unwrap_or_else(|| ticket_operation_initial_state(&operation)),
|
.unwrap_or_else(|| ticket_operation_initial_state(&operation));
|
||||||
));
|
bind_worker_ticket_operation_source(&source, &mut operation);
|
||||||
|
let source_context = worker_ticket_source_context(api, workspace_id, &source, before.as_ref());
|
||||||
|
backend = backend.with_event_attributes(source_context.attributes(operation_kind));
|
||||||
|
|
||||||
let result = execute_ticket_backend_operation(&backend, operation).map_err(Error::from)?;
|
let result = execute_ticket_backend_operation(&backend, operation).map_err(Error::from)?;
|
||||||
if let Some(read_target) = read_target.as_ref()
|
if is_mutation
|
||||||
&& let Ok(ticket) = backend.show(read_target.clone())
|
&& let Some(target) = target
|
||||||
&& let Some(event_index) = ticket.events.last().and_then(|event| {
|
&& let Ok(ticket) = backend.show(target)
|
||||||
event
|
|
||||||
.attributes
|
|
||||||
.get("event_sequence")
|
|
||||||
.and_then(|value| value.parse::<i64>().ok())
|
|
||||||
})
|
|
||||||
{
|
{
|
||||||
api.store.upsert_ticket_notification_cursor(
|
let event = ticket.events.last();
|
||||||
|
notify_ticket_recipients(
|
||||||
|
api,
|
||||||
workspace_id,
|
workspace_id,
|
||||||
&ticket.meta.id,
|
&ticket.meta.id,
|
||||||
&source.runtime_id,
|
event
|
||||||
&source.worker_id,
|
.and_then(|event| event.attributes.get("event_sequence"))
|
||||||
event_index,
|
.and_then(|value| value.parse::<i64>().ok())
|
||||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
|
.unwrap_or(ticket.events.len() as i64),
|
||||||
)?;
|
event.map(|event| event.kind.as_str()).unwrap_or("mutation"),
|
||||||
}
|
operation_kind,
|
||||||
if is_mutation {
|
&previous_state,
|
||||||
dispatch_pending_ticket_notifications(api, workspace_id);
|
ticket.meta.workflow_state.as_str(),
|
||||||
|
Some((source.runtime_id, source.worker_id)),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
@@ -2791,13 +2798,6 @@ fn bind_worker_ticket_operation_source(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ticket_read_target(operation: &TicketBackendOperation) -> Option<&TicketIdOrSlug> {
|
|
||||||
match operation {
|
|
||||||
TicketBackendOperation::Show { id } => Some(id),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn ticket_mutation_operation_kind(operation: &TicketBackendOperation) -> &'static str {
|
fn ticket_mutation_operation_kind(operation: &TicketBackendOperation) -> &'static str {
|
||||||
match operation {
|
match operation {
|
||||||
TicketBackendOperation::Create { .. } => "create",
|
TicketBackendOperation::Create { .. } => "create",
|
||||||
@@ -2830,12 +2830,10 @@ fn ticket_operation_initial_state(operation: &TicketBackendOperation) -> String
|
|||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct WorkerTicketSourceContext {
|
struct WorkerTicketSourceContext {
|
||||||
workspace_id: String,
|
|
||||||
runtime_id: String,
|
runtime_id: String,
|
||||||
worker_id: String,
|
worker_id: String,
|
||||||
actor_role: String,
|
actor_role: String,
|
||||||
assignment_id: Option<String>,
|
assignment_id: Option<String>,
|
||||||
orchestrator: Option<(String, String)>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerTicketSourceContext {
|
impl WorkerTicketSourceContext {
|
||||||
@@ -2887,7 +2885,6 @@ fn worker_ticket_source_context(
|
|||||||
});
|
});
|
||||||
let actor_role = worker_source_actor_role(is_current_assignment, is_orchestrator);
|
let actor_role = worker_source_actor_role(is_current_assignment, is_orchestrator);
|
||||||
WorkerTicketSourceContext {
|
WorkerTicketSourceContext {
|
||||||
workspace_id: workspace_id.to_string(),
|
|
||||||
runtime_id: source.runtime_id.clone(),
|
runtime_id: source.runtime_id.clone(),
|
||||||
worker_id: source.worker_id.clone(),
|
worker_id: source.worker_id.clone(),
|
||||||
actor_role: actor_role.to_string(),
|
actor_role: actor_role.to_string(),
|
||||||
@@ -2895,94 +2892,63 @@ fn worker_ticket_source_context(
|
|||||||
(assignment.runtime_id == source.runtime_id && assignment.worker_id == source.worker_id)
|
(assignment.runtime_id == source.runtime_id && assignment.worker_id == source.worker_id)
|
||||||
.then_some(assignment.assignment_id)
|
.then_some(assignment.assignment_id)
|
||||||
}),
|
}),
|
||||||
orchestrator: orchestrator.map(|worker| (worker.runtime_id, worker.worker_id)),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_ticket_notification_hook(
|
fn notify_ticket_recipients(
|
||||||
_api: &WorkspaceApi,
|
api: &WorkspaceApi,
|
||||||
source: WorkerTicketSourceContext,
|
workspace_id: &str,
|
||||||
operation_kind: &'static str,
|
ticket_id: &str,
|
||||||
previous_state: String,
|
event_sequence: i64,
|
||||||
) -> Arc<ticket::SqliteTicketMutationHook> {
|
event_kind: &str,
|
||||||
let invoked = AtomicBool::new(false);
|
source_operation_kind: &str,
|
||||||
let notification_id = new_id("tnfy");
|
previous_state: &str,
|
||||||
Arc::new(move |conn, event| {
|
current_state: &str,
|
||||||
if invoked.swap(true, Ordering::SeqCst) {
|
source: Option<(String, String)>,
|
||||||
return Ok(());
|
) {
|
||||||
}
|
|
||||||
let current_state: String = conn
|
|
||||||
.query_row(
|
|
||||||
"SELECT workflow_state FROM typed_tickets WHERE workspace_id = ?1 AND ticket_id = ?2",
|
|
||||||
rusqlite::params![source.workspace_id, event.ticket_id],
|
|
||||||
|row| row.get(0),
|
|
||||||
)
|
|
||||||
.map_err(|error| ticket::TicketError::Conflict(format!("read committed Ticket state for outbox: {error}")))?;
|
|
||||||
conn.execute(
|
|
||||||
r#"INSERT INTO ticket_notification_outbox (
|
|
||||||
notification_id, workspace_id, ticket_id, event_sequence,
|
|
||||||
source_runtime_id, source_worker_id, previous_state, current_state, created_at,
|
|
||||||
event_kind, source_operation_kind, source_actor_role, source_assignment_id
|
|
||||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)"#,
|
|
||||||
rusqlite::params![
|
|
||||||
notification_id,
|
|
||||||
source.workspace_id,
|
|
||||||
event.ticket_id,
|
|
||||||
event.event_index,
|
|
||||||
source.runtime_id,
|
|
||||||
source.worker_id,
|
|
||||||
previous_state,
|
|
||||||
current_state,
|
|
||||||
Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
|
|
||||||
event.event_kind.as_str(),
|
|
||||||
operation_kind,
|
|
||||||
source.actor_role,
|
|
||||||
source.assignment_id,
|
|
||||||
],
|
|
||||||
)
|
|
||||||
.map_err(|error| {
|
|
||||||
ticket::TicketError::Conflict(format!("insert Ticket notification outbox: {error}"))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let assigned: Option<(String, String)> = conn
|
|
||||||
.query_row(
|
|
||||||
r#"SELECT runtime_id, worker_id FROM ticket_current_worker_assignments
|
|
||||||
WHERE workspace_id = ?1 AND ticket_id = ?2"#,
|
|
||||||
rusqlite::params![source.workspace_id, event.ticket_id],
|
|
||||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
|
||||||
)
|
|
||||||
.optional()
|
|
||||||
.map_err(|error| {
|
|
||||||
ticket::TicketError::Conflict(format!(
|
|
||||||
"resolve assigned notification recipient: {error}"
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
let mut recipients = Vec::new();
|
let mut recipients = Vec::new();
|
||||||
if let Some((runtime_id, worker_id)) = assigned {
|
if let Some(assignment) = api
|
||||||
if runtime_id != source.runtime_id || worker_id != source.worker_id {
|
.store
|
||||||
recipients.push((runtime_id, worker_id, "assigned"));
|
.get_current_ticket_worker_assignment(workspace_id, ticket_id)
|
||||||
}
|
.ok()
|
||||||
}
|
.flatten()
|
||||||
if (matches!(previous_state.as_str(), "queued" | "inprogress")
|
|
||||||
|| matches!(current_state.as_str(), "queued" | "inprogress"))
|
|
||||||
&& let Some((runtime_id, worker_id)) = &source.orchestrator
|
|
||||||
&& (*runtime_id != source.runtime_id || *worker_id != source.worker_id)
|
|
||||||
{
|
{
|
||||||
recipients.push((runtime_id.clone(), worker_id.clone(), "orchestrator"));
|
recipients.push((assignment.runtime_id, assignment.worker_id));
|
||||||
|
}
|
||||||
|
if (matches!(previous_state, "queued" | "inprogress")
|
||||||
|
|| matches!(current_state, "queued" | "inprogress"))
|
||||||
|
&& let Some(orchestrator) = find_workspace_orchestrator(api)
|
||||||
|
{
|
||||||
|
recipients.push((orchestrator.runtime_id, orchestrator.worker_id));
|
||||||
}
|
}
|
||||||
recipients.sort();
|
recipients.sort();
|
||||||
recipients.dedup_by(|left, right| left.0 == right.0 && left.1 == right.1);
|
recipients.dedup();
|
||||||
for (runtime_id, worker_id, recipient_kind) in recipients {
|
|
||||||
conn.execute(
|
let source_fields = source
|
||||||
r#"INSERT OR IGNORE INTO ticket_notification_deliveries (
|
.as_ref()
|
||||||
notification_id, recipient_runtime_id, recipient_worker_id, recipient_kind, attempts
|
.map(|(runtime_id, worker_id)| {
|
||||||
) VALUES (?1, ?2, ?3, ?4, 0)"#,
|
format!(" source_runtime_id={runtime_id} source_worker_id={worker_id}")
|
||||||
rusqlite::params![notification_id, runtime_id, worker_id, recipient_kind],
|
|
||||||
)
|
|
||||||
.map_err(|error| ticket::TicketError::Conflict(format!("insert Ticket notification delivery: {error}")))?;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
})
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
for (runtime_id, worker_id) in recipients {
|
||||||
|
if source
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|source| source.0 == runtime_id && source.1 == worker_id)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let _ = api.runtime.send_input(
|
||||||
|
&runtime_id,
|
||||||
|
&worker_id,
|
||||||
|
WorkerInputRequest {
|
||||||
|
kind: WorkerInputKind::Notify,
|
||||||
|
content: format!(
|
||||||
|
"Ticket notification: workspace_id={workspace_id} ticket_id={ticket_id} event_sequence={event_sequence} event_kind={event_kind} source_operation_kind={source_operation_kind}.{source_fields} Reread the Ticket before acting.",
|
||||||
|
),
|
||||||
|
segments: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn authenticate_worker_mutation_source(
|
fn authenticate_worker_mutation_source(
|
||||||
@@ -3011,107 +2977,184 @@ fn authenticate_worker_mutation_source(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn dispatch_pending_ticket_notifications(api: &WorkspaceApi, workspace_id: &str) {
|
async fn run_orchestrator_turn_end_hook(api: WorkspaceApi) {
|
||||||
let Ok(deliveries) = api
|
let Ok(mut subscription) = api.runtime_subscription_broker.subscribe(
|
||||||
.store
|
EMBEDDED_WORKER_RUNTIME_ID,
|
||||||
.list_pending_ticket_notification_deliveries(workspace_id, 100)
|
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
|
||||||
else {
|
) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
for delivery in deliveries {
|
let mut worker_states = HashMap::new();
|
||||||
let Some((current_runtime_id, current_worker_id)) =
|
while let Some(update) = subscription.recv().await {
|
||||||
current_ticket_notification_recipient(api, &delivery)
|
match update {
|
||||||
else {
|
crate::runtime_subscription::BrokerSubscriptionEvent::Snapshot { snapshot, .. } => {
|
||||||
continue;
|
if let protocol::subscription::SubscriptionSnapshot::Workers { workers } = snapshot
|
||||||
};
|
|
||||||
if current_runtime_id == delivery.source_runtime_id
|
|
||||||
&& current_worker_id == delivery.source_worker_id
|
|
||||||
{
|
{
|
||||||
let _ = api.store.mark_ticket_notification_delivered(
|
worker_states.clear();
|
||||||
&delivery.notification_id,
|
for worker in workers {
|
||||||
&delivery.recipient_runtime_id,
|
let worker_id = worker.worker_id.to_string();
|
||||||
&delivery.recipient_worker_id,
|
maybe_dispatch_orchestrator_turn_end(&api, &worker_id, None, worker.state);
|
||||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
|
worker_states.insert(worker_id, worker.state);
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
if current_runtime_id != delivery.recipient_runtime_id
|
|
||||||
|| current_worker_id != delivery.recipient_worker_id
|
|
||||||
{
|
|
||||||
let _ = api.store.reroute_ticket_notification_delivery(
|
|
||||||
&delivery.notification_id,
|
|
||||||
&delivery.recipient_runtime_id,
|
|
||||||
&delivery.recipient_worker_id,
|
|
||||||
¤t_runtime_id,
|
|
||||||
¤t_worker_id,
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
if api
|
}
|
||||||
.store
|
crate::runtime_subscription::BrokerSubscriptionEvent::Event { payload, .. } => {
|
||||||
.get_ticket_notification_cursor(
|
match payload {
|
||||||
&delivery.workspace_id,
|
protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => {
|
||||||
&delivery.ticket_id,
|
let worker_id = worker.worker_id.to_string();
|
||||||
¤t_runtime_id,
|
let previous = worker_states.insert(worker_id.clone(), worker.state);
|
||||||
¤t_worker_id,
|
maybe_dispatch_orchestrator_turn_end(
|
||||||
|
&api,
|
||||||
|
&worker_id,
|
||||||
|
previous,
|
||||||
|
worker.state,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
protocol::subscription::SubscriptionEventPayload::WorkerRemoved {
|
||||||
|
worker_id,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
worker_states.remove(worker_id.as_str());
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
crate::runtime_subscription::BrokerSubscriptionEvent::Disconnected { .. } => {
|
||||||
|
worker_states.clear();
|
||||||
|
}
|
||||||
|
crate::runtime_subscription::BrokerSubscriptionEvent::Rejected { .. }
|
||||||
|
| crate::runtime_subscription::BrokerSubscriptionEvent::Closed { .. } => return,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn maybe_dispatch_orchestrator_turn_end(
|
||||||
|
api: &WorkspaceApi,
|
||||||
|
worker_id: &str,
|
||||||
|
previous: Option<protocol::subscription::SubscriptionWorkerState>,
|
||||||
|
current: protocol::subscription::SubscriptionWorkerState,
|
||||||
|
) {
|
||||||
|
use protocol::subscription::SubscriptionWorkerState;
|
||||||
|
|
||||||
|
if current != SubscriptionWorkerState::Idle
|
||||||
|
|| !matches!(
|
||||||
|
previous,
|
||||||
|
None | Some(SubscriptionWorkerState::Running)
|
||||||
|
| Some(SubscriptionWorkerState::Stopped)
|
||||||
|
| Some(SubscriptionWorkerState::Paused)
|
||||||
)
|
)
|
||||||
.ok()
|
|
||||||
.flatten()
|
|
||||||
.is_some_and(|cursor| cursor >= delivery.event_sequence)
|
|
||||||
{
|
{
|
||||||
let _ = api.store.mark_ticket_notification_delivered(
|
return;
|
||||||
&delivery.notification_id,
|
|
||||||
&delivery.recipient_runtime_id,
|
|
||||||
&delivery.recipient_worker_id,
|
|
||||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
let result = api.runtime.send_input(
|
let Some(orchestrator) = find_workspace_orchestrator(api) else {
|
||||||
&delivery.recipient_runtime_id,
|
return;
|
||||||
&delivery.recipient_worker_id,
|
};
|
||||||
|
if orchestrator.runtime_id == EMBEDDED_WORKER_RUNTIME_ID && orchestrator.worker_id == worker_id
|
||||||
|
{
|
||||||
|
dispatch_orchestrator_queue_attention(api);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dispatch_orchestrator_queue_attention(api: &WorkspaceApi) {
|
||||||
|
let Some(orchestrator) = find_workspace_orchestrator(api) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Ok(backend) = browser_ticket_backend(api) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Ok(mut queued) = backend.list(ticket::TicketListQuery::states([
|
||||||
|
ticket::TicketListState::Queued,
|
||||||
|
])) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
queued.sort_by(|left, right| left.id.cmp(&right.id));
|
||||||
|
if queued.is_empty() {
|
||||||
|
*api.orchestrator_attention_fingerprint
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Ok(inprogress) = backend.list(ticket::TicketListQuery::states([
|
||||||
|
ticket::TicketListState::InProgress,
|
||||||
|
])) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if !inprogress.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let fingerprint = queued
|
||||||
|
.iter()
|
||||||
|
.map(|ticket| ticket.id.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("|");
|
||||||
|
if api
|
||||||
|
.orchestrator_attention_fingerprint
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||||
|
.as_deref()
|
||||||
|
== Some(fingerprint.as_str())
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let shown = queued
|
||||||
|
.iter()
|
||||||
|
.take(ORCHESTRATOR_ATTENTION_TICKET_LIMIT)
|
||||||
|
.map(|ticket| {
|
||||||
|
format!(
|
||||||
|
"- {} — {}",
|
||||||
|
bounded_orchestrator_attention_text(&ticket.id, 80),
|
||||||
|
bounded_orchestrator_attention_text(&ticket.title, 240)
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
let omitted = queued
|
||||||
|
.len()
|
||||||
|
.saturating_sub(ORCHESTRATOR_ATTENTION_TICKET_LIMIT);
|
||||||
|
let omitted_line = if omitted == 0 {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!("Additional queued Tickets omitted from this notice: {omitted}\n")
|
||||||
|
};
|
||||||
|
let content = ORCHESTRATOR_ATTENTION_PROMPT
|
||||||
|
.replace("{{omitted_line}}", &omitted_line)
|
||||||
|
.replace("{{workspace_id}}", &api.config.workspace_id)
|
||||||
|
.replace("{{ticket_lines}}", &shown);
|
||||||
|
let accepted = api
|
||||||
|
.runtime
|
||||||
|
.send_input(
|
||||||
|
&orchestrator.runtime_id,
|
||||||
|
&orchestrator.worker_id,
|
||||||
WorkerInputRequest {
|
WorkerInputRequest {
|
||||||
kind: WorkerInputKind::System,
|
kind: WorkerInputKind::Notify,
|
||||||
content: format!(
|
content,
|
||||||
"Ticket notification: workspace_id={} ticket_id={} event_sequence={} event_kind={} source_operation_kind={} source_runtime_id={} source_worker_id={}. Reread the Ticket before acting.",
|
|
||||||
delivery.workspace_id,
|
|
||||||
delivery.ticket_id,
|
|
||||||
delivery.event_sequence,
|
|
||||||
delivery.event_kind,
|
|
||||||
delivery.source_operation_kind,
|
|
||||||
delivery.source_runtime_id,
|
|
||||||
delivery.source_worker_id
|
|
||||||
),
|
|
||||||
segments: None,
|
segments: None,
|
||||||
},
|
},
|
||||||
);
|
)
|
||||||
match result {
|
.is_ok_and(|result| result.state == WorkerOperationState::Accepted);
|
||||||
Ok(result) if result.state == WorkerOperationState::Accepted => {
|
if accepted {
|
||||||
let _ = api.store.mark_ticket_notification_delivered(
|
*api.orchestrator_attention_fingerprint
|
||||||
&delivery.notification_id,
|
.lock()
|
||||||
&delivery.recipient_runtime_id,
|
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(fingerprint);
|
||||||
&delivery.recipient_worker_id,
|
|
||||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(result) => {
|
|
||||||
let _ = api.store.mark_ticket_notification_failed(
|
|
||||||
&delivery.notification_id,
|
|
||||||
&delivery.recipient_runtime_id,
|
|
||||||
&delivery.recipient_worker_id,
|
|
||||||
&format!("Runtime rejected notification: {:?}", result.diagnostics),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(error) => {
|
|
||||||
let _ = api.store.mark_ticket_notification_failed(
|
|
||||||
&delivery.notification_id,
|
|
||||||
&delivery.recipient_runtime_id,
|
|
||||||
&delivery.recipient_worker_id,
|
|
||||||
&error.into_error().to_string(),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bounded_orchestrator_attention_text(input: &str, max_chars: usize) -> String {
|
||||||
|
let mut output = String::new();
|
||||||
|
for (index, character) in input.chars().enumerate() {
|
||||||
|
if index == max_chars {
|
||||||
|
output.push('…');
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
output.push(if character.is_control() {
|
||||||
|
' '
|
||||||
|
} else {
|
||||||
|
character
|
||||||
|
});
|
||||||
|
}
|
||||||
|
output
|
||||||
}
|
}
|
||||||
|
|
||||||
fn find_workspace_orchestrator(api: &WorkspaceApi) -> Option<WorkerSummary> {
|
fn find_workspace_orchestrator(api: &WorkspaceApi) -> Option<WorkerSummary> {
|
||||||
@@ -3140,24 +3183,6 @@ fn find_workspace_orchestrator(api: &WorkspaceApi) -> Option<WorkerSummary> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn current_ticket_notification_recipient(
|
|
||||||
api: &WorkspaceApi,
|
|
||||||
delivery: &crate::store::TicketNotificationDeliveryRecord,
|
|
||||||
) -> Option<(String, String)> {
|
|
||||||
match delivery.recipient_kind.as_str() {
|
|
||||||
"assigned" => api
|
|
||||||
.store
|
|
||||||
.get_current_ticket_worker_assignment(&delivery.workspace_id, &delivery.ticket_id)
|
|
||||||
.ok()
|
|
||||||
.flatten()
|
|
||||||
.map(|assignment| (assignment.runtime_id, assignment.worker_id)),
|
|
||||||
"orchestrator" => {
|
|
||||||
find_workspace_orchestrator(api).map(|worker| (worker.runtime_id, worker.worker_id))
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
struct MemoryDocumentResponse {
|
struct MemoryDocumentResponse {
|
||||||
body_md: String,
|
body_md: String,
|
||||||
@@ -3701,6 +3726,10 @@ async fn scoped_start_workspace_orchestrator(
|
|||||||
restored.diagnostics,
|
restored.diagnostics,
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
*api.orchestrator_attention_fingerprint
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
|
||||||
|
dispatch_orchestrator_queue_attention(&api);
|
||||||
return Ok(Json(workspace_orchestrator_response(&api, "restored")));
|
return Ok(Json(workspace_orchestrator_response(&api, "restored")));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3734,6 +3763,10 @@ async fn scoped_start_workspace_orchestrator(
|
|||||||
result.diagnostics,
|
result.diagnostics,
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
*api.orchestrator_attention_fingerprint
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
|
||||||
|
dispatch_orchestrator_queue_attention(&api);
|
||||||
Ok(Json(workspace_orchestrator_response(&api, "created")))
|
Ok(Json(workspace_orchestrator_response(&api, "created")))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4726,7 +4759,6 @@ async fn scoped_restore_runtime_worker(
|
|||||||
.into());
|
.into());
|
||||||
}
|
}
|
||||||
assign_ticket_worker_from_lifecycle(&api, assignment, &runtime_id, &worker_id)?;
|
assign_ticket_worker_from_lifecycle(&api, assignment, &runtime_id, &worker_id)?;
|
||||||
dispatch_pending_ticket_notifications(&api, &workspace_id);
|
|
||||||
return Ok(Json(WorkerRestoreResponse {
|
return Ok(Json(WorkerRestoreResponse {
|
||||||
workspace_id,
|
workspace_id,
|
||||||
runtime_id,
|
runtime_id,
|
||||||
@@ -4747,7 +4779,6 @@ async fn scoped_restore_runtime_worker(
|
|||||||
if let Some(assignment) = assignment_request.as_ref() {
|
if let Some(assignment) = assignment_request.as_ref() {
|
||||||
assign_ticket_worker_from_lifecycle(&api, assignment, &runtime_id, &worker_id)?;
|
assign_ticket_worker_from_lifecycle(&api, assignment, &runtime_id, &worker_id)?;
|
||||||
}
|
}
|
||||||
dispatch_pending_ticket_notifications(&api, &workspace_id);
|
|
||||||
Ok(response)
|
Ok(response)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -6420,7 +6451,6 @@ async fn create_runtime_worker(
|
|||||||
if let Some(assignment) = request.ticket_assignment.as_ref() {
|
if let Some(assignment) = request.ticket_assignment.as_ref() {
|
||||||
if let Some(worker) = existing_lifecycle_assignment_worker(&api, assignment, &runtime_id)? {
|
if let Some(worker) = existing_lifecycle_assignment_worker(&api, assignment, &runtime_id)? {
|
||||||
assign_ticket_worker_from_lifecycle(&api, assignment, &runtime_id, &worker.worker_id)?;
|
assign_ticket_worker_from_lifecycle(&api, assignment, &runtime_id, &worker.worker_id)?;
|
||||||
dispatch_pending_ticket_notifications(&api, api.workspace_id());
|
|
||||||
return Ok(Json(WorkerSpawnResult {
|
return Ok(Json(WorkerSpawnResult {
|
||||||
state: WorkerOperationState::Accepted,
|
state: WorkerOperationState::Accepted,
|
||||||
worker: Some(worker),
|
worker: Some(worker),
|
||||||
@@ -9885,7 +9915,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn authenticated_worker_ticket_mutation_routes_durable_assignment_notification() {
|
async fn authenticated_worker_ticket_mutation_notifies_current_assignment() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
let api = test_api(dir.path()).await;
|
let api = test_api(dir.path()).await;
|
||||||
let spawn = |name: &str| WorkerSpawnRequest {
|
let spawn = |name: &str| WorkerSpawnRequest {
|
||||||
@@ -9996,47 +10026,6 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert!(committed_event.attributes.contains_key("event_id"));
|
assert!(committed_event.attributes.contains_key("event_id"));
|
||||||
assert!(committed_event.attributes.contains_key("event_sequence"));
|
assert!(committed_event.attributes.contains_key("event_sequence"));
|
||||||
let event_sequence = committed_event
|
|
||||||
.attributes
|
|
||||||
.get("event_sequence")
|
|
||||||
.unwrap()
|
|
||||||
.parse::<i64>()
|
|
||||||
.unwrap();
|
|
||||||
let response = build_router(api.clone())
|
|
||||||
.oneshot(
|
|
||||||
Request::builder()
|
|
||||||
.method("GET")
|
|
||||||
.uri(format!(
|
|
||||||
"/api/w/{TEST_WORKSPACE_ID}/tickets/{}/record",
|
|
||||||
ticket_ref.id
|
|
||||||
))
|
|
||||||
.header("x-yoi-runtime-id", EMBEDDED_WORKER_RUNTIME_ID)
|
|
||||||
.header("x-yoi-worker-id", &source_worker.worker_id)
|
|
||||||
.body(Body::empty())
|
|
||||||
.unwrap(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(response.status(), StatusCode::OK);
|
|
||||||
assert_eq!(
|
|
||||||
api.store
|
|
||||||
.get_ticket_notification_cursor(
|
|
||||||
TEST_WORKSPACE_ID,
|
|
||||||
&ticket_ref.id,
|
|
||||||
EMBEDDED_WORKER_RUNTIME_ID,
|
|
||||||
&source_worker.worker_id,
|
|
||||||
)
|
|
||||||
.unwrap(),
|
|
||||||
Some(event_sequence)
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
api.store
|
|
||||||
.list_pending_ticket_notification_deliveries(TEST_WORKSPACE_ID, 10)
|
|
||||||
.unwrap()
|
|
||||||
.is_empty(),
|
|
||||||
"accepted Runtime system input must complete the outbox delivery"
|
|
||||||
);
|
|
||||||
|
|
||||||
let Json(stale_report) = execute_worker_ticket_test_operation(
|
let Json(stale_report) = execute_worker_ticket_test_operation(
|
||||||
State(api.clone()),
|
State(api.clone()),
|
||||||
AxumPath(ScopedWorkspacePath {
|
AxumPath(ScopedWorkspacePath {
|
||||||
@@ -10132,7 +10121,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn queued_ticket_mutation_targets_current_orchestrator() {
|
async fn queued_ticket_mutation_succeeds_without_orchestrator() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
let api = test_api(dir.path()).await;
|
let api = test_api(dir.path()).await;
|
||||||
let source = api
|
let source = api
|
||||||
@@ -10163,41 +10152,6 @@ mod tests {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
.worker
|
.worker
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let orchestrator = api
|
|
||||||
.runtime
|
|
||||||
.spawn_worker(
|
|
||||||
EMBEDDED_WORKER_RUNTIME_ID,
|
|
||||||
WorkerSpawnRequest {
|
|
||||||
requested_worker_name: Some("workspace-orchestrator".to_string()),
|
|
||||||
intent: WorkerSpawnIntent::WorkspaceOrchestrator,
|
|
||||||
acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted {
|
|
||||||
expected_segments: 0,
|
|
||||||
},
|
|
||||||
profile: ProfileSelector::Builtin("builtin:orchestrator".to_string()),
|
|
||||||
ticket_assignment: None,
|
|
||||||
initial_input: None,
|
|
||||||
working_directory_request: None,
|
|
||||||
resolved_working_directory_request: None,
|
|
||||||
resolved_working_directory: None,
|
|
||||||
resolved_config_bundle: None,
|
|
||||||
resolved_workspace_api: Some(test_worker_workspace_api(
|
|
||||||
EMBEDDED_WORKER_RUNTIME_ID,
|
|
||||||
)),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap()
|
|
||||||
.worker
|
|
||||||
.unwrap();
|
|
||||||
api.runtime
|
|
||||||
.stop_worker(
|
|
||||||
EMBEDDED_WORKER_RUNTIME_ID,
|
|
||||||
&orchestrator.worker_id,
|
|
||||||
WorkerLifecycleRequest {
|
|
||||||
reason: Some("test pending delivery".to_string()),
|
|
||||||
ticket_assignment: None,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let backend = browser_ticket_backend(&api).unwrap();
|
let backend = browser_ticket_backend(&api).unwrap();
|
||||||
let mut input = ticket::NewTicket::new("Queued notification");
|
let mut input = ticket::NewTicket::new("Queued notification");
|
||||||
input.workflow_state = Some(TicketWorkflowState::Queued);
|
input.workflow_state = Some(TicketWorkflowState::Queued);
|
||||||
@@ -10224,16 +10178,82 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
}
|
||||||
api.store
|
|
||||||
.count_ticket_notification_deliveries_for_recipient(
|
#[tokio::test]
|
||||||
TEST_WORKSPACE_ID,
|
async fn orchestrator_running_to_idle_recovers_queued_ticket_without_notification_memory() {
|
||||||
&ticket_ref.id,
|
let dir = tempfile::tempdir().unwrap();
|
||||||
EMBEDDED_WORKER_RUNTIME_ID,
|
let api = test_api(dir.path()).await;
|
||||||
&orchestrator.worker_id,
|
let backend = browser_ticket_backend(&api).unwrap();
|
||||||
|
let ticket_ref = backend
|
||||||
|
.create(ticket::NewTicket::new("Recover queued work"))
|
||||||
|
.unwrap();
|
||||||
|
backend
|
||||||
|
.mark_intake_ready(
|
||||||
|
TicketIdOrSlug::Id(ticket_ref.id.clone()),
|
||||||
|
ticket::TicketIntakeSummary {
|
||||||
|
author: Some("intake".to_string()),
|
||||||
|
body: MarkdownText::new("Ready"),
|
||||||
|
references: Vec::new(),
|
||||||
|
},
|
||||||
|
ticket::TicketStateChange {
|
||||||
|
from: "planning".to_string(),
|
||||||
|
to: "ready".to_string(),
|
||||||
|
reason: "ready".to_string(),
|
||||||
|
author: Some("intake".to_string()),
|
||||||
|
body: MarkdownText::new("Ready"),
|
||||||
|
references: Vec::new(),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap();
|
||||||
1
|
backend
|
||||||
|
.queue_ready(TicketIdOrSlug::Id(ticket_ref.id.clone()), "browser-user")
|
||||||
|
.unwrap();
|
||||||
|
*api.orchestrator_attention_fingerprint.lock().unwrap() = Some(ticket_ref.id.clone());
|
||||||
|
|
||||||
|
let Json(started) = scoped_start_workspace_orchestrator(
|
||||||
|
State(api.clone()),
|
||||||
|
AxumPath(ScopedWorkspacePath {
|
||||||
|
workspace_id: TEST_WORKSPACE_ID.to_string(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(started.online);
|
||||||
|
assert_eq!(
|
||||||
|
api.orchestrator_attention_fingerprint
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some(ticket_ref.id.as_str())
|
||||||
|
);
|
||||||
|
*api.orchestrator_attention_fingerprint.lock().unwrap() = None;
|
||||||
|
let worker_id = started.worker.as_ref().unwrap().worker_id.clone();
|
||||||
|
maybe_dispatch_orchestrator_turn_end(
|
||||||
|
&api,
|
||||||
|
&worker_id,
|
||||||
|
Some(protocol::subscription::SubscriptionWorkerState::Idle),
|
||||||
|
protocol::subscription::SubscriptionWorkerState::Idle,
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
api.orchestrator_attention_fingerprint
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
maybe_dispatch_orchestrator_turn_end(
|
||||||
|
&api,
|
||||||
|
&worker_id,
|
||||||
|
Some(protocol::subscription::SubscriptionWorkerState::Running),
|
||||||
|
protocol::subscription::SubscriptionWorkerState::Idle,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
api.orchestrator_attention_fingerprint
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some(ticket_ref.id.as_str())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -10551,7 +10571,6 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(queued.state, "queued");
|
assert_eq!(queued.state, "queued");
|
||||||
assert_eq!(queued.queued_by.as_deref(), Some("browser-user"));
|
assert_eq!(queued.queued_by.as_deref(), Some("browser-user"));
|
||||||
|
|
||||||
let Json(reviewed) = scoped_review_ticket(
|
let Json(reviewed) = scoped_review_ticket(
|
||||||
State(api.clone()),
|
State(api.clone()),
|
||||||
AxumPath(path()),
|
AxumPath(path()),
|
||||||
|
|||||||
@@ -122,6 +122,11 @@ const MIGRATIONS: &[Migration] = &[
|
|||||||
name: "remove per-Worker Workspace credentials",
|
name: "remove per-Worker Workspace credentials",
|
||||||
apply: remove_worker_workspace_credentials,
|
apply: remove_worker_workspace_credentials,
|
||||||
},
|
},
|
||||||
|
Migration {
|
||||||
|
version: 22,
|
||||||
|
name: "drop Ticket notification outbox",
|
||||||
|
apply: drop_ticket_notification_tables,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
struct Migration {
|
struct Migration {
|
||||||
@@ -296,31 +301,6 @@ pub struct TicketWorkerAssignmentUpdate {
|
|||||||
pub previous: Option<TicketWorkerAssignmentRecord>,
|
pub previous: Option<TicketWorkerAssignmentRecord>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
|
||||||
pub struct TicketNotificationRecipient {
|
|
||||||
pub runtime_id: String,
|
|
||||||
pub worker_id: String,
|
|
||||||
pub recipient_kind: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
|
||||||
pub struct TicketNotificationDeliveryRecord {
|
|
||||||
pub notification_id: String,
|
|
||||||
pub workspace_id: String,
|
|
||||||
pub ticket_id: String,
|
|
||||||
pub event_sequence: i64,
|
|
||||||
pub event_kind: String,
|
|
||||||
pub source_operation_kind: String,
|
|
||||||
pub source_actor_role: String,
|
|
||||||
pub source_assignment_id: Option<String>,
|
|
||||||
pub source_runtime_id: String,
|
|
||||||
pub source_worker_id: String,
|
|
||||||
pub recipient_runtime_id: String,
|
|
||||||
pub recipient_worker_id: String,
|
|
||||||
pub recipient_kind: String,
|
|
||||||
pub attempts: i64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct WorkdirRegistryRecord {
|
pub struct WorkdirRegistryRecord {
|
||||||
pub workspace_id: String,
|
pub workspace_id: String,
|
||||||
@@ -625,70 +605,6 @@ pub trait ControlPlaneStore: Send + Sync {
|
|||||||
limit: usize,
|
limit: usize,
|
||||||
) -> Result<Vec<TicketWorkerAssignmentEventRecord>>;
|
) -> Result<Vec<TicketWorkerAssignmentEventRecord>>;
|
||||||
|
|
||||||
fn enqueue_ticket_notification(
|
|
||||||
&self,
|
|
||||||
notification_id: &str,
|
|
||||||
workspace_id: &str,
|
|
||||||
ticket_id: &str,
|
|
||||||
event_sequence: i64,
|
|
||||||
source_runtime_id: &str,
|
|
||||||
source_worker_id: &str,
|
|
||||||
previous_state: &str,
|
|
||||||
current_state: &str,
|
|
||||||
created_at: &str,
|
|
||||||
recipients: &[TicketNotificationRecipient],
|
|
||||||
) -> Result<()>;
|
|
||||||
fn list_pending_ticket_notification_deliveries(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
limit: usize,
|
|
||||||
) -> Result<Vec<TicketNotificationDeliveryRecord>>;
|
|
||||||
fn count_ticket_notification_deliveries_for_recipient(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
ticket_id: &str,
|
|
||||||
runtime_id: &str,
|
|
||||||
worker_id: &str,
|
|
||||||
) -> Result<usize>;
|
|
||||||
fn mark_ticket_notification_delivered(
|
|
||||||
&self,
|
|
||||||
notification_id: &str,
|
|
||||||
recipient_runtime_id: &str,
|
|
||||||
recipient_worker_id: &str,
|
|
||||||
delivered_at: &str,
|
|
||||||
) -> Result<()>;
|
|
||||||
fn mark_ticket_notification_failed(
|
|
||||||
&self,
|
|
||||||
notification_id: &str,
|
|
||||||
recipient_runtime_id: &str,
|
|
||||||
recipient_worker_id: &str,
|
|
||||||
error: &str,
|
|
||||||
) -> Result<()>;
|
|
||||||
fn reroute_ticket_notification_delivery(
|
|
||||||
&self,
|
|
||||||
notification_id: &str,
|
|
||||||
old_runtime_id: &str,
|
|
||||||
old_worker_id: &str,
|
|
||||||
new_runtime_id: &str,
|
|
||||||
new_worker_id: &str,
|
|
||||||
) -> Result<()>;
|
|
||||||
fn upsert_ticket_notification_cursor(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
ticket_id: &str,
|
|
||||||
runtime_id: &str,
|
|
||||||
worker_id: &str,
|
|
||||||
event_index: i64,
|
|
||||||
updated_at: &str,
|
|
||||||
) -> Result<()>;
|
|
||||||
fn get_ticket_notification_cursor(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
ticket_id: &str,
|
|
||||||
runtime_id: &str,
|
|
||||||
worker_id: &str,
|
|
||||||
) -> Result<Option<i64>>;
|
|
||||||
|
|
||||||
fn upsert_workdir_registry(&self, record: &WorkdirRegistryRecord) -> Result<()>;
|
fn upsert_workdir_registry(&self, record: &WorkdirRegistryRecord) -> Result<()>;
|
||||||
fn get_workdir_registry(
|
fn get_workdir_registry(
|
||||||
&self,
|
&self,
|
||||||
@@ -2183,239 +2099,6 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn enqueue_ticket_notification(
|
|
||||||
&self,
|
|
||||||
notification_id: &str,
|
|
||||||
workspace_id: &str,
|
|
||||||
ticket_id: &str,
|
|
||||||
event_sequence: i64,
|
|
||||||
source_runtime_id: &str,
|
|
||||||
source_worker_id: &str,
|
|
||||||
previous_state: &str,
|
|
||||||
current_state: &str,
|
|
||||||
created_at: &str,
|
|
||||||
recipients: &[TicketNotificationRecipient],
|
|
||||||
) -> Result<()> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
let tx = conn.unchecked_transaction()?;
|
|
||||||
tx.execute(
|
|
||||||
r#"INSERT INTO ticket_notification_outbox (
|
|
||||||
notification_id, workspace_id, ticket_id, event_sequence,
|
|
||||||
source_runtime_id, source_worker_id, previous_state, current_state, created_at
|
|
||||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"#,
|
|
||||||
params![
|
|
||||||
notification_id,
|
|
||||||
workspace_id,
|
|
||||||
ticket_id,
|
|
||||||
event_sequence,
|
|
||||||
source_runtime_id,
|
|
||||||
source_worker_id,
|
|
||||||
previous_state,
|
|
||||||
current_state,
|
|
||||||
created_at,
|
|
||||||
],
|
|
||||||
)?;
|
|
||||||
for recipient in recipients {
|
|
||||||
tx.execute(
|
|
||||||
r#"INSERT OR IGNORE INTO ticket_notification_deliveries (
|
|
||||||
notification_id, recipient_runtime_id, recipient_worker_id,
|
|
||||||
recipient_kind, attempts
|
|
||||||
) VALUES (?1, ?2, ?3, ?4, 0)"#,
|
|
||||||
params![
|
|
||||||
notification_id,
|
|
||||||
recipient.runtime_id,
|
|
||||||
recipient.worker_id,
|
|
||||||
recipient.recipient_kind,
|
|
||||||
],
|
|
||||||
)?;
|
|
||||||
}
|
|
||||||
tx.commit()?;
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn list_pending_ticket_notification_deliveries(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
limit: usize,
|
|
||||||
) -> Result<Vec<TicketNotificationDeliveryRecord>> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
let mut stmt = conn.prepare(
|
|
||||||
r#"SELECT o.notification_id, o.workspace_id, o.ticket_id, o.event_sequence,
|
|
||||||
o.event_kind, o.source_operation_kind, o.source_actor_role,
|
|
||||||
o.source_assignment_id, o.source_runtime_id, o.source_worker_id,
|
|
||||||
d.recipient_runtime_id, d.recipient_worker_id, d.recipient_kind, d.attempts
|
|
||||||
FROM ticket_notification_deliveries AS d
|
|
||||||
JOIN ticket_notification_outbox AS o ON o.notification_id = d.notification_id
|
|
||||||
WHERE o.workspace_id = ?1 AND d.delivered_at IS NULL
|
|
||||||
ORDER BY o.created_at ASC, o.notification_id ASC
|
|
||||||
LIMIT ?2"#,
|
|
||||||
)?;
|
|
||||||
let rows = stmt.query_map(params![workspace_id, limit as i64], |row| {
|
|
||||||
Ok(TicketNotificationDeliveryRecord {
|
|
||||||
notification_id: row.get(0)?,
|
|
||||||
workspace_id: row.get(1)?,
|
|
||||||
ticket_id: row.get(2)?,
|
|
||||||
event_sequence: row.get(3)?,
|
|
||||||
event_kind: row.get(4)?,
|
|
||||||
source_operation_kind: row.get(5)?,
|
|
||||||
source_actor_role: row.get(6)?,
|
|
||||||
source_assignment_id: row.get(7)?,
|
|
||||||
source_runtime_id: row.get(8)?,
|
|
||||||
source_worker_id: row.get(9)?,
|
|
||||||
recipient_runtime_id: row.get(10)?,
|
|
||||||
recipient_worker_id: row.get(11)?,
|
|
||||||
recipient_kind: row.get(12)?,
|
|
||||||
attempts: row.get(13)?,
|
|
||||||
})
|
|
||||||
})?;
|
|
||||||
rows.collect::<std::result::Result<Vec<_>, _>>()
|
|
||||||
.map_err(Error::from)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn count_ticket_notification_deliveries_for_recipient(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
ticket_id: &str,
|
|
||||||
runtime_id: &str,
|
|
||||||
worker_id: &str,
|
|
||||||
) -> Result<usize> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
let count = conn.query_row(
|
|
||||||
r#"SELECT COUNT(*)
|
|
||||||
FROM ticket_notification_deliveries AS d
|
|
||||||
JOIN ticket_notification_outbox AS o ON o.notification_id = d.notification_id
|
|
||||||
WHERE o.workspace_id = ?1 AND o.ticket_id = ?2
|
|
||||||
AND d.recipient_runtime_id = ?3 AND d.recipient_worker_id = ?4"#,
|
|
||||||
params![workspace_id, ticket_id, runtime_id, worker_id],
|
|
||||||
|row| row.get::<_, i64>(0),
|
|
||||||
)?;
|
|
||||||
Ok(count as usize)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn mark_ticket_notification_delivered(
|
|
||||||
&self,
|
|
||||||
notification_id: &str,
|
|
||||||
recipient_runtime_id: &str,
|
|
||||||
recipient_worker_id: &str,
|
|
||||||
delivered_at: &str,
|
|
||||||
) -> Result<()> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
conn.execute(
|
|
||||||
r#"UPDATE ticket_notification_deliveries
|
|
||||||
SET delivered_at = ?4, last_error = NULL, attempts = attempts + 1
|
|
||||||
WHERE notification_id = ?1 AND recipient_runtime_id = ?2 AND recipient_worker_id = ?3"#,
|
|
||||||
params![notification_id, recipient_runtime_id, recipient_worker_id, delivered_at],
|
|
||||||
)?;
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn mark_ticket_notification_failed(
|
|
||||||
&self,
|
|
||||||
notification_id: &str,
|
|
||||||
recipient_runtime_id: &str,
|
|
||||||
recipient_worker_id: &str,
|
|
||||||
error: &str,
|
|
||||||
) -> Result<()> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
conn.execute(
|
|
||||||
r#"UPDATE ticket_notification_deliveries
|
|
||||||
SET last_error = ?4, attempts = attempts + 1
|
|
||||||
WHERE notification_id = ?1 AND recipient_runtime_id = ?2 AND recipient_worker_id = ?3"#,
|
|
||||||
params![notification_id, recipient_runtime_id, recipient_worker_id, error],
|
|
||||||
)?;
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn reroute_ticket_notification_delivery(
|
|
||||||
&self,
|
|
||||||
notification_id: &str,
|
|
||||||
old_runtime_id: &str,
|
|
||||||
old_worker_id: &str,
|
|
||||||
new_runtime_id: &str,
|
|
||||||
new_worker_id: &str,
|
|
||||||
) -> Result<()> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
let tx = conn.unchecked_transaction()?;
|
|
||||||
let recipient_kind: Option<String> = tx
|
|
||||||
.query_row(
|
|
||||||
r#"SELECT recipient_kind FROM ticket_notification_deliveries
|
|
||||||
WHERE notification_id = ?1 AND recipient_runtime_id = ?2 AND recipient_worker_id = ?3"#,
|
|
||||||
params![notification_id, old_runtime_id, old_worker_id],
|
|
||||||
|row| row.get(0),
|
|
||||||
)
|
|
||||||
.optional()?;
|
|
||||||
if let Some(recipient_kind) = recipient_kind {
|
|
||||||
tx.execute(
|
|
||||||
r#"INSERT OR IGNORE INTO ticket_notification_deliveries (
|
|
||||||
notification_id, recipient_runtime_id, recipient_worker_id, recipient_kind, attempts
|
|
||||||
) VALUES (?1, ?2, ?3, ?4, 0)"#,
|
|
||||||
params![notification_id, new_runtime_id, new_worker_id, recipient_kind],
|
|
||||||
)?;
|
|
||||||
tx.execute(
|
|
||||||
r#"DELETE FROM ticket_notification_deliveries
|
|
||||||
WHERE notification_id = ?1 AND recipient_runtime_id = ?2 AND recipient_worker_id = ?3"#,
|
|
||||||
params![notification_id, old_runtime_id, old_worker_id],
|
|
||||||
)?;
|
|
||||||
}
|
|
||||||
tx.commit()?;
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn upsert_ticket_notification_cursor(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
ticket_id: &str,
|
|
||||||
runtime_id: &str,
|
|
||||||
worker_id: &str,
|
|
||||||
event_index: i64,
|
|
||||||
updated_at: &str,
|
|
||||||
) -> Result<()> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
conn.execute(
|
|
||||||
r#"INSERT INTO ticket_notification_cursors (
|
|
||||||
workspace_id, ticket_id, runtime_id, worker_id, last_event_index, updated_at
|
|
||||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
|
||||||
ON CONFLICT(workspace_id, ticket_id, runtime_id, worker_id) DO UPDATE SET
|
|
||||||
last_event_index = MAX(last_event_index, excluded.last_event_index),
|
|
||||||
updated_at = excluded.updated_at"#,
|
|
||||||
params![
|
|
||||||
workspace_id,
|
|
||||||
ticket_id,
|
|
||||||
runtime_id,
|
|
||||||
worker_id,
|
|
||||||
event_index,
|
|
||||||
updated_at
|
|
||||||
],
|
|
||||||
)?;
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_ticket_notification_cursor(
|
|
||||||
&self,
|
|
||||||
workspace_id: &str,
|
|
||||||
ticket_id: &str,
|
|
||||||
runtime_id: &str,
|
|
||||||
worker_id: &str,
|
|
||||||
) -> Result<Option<i64>> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
conn.query_row(
|
|
||||||
r#"SELECT last_event_index FROM ticket_notification_cursors
|
|
||||||
WHERE workspace_id = ?1 AND ticket_id = ?2 AND runtime_id = ?3 AND worker_id = ?4"#,
|
|
||||||
params![workspace_id, ticket_id, runtime_id, worker_id],
|
|
||||||
|row| row.get(0),
|
|
||||||
)
|
|
||||||
.optional()
|
|
||||||
.map_err(Error::from)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn upsert_workdir_registry(&self, record: &WorkdirRegistryRecord) -> Result<()> {
|
fn upsert_workdir_registry(&self, record: &WorkdirRegistryRecord) -> Result<()> {
|
||||||
self.with_conn(|conn| {
|
self.with_conn(|conn| {
|
||||||
conn.execute(
|
conn.execute(
|
||||||
@@ -3343,6 +3026,17 @@ fn remove_worker_workspace_credentials(conn: &Connection) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn drop_ticket_notification_tables(conn: &Connection) -> Result<()> {
|
||||||
|
conn.execute_batch(
|
||||||
|
r#"
|
||||||
|
DROP TABLE IF EXISTS ticket_notification_cursors;
|
||||||
|
DROP TABLE IF EXISTS ticket_notification_deliveries;
|
||||||
|
DROP TABLE IF EXISTS ticket_notification_outbox;
|
||||||
|
"#,
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn create_objective_event_tables(conn: &Connection) -> Result<()> {
|
fn create_objective_event_tables(conn: &Connection) -> Result<()> {
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
r#"
|
r#"
|
||||||
@@ -4014,6 +3708,9 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
|
|||||||
"ticket_targets",
|
"ticket_targets",
|
||||||
"ticket_target_paths",
|
"ticket_target_paths",
|
||||||
"ticket_worker_links",
|
"ticket_worker_links",
|
||||||
|
"ticket_notification_outbox",
|
||||||
|
"ticket_notification_deliveries",
|
||||||
|
"ticket_notification_cursors",
|
||||||
] {
|
] {
|
||||||
assert!(!table_exists(&conn, table).unwrap(), "{table} still exists");
|
assert!(!table_exists(&conn, table).unwrap(), "{table} still exists");
|
||||||
}
|
}
|
||||||
@@ -4025,7 +3722,7 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
|
|||||||
let db = dir.path().join("control-plane.sqlite");
|
let db = dir.path().join("control-plane.sqlite");
|
||||||
let store = SqliteWorkspaceStore::open(&db).unwrap();
|
let store = SqliteWorkspaceStore::open(&db).unwrap();
|
||||||
|
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 21);
|
assert_eq!(store.schema_version().await.unwrap(), 22);
|
||||||
assert!(
|
assert!(
|
||||||
!store
|
!store
|
||||||
.with_conn(|conn| table_exists(conn, "worker_workspace_credentials"))
|
.with_conn(|conn| table_exists(conn, "worker_workspace_credentials"))
|
||||||
@@ -4042,7 +3739,7 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
|
|||||||
store.upsert_workspace(&record).await.unwrap();
|
store.upsert_workspace(&record).await.unwrap();
|
||||||
|
|
||||||
let reopened = SqliteWorkspaceStore::open(&db).unwrap();
|
let reopened = SqliteWorkspaceStore::open(&db).unwrap();
|
||||||
assert_eq!(reopened.schema_version().await.unwrap(), 21);
|
assert_eq!(reopened.schema_version().await.unwrap(), 22);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
reopened.get_workspace("local-dev").await.unwrap(),
|
reopened.get_workspace("local-dev").await.unwrap(),
|
||||||
Some(record)
|
Some(record)
|
||||||
@@ -4288,95 +3985,6 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn notification_outbox_is_durable() {
|
|
||||||
let dir = tempfile::tempdir().unwrap();
|
|
||||||
let db = dir.path().join("server.db");
|
|
||||||
let store = SqliteWorkspaceStore::open(&db).unwrap();
|
|
||||||
store
|
|
||||||
.upsert_workspace(&WorkspaceRecord {
|
|
||||||
workspace_id: "workspace-a".to_string(),
|
|
||||||
owner_account_id: None,
|
|
||||||
display_name: "Workspace A".to_string(),
|
|
||||||
state: "active".to_string(),
|
|
||||||
created_at: "2026-07-31T00:00:00Z".to_string(),
|
|
||||||
updated_at: "2026-07-31T00:00:00Z".to_string(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
store
|
|
||||||
.enqueue_ticket_notification(
|
|
||||||
"notification-1",
|
|
||||||
"workspace-a",
|
|
||||||
"ticket-1",
|
|
||||||
4,
|
|
||||||
"runtime-1",
|
|
||||||
"worker-1",
|
|
||||||
"queued",
|
|
||||||
"inprogress",
|
|
||||||
"2026-07-31T00:00:02Z",
|
|
||||||
&[TicketNotificationRecipient {
|
|
||||||
runtime_id: "runtime-1".to_string(),
|
|
||||||
worker_id: "worker-2".to_string(),
|
|
||||||
recipient_kind: "assigned".to_string(),
|
|
||||||
}],
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
drop(store);
|
|
||||||
let store = SqliteWorkspaceStore::open(&db).unwrap();
|
|
||||||
let pending = store
|
|
||||||
.list_pending_ticket_notification_deliveries("workspace-a", 10)
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(pending.len(), 1);
|
|
||||||
assert_eq!(pending[0].event_sequence, 4);
|
|
||||||
store
|
|
||||||
.reroute_ticket_notification_delivery(
|
|
||||||
"notification-1",
|
|
||||||
"runtime-1",
|
|
||||||
"worker-2",
|
|
||||||
"runtime-2",
|
|
||||||
"worker-3",
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
store
|
|
||||||
.count_ticket_notification_deliveries_for_recipient(
|
|
||||||
"workspace-a",
|
|
||||||
"ticket-1",
|
|
||||||
"runtime-1",
|
|
||||||
"worker-2",
|
|
||||||
)
|
|
||||||
.unwrap(),
|
|
||||||
0
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
store
|
|
||||||
.count_ticket_notification_deliveries_for_recipient(
|
|
||||||
"workspace-a",
|
|
||||||
"ticket-1",
|
|
||||||
"runtime-2",
|
|
||||||
"worker-3",
|
|
||||||
)
|
|
||||||
.unwrap(),
|
|
||||||
1
|
|
||||||
);
|
|
||||||
store
|
|
||||||
.mark_ticket_notification_delivered(
|
|
||||||
"notification-1",
|
|
||||||
"runtime-2",
|
|
||||||
"worker-3",
|
|
||||||
"2026-07-31T00:00:03Z",
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
assert!(
|
|
||||||
store
|
|
||||||
.list_pending_ticket_notification_deliveries("workspace-a", 10)
|
|
||||||
.unwrap()
|
|
||||||
.is_empty()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fresh_schema_matches_workspace_db_v0_boundaries() {
|
fn fresh_schema_matches_workspace_db_v0_boundaries() {
|
||||||
let conn = Connection::open_in_memory().unwrap();
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
@@ -4427,6 +4035,9 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
|
|||||||
"ticket_targets",
|
"ticket_targets",
|
||||||
"ticket_target_paths",
|
"ticket_target_paths",
|
||||||
"ticket_worker_links",
|
"ticket_worker_links",
|
||||||
|
"ticket_notification_outbox",
|
||||||
|
"ticket_notification_deliveries",
|
||||||
|
"ticket_notification_cursors",
|
||||||
] {
|
] {
|
||||||
assert!(
|
assert!(
|
||||||
!tables.contains(forbidden),
|
!tables.contains(forbidden),
|
||||||
@@ -4579,7 +4190,7 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let store = SqliteWorkspaceStore::from_connection(conn).unwrap();
|
let store = SqliteWorkspaceStore::from_connection(conn).unwrap();
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 21);
|
assert_eq!(store.schema_version().await.unwrap(), 22);
|
||||||
|
|
||||||
store
|
store
|
||||||
.with_conn(|conn| {
|
.with_conn(|conn| {
|
||||||
@@ -4616,6 +4227,9 @@ CREATE TABLE ticket_worker_links (ticket_id TEXT, worker_ref_key TEXT);
|
|||||||
"ticket_targets",
|
"ticket_targets",
|
||||||
"ticket_target_paths",
|
"ticket_target_paths",
|
||||||
"ticket_worker_links",
|
"ticket_worker_links",
|
||||||
|
"ticket_notification_outbox",
|
||||||
|
"ticket_notification_deliveries",
|
||||||
|
"ticket_notification_cursors",
|
||||||
] {
|
] {
|
||||||
assert!(
|
assert!(
|
||||||
!tables.contains(forbidden),
|
!tables.contains(forbidden),
|
||||||
@@ -4765,7 +4379,7 @@ CREATE TABLE ticket_assignment_operations (
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn repository_records_round_trip() {
|
async fn repository_records_round_trip() {
|
||||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 21);
|
assert_eq!(store.schema_version().await.unwrap(), 22);
|
||||||
let workspace = WorkspaceRecord {
|
let workspace = WorkspaceRecord {
|
||||||
workspace_id: "local-dev".to_string(),
|
workspace_id: "local-dev".to_string(),
|
||||||
owner_account_id: None,
|
owner_account_id: None,
|
||||||
@@ -4803,7 +4417,7 @@ CREATE TABLE ticket_assignment_operations (
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn memory_authority_records_round_trip_and_close_staging() {
|
async fn memory_authority_records_round_trip_and_close_staging() {
|
||||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 21);
|
assert_eq!(store.schema_version().await.unwrap(), 22);
|
||||||
let workspace = WorkspaceRecord {
|
let workspace = WorkspaceRecord {
|
||||||
workspace_id: "local-dev".to_string(),
|
workspace_id: "local-dev".to_string(),
|
||||||
owner_account_id: None,
|
owner_account_id: None,
|
||||||
@@ -4981,7 +4595,7 @@ CREATE TABLE ticket_assignment_operations (
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn account_and_login_records_round_trip() {
|
async fn account_and_login_records_round_trip() {
|
||||||
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
let store = SqliteWorkspaceStore::in_memory().unwrap();
|
||||||
assert_eq!(store.schema_version().await.unwrap(), 21);
|
assert_eq!(store.schema_version().await.unwrap(), 22);
|
||||||
let now = "2026-07-22T00:00:00Z".to_string();
|
let now = "2026-07-22T00:00:00Z".to_string();
|
||||||
let account = AccountRecord {
|
let account = AccountRecord {
|
||||||
account_id: "acct-user-alice".to_string(),
|
account_id: "acct-user-alice".to_string(),
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
Workspace Orchestrator attention: authoritative Ticket state still contains queued work after the previous turn or after Server recovery.
|
||||||
|
|
||||||
|
Workspace: {{workspace_id}}
|
||||||
|
Remaining queued Tickets (bounded):
|
||||||
|
{{ticket_lines}}
|
||||||
|
{{omitted_line}}
|
||||||
|
Reread the listed Tickets, their relations, orchestration plans, current assignments, Workers, and Workdirs before acting. Continue only work already authorized by the human `ready -> queued` transition. Do not drain the queue automatically and do not create duplicate assignments, Workers, Workdirs, or merges. If no Ticket is currently actionable, record the durable waiting reason on the authoritative Ticket or orchestration plan and stop. Before implementation side effects, record the accepted `queued -> inprogress` transition.
|
||||||
@@ -1,4 +1,7 @@
|
|||||||
import { parseSigilSegments } from "./composer-command.ts";
|
import {
|
||||||
|
buildComposerRequest,
|
||||||
|
parseSigilSegments,
|
||||||
|
} from "./composer-command.ts";
|
||||||
|
|
||||||
declare const Deno: { test(name: string, fn: () => void): void };
|
declare const Deno: { test(name: string, fn: () => void): void };
|
||||||
|
|
||||||
@@ -21,3 +24,14 @@ Deno.test("parseSigilSegments leaves hash sigils as plain text", () => {
|
|||||||
content: "ask #memory",
|
content: "ask #memory",
|
||||||
}]);
|
}]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Deno.test("notify command exposes the operation instead of a System-role input", () => {
|
||||||
|
assertEquals(buildComposerRequest(":notify reread the Ticket"), {
|
||||||
|
ok: true,
|
||||||
|
request: { kind: "notify", content: "reread the Ticket" },
|
||||||
|
});
|
||||||
|
assertEquals(buildComposerRequest(":system reread the Ticket"), {
|
||||||
|
ok: false,
|
||||||
|
message: "Unknown command: system. Type :help for available commands.",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import type { Segment } from "$lib/generated/protocol";
|
|||||||
|
|
||||||
export type WorkerConsoleInputKind =
|
export type WorkerConsoleInputKind =
|
||||||
| "user"
|
| "user"
|
||||||
| "system"
|
| "notify"
|
||||||
| "compact"
|
| "compact"
|
||||||
| "list_rewind_targets"
|
| "list_rewind_targets"
|
||||||
| "register_peer";
|
| "register_peer";
|
||||||
@@ -53,9 +53,9 @@ const COMMANDS: Record<string, CommandSpec> = {
|
|||||||
description:
|
description:
|
||||||
"Register another existing Worker as a reciprocal metadata peer.",
|
"Register another existing Worker as a reciprocal metadata peer.",
|
||||||
},
|
},
|
||||||
system: {
|
notify: {
|
||||||
usage: ":system <message>",
|
usage: ":notify <message>",
|
||||||
description: "Send an agent-visible system notification to the Worker.",
|
description: "Send an agent-visible notification to the Worker.",
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -125,12 +125,12 @@ function buildColonCommand(commandLine: string): ComposerCommandResult {
|
|||||||
request: { kind: "register_peer", content: argv[0] },
|
request: { kind: "register_peer", content: argv[0] },
|
||||||
notice: `peer metadata registration requested with \`${argv[0]}\``,
|
notice: `peer metadata registration requested with \`${argv[0]}\``,
|
||||||
};
|
};
|
||||||
case "system": {
|
case "notify": {
|
||||||
const message = commandLine.trim().slice(name.length).trimStart();
|
const message = commandLine.trim().slice(name.length).trimStart();
|
||||||
if (!message) {
|
if (!message) {
|
||||||
return invalidUsage("system");
|
return invalidUsage("notify");
|
||||||
}
|
}
|
||||||
return { ok: true, request: { kind: "system", content: message } };
|
return { ok: true, request: { kind: "notify", content: message } };
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return {
|
return {
|
||||||
@@ -158,7 +158,7 @@ function helpCommand(argv: string[]): ComposerCommandResult {
|
|||||||
notice: `command: ${name} — usage: ${spec.usage}. ${spec.description}`,
|
notice: `command: ${name} — usage: ${spec.usage}. ${spec.description}`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
const list = ["help", "noop", "compact", "rewind", "peer", "system"]
|
const list = ["help", "noop", "compact", "rewind", "peer", "notify"]
|
||||||
.map((command) => `${command} (${COMMANDS[command].usage})`)
|
.map((command) => `${command} (${COMMANDS[command].usage})`)
|
||||||
.join(", ");
|
.join(", ");
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ export const COLON_COMMAND_COMPLETIONS: ComposerCompletionEntry[] = [
|
|||||||
{ value: "rewind", description: "List rewind targets" },
|
{ value: "rewind", description: "List rewind targets" },
|
||||||
{ value: "rollback", description: "Alias for rewind" },
|
{ value: "rollback", description: "Alias for rewind" },
|
||||||
{ value: "peer", description: "Register metadata peer" },
|
{ value: "peer", description: "Register metadata peer" },
|
||||||
{ value: "system", description: "Send system notification" },
|
{ value: "notify", description: "Send Worker notification" },
|
||||||
];
|
];
|
||||||
|
|
||||||
export function completionTokenAt(
|
export function completionTokenAt(
|
||||||
|
|||||||
+1
-1
@@ -428,7 +428,7 @@
|
|||||||
],
|
],
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
case "system":
|
case "notify":
|
||||||
return {
|
return {
|
||||||
method: "notify",
|
method: "notify",
|
||||||
params: { message: request.content, auto_run: true },
|
params: { message: request.content, auto_run: true },
|
||||||
|
|||||||
Reference in New Issue
Block a user