Compare commits
28
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8fe0525295 | ||
|
|
e6703ed18a | ||
|
|
ee75272917 | ||
|
|
620ecbafcb | ||
|
|
cf394403a6 | ||
|
|
8f0d7fa3c0 | ||
|
|
cc7266c801 | ||
|
|
e0ac732769 | ||
|
|
c79db24016 | ||
|
|
485918ebe3 | ||
|
|
d9f399b97b | ||
|
|
b8a5c60ff9 | ||
|
|
2d4b93dde7 | ||
|
|
bd893f271a | ||
|
|
2c8e617b2a | ||
|
|
8207e560e9 | ||
|
|
22b6f4e71d | ||
|
|
5c5921fcd2 | ||
|
|
a2781f57e9 | ||
|
|
fd391ef705 | ||
|
|
36df79e561 | ||
|
|
636dc14616 | ||
|
|
fb115fbb7e | ||
|
|
ba009c0a20 | ||
|
|
50726e4cf3 | ||
|
|
f98a123e40 | ||
|
|
dd2ca54874 | ||
|
|
da90ac74b4 |
@@ -50,6 +50,9 @@ pub enum EngineError {
|
|||||||
/// Config warnings (unsupported options)
|
/// Config warnings (unsupported options)
|
||||||
#[error("Config warnings: {}", .0.iter().map(|w| w.to_string()).collect::<Vec<_>>().join(", "))]
|
#[error("Config warnings: {}", .0.iter().map(|w| w.to_string()).collect::<Vec<_>>().join(", "))]
|
||||||
ConfigWarnings(Vec<ConfigWarning>),
|
ConfigWarnings(Vec<ConfigWarning>),
|
||||||
|
/// A durable-history observer rejected an item before it entered history.
|
||||||
|
#[error("History append failed: {0}")]
|
||||||
|
HistoryAppend(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Tool registration error
|
/// Tool registration error
|
||||||
@@ -222,10 +225,10 @@ pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
|
|||||||
/// truncation have been applied — i.e. on the same data that
|
/// truncation have been applied — i.e. on the same data that
|
||||||
/// enters history.
|
/// enters history.
|
||||||
tool_result_cbs: Vec<Box<dyn Fn(&ToolResult) + Send + Sync>>,
|
tool_result_cbs: Vec<Box<dyn Fn(&ToolResult) + Send + Sync>>,
|
||||||
/// History-append callbacks. Invoked for non-streamed items when they
|
/// History-append callbacks. Invoked before non-streamed items enter
|
||||||
/// are appended to persistent engine history, so upper layers can
|
/// engine history. An error rejects the item and aborts the turn, allowing
|
||||||
/// broadcast those items using history itself as the source of truth.
|
/// upper layers to make durable storage the commit gate.
|
||||||
history_append_cbs: Vec<Box<dyn Fn(&Item) + Send + Sync>>,
|
history_append_cbs: Vec<Box<dyn Fn(&Item) -> Result<(), String> + Send + Sync>>,
|
||||||
/// Request configuration (max_tokens, temperature, etc.)
|
/// Request configuration (max_tokens, temperature, etc.)
|
||||||
request_config: RequestConfig,
|
request_config: RequestConfig,
|
||||||
/// Whether the previous run was interrupted
|
/// Whether the previous run was interrupted
|
||||||
@@ -498,23 +501,31 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register a callback invoked for items appended directly to engine
|
/// Register a fallible callback invoked before an item enters engine
|
||||||
/// history outside streaming timeline callbacks.
|
/// history. Returning an error rejects that item and aborts the turn.
|
||||||
pub fn on_history_append(&mut self, callback: impl Fn(&Item) + Send + Sync + 'static) {
|
pub fn on_history_append(
|
||||||
|
&mut self,
|
||||||
|
callback: impl Fn(&Item) -> Result<(), String> + Send + Sync + 'static,
|
||||||
|
) {
|
||||||
self.history_append_cbs.push(Box::new(callback));
|
self.history_append_cbs.push(Box::new(callback));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn emit_history_append(&self, item: &Item) {
|
fn emit_history_append(&self, item: &Item) -> Result<(), EngineError> {
|
||||||
for cb in &self.history_append_cbs {
|
for cb in &self.history_append_cbs {
|
||||||
cb(item);
|
cb(item).map_err(EngineError::HistoryAppend)?;
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn append_history_items(&mut self, items: impl IntoIterator<Item = Item>) {
|
fn append_history_items(
|
||||||
|
&mut self,
|
||||||
|
items: impl IntoIterator<Item = Item>,
|
||||||
|
) -> Result<(), EngineError> {
|
||||||
for item in items {
|
for item in items {
|
||||||
self.emit_history_append(&item);
|
self.emit_history_append(&item)?;
|
||||||
self.history.push(item);
|
self.history.push(item);
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn request_trace_payload(&self, request: &Request) -> Value {
|
fn request_trace_payload(&self, request: &Request) -> Value {
|
||||||
@@ -1125,9 +1136,13 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
// These are committed *before* the per-request clone so they
|
// These are committed *before* the per-request clone so they
|
||||||
// participate in the LLM request below and get persisted by
|
// participate in the LLM request below and get persisted by
|
||||||
// the caller that owns durable history.
|
// the caller that owns durable history.
|
||||||
let pending = self.interceptor.pending_history_appends().await;
|
let pending = self
|
||||||
|
.interceptor
|
||||||
|
.pending_history_appends()
|
||||||
|
.await
|
||||||
|
.map_err(EngineError::HistoryAppend)?;
|
||||||
if !pending.is_empty() {
|
if !pending.is_empty() {
|
||||||
self.append_history_items(pending);
|
self.append_history_items(pending)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clone the history into a per-request context. Everything
|
// Clone the history into a per-request context. Everything
|
||||||
@@ -1202,7 +1217,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
return Err(EngineError::Aborted(reason));
|
return Err(EngineError::Aborted(reason));
|
||||||
}
|
}
|
||||||
PreRequestAction::YieldWith(items) => {
|
PreRequestAction::YieldWith(items) => {
|
||||||
self.append_history_items(items.clone());
|
self.append_history_items(items.clone())?;
|
||||||
request_context.extend(items);
|
request_context.extend(items);
|
||||||
info!("Yielded by interceptor after pre-request history append");
|
info!("Yielded by interceptor after pre-request history append");
|
||||||
for cb in &self.turn_end_cbs {
|
for cb in &self.turn_end_cbs {
|
||||||
@@ -1220,7 +1235,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
return Ok(EngineResult::Yielded);
|
return Ok(EngineResult::Yielded);
|
||||||
}
|
}
|
||||||
PreRequestAction::ContinueWith(items) => {
|
PreRequestAction::ContinueWith(items) => {
|
||||||
self.append_history_items(items.clone());
|
self.append_history_items(items.clone())?;
|
||||||
request_context.extend(items);
|
request_context.extend(items);
|
||||||
}
|
}
|
||||||
PreRequestAction::Continue => {}
|
PreRequestAction::Continue => {}
|
||||||
@@ -1280,7 +1295,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
let assistant_items =
|
let assistant_items =
|
||||||
self.build_assistant_items(&reasoning_items, &text_blocks, &[]);
|
self.build_assistant_items(&reasoning_items, &text_blocks, &[]);
|
||||||
if !assistant_items.is_empty() {
|
if !assistant_items.is_empty() {
|
||||||
self.append_history_items(assistant_items);
|
self.append_history_items(assistant_items)?;
|
||||||
}
|
}
|
||||||
self.emit_llm_continuation(
|
self.emit_llm_continuation(
|
||||||
current_llm_call,
|
current_llm_call,
|
||||||
@@ -1307,7 +1322,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
let tool_calls = self.tool_call_collector.take_collected();
|
let tool_calls = self.tool_call_collector.take_collected();
|
||||||
let assistant_items =
|
let assistant_items =
|
||||||
self.build_assistant_items(&reasoning_items, &text_blocks, &tool_calls);
|
self.build_assistant_items(&reasoning_items, &text_blocks, &tool_calls);
|
||||||
self.append_history_items(assistant_items);
|
self.append_history_items(assistant_items)?;
|
||||||
|
|
||||||
if tool_calls.is_empty() {
|
if tool_calls.is_empty() {
|
||||||
match self.interceptor.on_turn_end(&self.history).await {
|
match self.interceptor.on_turn_end(&self.history).await {
|
||||||
@@ -1316,7 +1331,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
return Ok(EngineResult::Finished);
|
return Ok(EngineResult::Finished);
|
||||||
}
|
}
|
||||||
TurnEndAction::ContinueWithMessages(additional) => {
|
TurnEndAction::ContinueWithMessages(additional) => {
|
||||||
self.append_history_items(additional);
|
self.append_history_items(additional)?;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
TurnEndAction::Pause => {
|
TurnEndAction::Pause => {
|
||||||
@@ -1610,7 +1625,7 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
result.is_error,
|
result.is_error,
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
self.append_history_items(items);
|
self.append_history_items(items)?;
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
@@ -1815,12 +1830,15 @@ impl<C: LlmClient> Engine<C, Mutable> {
|
|||||||
self.history = items;
|
self.history = items;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Append items to history and notify history-append observers for each
|
/// Append items to history after every history-append observer accepts the
|
||||||
/// item before it lands. This is the only public Mutable-state API for
|
/// item. This is the only public Mutable-state API for growing engine
|
||||||
/// growing engine history; callers that need session-log persistence must
|
/// history; callers that need session-log persistence must install
|
||||||
/// install [`on_history_append`](Self::on_history_append) before calling it.
|
/// [`on_history_append`](Self::on_history_append) before calling it.
|
||||||
pub fn append_history(&mut self, items: impl IntoIterator<Item = Item>) {
|
pub fn append_history(
|
||||||
self.append_history_items(items);
|
&mut self,
|
||||||
|
items: impl IntoIterator<Item = Item>,
|
||||||
|
) -> Result<(), EngineError> {
|
||||||
|
self.append_history_items(items)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Truncate history without emitting append callbacks.
|
/// Truncate history without emitting append callbacks.
|
||||||
@@ -1969,9 +1987,9 @@ impl<C: LlmClient> Engine<C, Locked> {
|
|||||||
PromptAction::Continue => Vec::new(),
|
PromptAction::Continue => Vec::new(),
|
||||||
PromptAction::ContinueWith(items) => items,
|
PromptAction::ContinueWith(items) => items,
|
||||||
};
|
};
|
||||||
self.append_history_items(std::iter::once(user_item));
|
self.append_history_items(std::iter::once(user_item))?;
|
||||||
if !extras.is_empty() {
|
if !extras.is_empty() {
|
||||||
self.append_history_items(extras);
|
self.append_history_items(extras)?;
|
||||||
}
|
}
|
||||||
let result = self.run_turn_loop().await;
|
let result = self.run_turn_loop().await;
|
||||||
self.finalize_interruption(result).await
|
self.finalize_interruption(result).await
|
||||||
|
|||||||
@@ -158,8 +158,8 @@ pub trait Interceptor: Send + Sync {
|
|||||||
/// reproducible per-request transformations (pruning, content
|
/// reproducible per-request transformations (pruning, content
|
||||||
/// trimming, cache anchors) that depend only on the existing
|
/// trimming, cache anchors) that depend only on the existing
|
||||||
/// history.
|
/// history.
|
||||||
async fn pending_history_appends(&self) -> Vec<Item> {
|
async fn pending_history_appends(&self) -> Result<Vec<Item>, String> {
|
||||||
Vec::new()
|
Ok(Vec::new())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Called before each LLM request. The context starts as a clone
|
/// Called before each LLM request. The context starts as a clone
|
||||||
|
|||||||
@@ -44,12 +44,18 @@ fn test_mutable_history_manipulation() {
|
|||||||
assert!(engine.history().is_empty());
|
assert!(engine.history().is_empty());
|
||||||
|
|
||||||
// Add to history
|
// Add to history
|
||||||
engine.append_history(vec![Item::user_message("Hello")]);
|
engine
|
||||||
engine.append_history(vec![Item::assistant_message("Hi there!")]);
|
.append_history(vec![Item::user_message("Hello")])
|
||||||
|
.unwrap();
|
||||||
|
engine
|
||||||
|
.append_history(vec![Item::assistant_message("Hi there!")])
|
||||||
|
.unwrap();
|
||||||
assert_eq!(engine.history().len(), 2);
|
assert_eq!(engine.history().len(), 2);
|
||||||
|
|
||||||
// Append to history via the callback-aware API.
|
// Append to history via the callback-aware API.
|
||||||
engine.append_history(vec![Item::user_message("How are you?")]);
|
engine
|
||||||
|
.append_history(vec![Item::user_message("How are you?")])
|
||||||
|
.unwrap();
|
||||||
assert_eq!(engine.history().len(), 3);
|
assert_eq!(engine.history().len(), 3);
|
||||||
|
|
||||||
// Clear history
|
// Clear history
|
||||||
@@ -86,15 +92,20 @@ fn test_mutable_append_history() {
|
|||||||
if let Some(text) = item.as_text() {
|
if let Some(text) = item.as_text() {
|
||||||
observed_for_callback.lock().unwrap().push(text.to_string());
|
observed_for_callback.lock().unwrap().push(text.to_string());
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
});
|
});
|
||||||
|
|
||||||
engine.append_history(vec![Item::user_message("First")]);
|
engine
|
||||||
|
.append_history(vec![Item::user_message("First")])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
engine.append_history(vec![
|
engine
|
||||||
|
.append_history(vec![
|
||||||
Item::assistant_message("Response 1"),
|
Item::assistant_message("Response 1"),
|
||||||
Item::user_message("Second"),
|
Item::user_message("Second"),
|
||||||
Item::assistant_message("Response 2"),
|
Item::assistant_message("Response 2"),
|
||||||
]);
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(engine.history().len(), 4);
|
assert_eq!(engine.history().len(), 4);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -157,6 +168,40 @@ fn test_mutable_can_register_tool() {
|
|||||||
engine.register_tool(tool.definition());
|
engine.register_tool(tool.definition());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A durable-history failure on a tool call must stop the turn before the
|
||||||
|
/// tool can produce an external side effect.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn history_append_failure_stops_before_tool_execution() {
|
||||||
|
let client = MockLlmClient::new(vec![
|
||||||
|
Event::tool_use_start(0, "call_1", "count_tool"),
|
||||||
|
Event::tool_input_delta(0, r#"{}"#),
|
||||||
|
Event::tool_use_stop(0),
|
||||||
|
Event::Status(StatusEvent {
|
||||||
|
status: ResponseStatus::Completed,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
let tool = CountingTool::new("count_tool");
|
||||||
|
let mut engine = Engine::new(client);
|
||||||
|
engine.register_tool(tool.definition());
|
||||||
|
engine.on_history_append(|item| {
|
||||||
|
if item.is_tool_call() {
|
||||||
|
Err("simulated ENOSPC".to_string())
|
||||||
|
} else {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut engine = engine.lock();
|
||||||
|
let error = engine.run("use the tool").await.unwrap_err();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(error, EngineError::HistoryAppend(ref message) if message == "simulated ENOSPC")
|
||||||
|
);
|
||||||
|
assert_eq!(tool.call_count(), 0);
|
||||||
|
assert_eq!(engine.history().len(), 1);
|
||||||
|
assert_eq!(engine.history()[0].as_text(), Some("use the tool"));
|
||||||
|
}
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
// State Transition Tests
|
// State Transition Tests
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
@@ -168,8 +213,12 @@ fn test_lock_transition() {
|
|||||||
let mut engine = Engine::new(client);
|
let mut engine = Engine::new(client);
|
||||||
|
|
||||||
engine.set_system_prompt("System");
|
engine.set_system_prompt("System");
|
||||||
engine.append_history(vec![Item::user_message("Hello")]);
|
engine
|
||||||
engine.append_history(vec![Item::assistant_message("Hi")]);
|
.append_history(vec![Item::user_message("Hello")])
|
||||||
|
.unwrap();
|
||||||
|
engine
|
||||||
|
.append_history(vec![Item::assistant_message("Hi")])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// Lock
|
// Lock
|
||||||
let locked_engine = engine.lock();
|
let locked_engine = engine.lock();
|
||||||
@@ -186,14 +235,18 @@ fn test_unlock_transition() {
|
|||||||
let client = MockLlmClient::new(vec![]);
|
let client = MockLlmClient::new(vec![]);
|
||||||
let mut engine = Engine::new(client);
|
let mut engine = Engine::new(client);
|
||||||
|
|
||||||
engine.append_history(vec![Item::user_message("Hello")]);
|
engine
|
||||||
|
.append_history(vec![Item::user_message("Hello")])
|
||||||
|
.unwrap();
|
||||||
let locked_engine = engine.lock();
|
let locked_engine = engine.lock();
|
||||||
|
|
||||||
// Unlock
|
// Unlock
|
||||||
let mut engine = locked_engine.unlock();
|
let mut engine = locked_engine.unlock();
|
||||||
|
|
||||||
// History operations are available again in Mutable state
|
// History operations are available again in Mutable state
|
||||||
engine.append_history(vec![Item::assistant_message("Hi")]);
|
engine
|
||||||
|
.append_history(vec![Item::assistant_message("Hi")])
|
||||||
|
.unwrap();
|
||||||
engine.clear_history();
|
engine.clear_history();
|
||||||
assert!(engine.history().is_empty());
|
assert!(engine.history().is_empty());
|
||||||
}
|
}
|
||||||
@@ -316,8 +369,12 @@ async fn test_locked_prefix_len_tracking() {
|
|||||||
let mut engine = Engine::new(client);
|
let mut engine = Engine::new(client);
|
||||||
|
|
||||||
// Add items beforehand
|
// Add items beforehand
|
||||||
engine.append_history(vec![Item::user_message("Pre-existing message 1")]);
|
engine
|
||||||
engine.append_history(vec![Item::assistant_message("Pre-existing response 1")]);
|
.append_history(vec![Item::user_message("Pre-existing message 1")])
|
||||||
|
.unwrap();
|
||||||
|
engine
|
||||||
|
.append_history(vec![Item::assistant_message("Pre-existing response 1")])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(engine.history().len(), 2);
|
assert_eq!(engine.history().len(), 2);
|
||||||
|
|
||||||
@@ -387,10 +444,12 @@ async fn test_unlock_edit_relock() {
|
|||||||
]]);
|
]]);
|
||||||
|
|
||||||
let mut engine = Engine::new(client);
|
let mut engine = Engine::new(client);
|
||||||
engine.append_history(vec![
|
engine
|
||||||
|
.append_history(vec![
|
||||||
Item::user_message("Hello"),
|
Item::user_message("Hello"),
|
||||||
Item::assistant_message("Hi"),
|
Item::assistant_message("Hi"),
|
||||||
]);
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// Lock -> Unlock
|
// Lock -> Unlock
|
||||||
let locked = engine.lock();
|
let locked = engine.lock();
|
||||||
@@ -400,7 +459,9 @@ async fn test_unlock_edit_relock() {
|
|||||||
|
|
||||||
// Edit history
|
// Edit history
|
||||||
unlocked.clear_history();
|
unlocked.clear_history();
|
||||||
unlocked.append_history(vec![Item::user_message("Fresh start")]);
|
unlocked
|
||||||
|
.append_history(vec![Item::user_message("Fresh start")])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// Re-lock
|
// Re-lock
|
||||||
let relocked = unlocked.lock();
|
let relocked = unlocked.lock();
|
||||||
|
|||||||
@@ -83,10 +83,14 @@ pub struct FeatureConfigPartial {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub web: Option<FeatureFlagConfigPartial>,
|
pub web: Option<FeatureFlagConfigPartial>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub workers: Option<FeatureFlagConfigPartial>,
|
pub sub_worker: Option<FeatureFlagConfigPartial>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub worker: Option<FeatureFlagConfigPartial>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub objective: Option<FeatureFlagConfigPartial>,
|
pub objective: Option<FeatureFlagConfigPartial>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
pub manage_workdir: Option<FeatureFlagConfigPartial>,
|
||||||
|
#[serde(default)]
|
||||||
pub ticket: Option<TicketFeatureConfigPartial>,
|
pub ticket: Option<TicketFeatureConfigPartial>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub plugins: Option<FeatureFlagConfigPartial>,
|
pub plugins: Option<FeatureFlagConfigPartial>,
|
||||||
@@ -98,12 +102,22 @@ impl FeatureConfigPartial {
|
|||||||
task: merge_option(self.task, other.task, FeatureFlagConfigPartial::merge),
|
task: merge_option(self.task, other.task, FeatureFlagConfigPartial::merge),
|
||||||
memory: merge_option(self.memory, other.memory, MemoryFeatureConfigPartial::merge),
|
memory: merge_option(self.memory, other.memory, MemoryFeatureConfigPartial::merge),
|
||||||
web: merge_option(self.web, other.web, FeatureFlagConfigPartial::merge),
|
web: merge_option(self.web, other.web, FeatureFlagConfigPartial::merge),
|
||||||
workers: merge_option(self.workers, other.workers, FeatureFlagConfigPartial::merge),
|
sub_worker: merge_option(
|
||||||
|
self.sub_worker,
|
||||||
|
other.sub_worker,
|
||||||
|
FeatureFlagConfigPartial::merge,
|
||||||
|
),
|
||||||
|
worker: merge_option(self.worker, other.worker, FeatureFlagConfigPartial::merge),
|
||||||
objective: merge_option(
|
objective: merge_option(
|
||||||
self.objective,
|
self.objective,
|
||||||
other.objective,
|
other.objective,
|
||||||
FeatureFlagConfigPartial::merge,
|
FeatureFlagConfigPartial::merge,
|
||||||
),
|
),
|
||||||
|
manage_workdir: merge_option(
|
||||||
|
self.manage_workdir,
|
||||||
|
other.manage_workdir,
|
||||||
|
FeatureFlagConfigPartial::merge,
|
||||||
|
),
|
||||||
ticket: merge_option(self.ticket, other.ticket, TicketFeatureConfigPartial::merge),
|
ticket: merge_option(self.ticket, other.ticket, TicketFeatureConfigPartial::merge),
|
||||||
plugins: merge_option(self.plugins, other.plugins, FeatureFlagConfigPartial::merge),
|
plugins: merge_option(self.plugins, other.plugins, FeatureFlagConfigPartial::merge),
|
||||||
}
|
}
|
||||||
@@ -172,14 +186,22 @@ impl From<FeatureConfigPartial> for FeatureConfig {
|
|||||||
.map(MemoryFeatureConfig::from)
|
.map(MemoryFeatureConfig::from)
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
web: value.web.map(FeatureFlagConfig::from).unwrap_or_default(),
|
web: value.web.map(FeatureFlagConfig::from).unwrap_or_default(),
|
||||||
workers: value
|
sub_worker: value
|
||||||
.workers
|
.sub_worker
|
||||||
|
.map(FeatureFlagConfig::from)
|
||||||
|
.unwrap_or_default(),
|
||||||
|
worker: value
|
||||||
|
.worker
|
||||||
.map(FeatureFlagConfig::from)
|
.map(FeatureFlagConfig::from)
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
objective: value
|
objective: value
|
||||||
.objective
|
.objective
|
||||||
.map(FeatureFlagConfig::from)
|
.map(FeatureFlagConfig::from)
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
|
manage_workdir: value
|
||||||
|
.manage_workdir
|
||||||
|
.map(FeatureFlagConfig::from)
|
||||||
|
.unwrap_or_default(),
|
||||||
ticket: value
|
ticket: value
|
||||||
.ticket
|
.ticket
|
||||||
.map(TicketFeatureConfig::from)
|
.map(TicketFeatureConfig::from)
|
||||||
@@ -256,8 +278,10 @@ impl From<FeatureConfig> for FeatureConfigPartial {
|
|||||||
task: Some(value.task.into()),
|
task: Some(value.task.into()),
|
||||||
memory: Some(value.memory.into()),
|
memory: Some(value.memory.into()),
|
||||||
web: Some(value.web.into()),
|
web: Some(value.web.into()),
|
||||||
workers: Some(value.workers.into()),
|
sub_worker: Some(value.sub_worker.into()),
|
||||||
|
worker: Some(value.worker.into()),
|
||||||
objective: Some(value.objective.into()),
|
objective: Some(value.objective.into()),
|
||||||
|
manage_workdir: Some(value.manage_workdir.into()),
|
||||||
ticket: Some(value.ticket.into()),
|
ticket: Some(value.ticket.into()),
|
||||||
plugins: Some(value.plugins.into()),
|
plugins: Some(value.plugins.into()),
|
||||||
}
|
}
|
||||||
@@ -405,6 +429,15 @@ pub(crate) fn reject_removed_manifest_fields(s: &str) -> Result<(), toml::de::Er
|
|||||||
"unknown field in manifest: memory.extract_worker_max_input_tokens (removed)",
|
"unknown field in manifest: memory.extract_worker_max_input_tokens (removed)",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
if value
|
||||||
|
.get("feature")
|
||||||
|
.and_then(toml::Value::as_table)
|
||||||
|
.is_some_and(|table| table.contains_key("workers"))
|
||||||
|
{
|
||||||
|
return Err(toml::de::Error::custom(
|
||||||
|
"unknown field in manifest: feature.workers (removed; use feature.sub_worker)",
|
||||||
|
));
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -412,8 +445,8 @@ impl WorkerManifestConfig {
|
|||||||
/// Parse a partial manifest from a TOML string. Unknown top-level or
|
/// Parse a partial manifest from a TOML string. Unknown top-level or
|
||||||
/// nested fields emit a `tracing::warn!` and are ignored; use
|
/// nested fields emit a `tracing::warn!` and are ignored; use
|
||||||
/// `tracing_subscriber` with `WARN` enabled to surface them to the
|
/// `tracing_subscriber` with `WARN` enabled to surface them to the
|
||||||
/// operator. Removed fields that must not be silently ignored (currently
|
/// operator. Removed fields with an explicit replacement (including
|
||||||
/// `compaction.prune_protected_turns`) are rejected before deserialization.
|
/// `feature.workers`) are rejected before deserialization.
|
||||||
pub fn from_toml(s: &str) -> Result<Self, toml::de::Error> {
|
pub fn from_toml(s: &str) -> Result<Self, toml::de::Error> {
|
||||||
reject_removed_manifest_fields(s)?;
|
reject_removed_manifest_fields(s)?;
|
||||||
let de = toml::Deserializer::parse(s)?;
|
let de = toml::Deserializer::parse(s)?;
|
||||||
@@ -1802,8 +1835,9 @@ worker_max_turns = 7
|
|||||||
assert!(!manifest.feature.task.enabled);
|
assert!(!manifest.feature.task.enabled);
|
||||||
assert!(!manifest.feature.memory.enabled);
|
assert!(!manifest.feature.memory.enabled);
|
||||||
assert!(!manifest.feature.web.enabled);
|
assert!(!manifest.feature.web.enabled);
|
||||||
assert!(!manifest.feature.workers.enabled);
|
assert!(!manifest.feature.sub_worker.enabled);
|
||||||
assert!(!manifest.feature.objective.enabled);
|
assert!(!manifest.feature.objective.enabled);
|
||||||
|
assert!(!manifest.feature.manage_workdir.enabled);
|
||||||
assert!(!manifest.feature.ticket.enabled);
|
assert!(!manifest.feature.ticket.enabled);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1814,6 +1848,9 @@ worker_max_turns = 7
|
|||||||
[feature.task]
|
[feature.task]
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
|
[feature.manage_workdir]
|
||||||
|
enabled = true
|
||||||
|
|
||||||
[feature.ticket]
|
[feature.ticket]
|
||||||
enabled = true
|
enabled = true
|
||||||
authoring = false
|
authoring = false
|
||||||
@@ -1848,6 +1885,7 @@ orchestration_control = false
|
|||||||
.try_into()
|
.try_into()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(manifest.feature.task.enabled);
|
assert!(manifest.feature.task.enabled);
|
||||||
|
assert!(manifest.feature.manage_workdir.enabled);
|
||||||
assert!(manifest.feature.ticket.enabled);
|
assert!(manifest.feature.ticket.enabled);
|
||||||
assert!(!manifest.feature.ticket.authoring);
|
assert!(!manifest.feature.ticket.authoring);
|
||||||
assert!(!manifest.feature.ticket.thread);
|
assert!(!manifest.feature.ticket.thread);
|
||||||
@@ -1865,6 +1903,9 @@ orchestration_control = false
|
|||||||
[feature.memory]
|
[feature.memory]
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
|
[feature.manage_workdir]
|
||||||
|
enabled = false
|
||||||
|
|
||||||
[feature.ticket]
|
[feature.ticket]
|
||||||
enabled = true
|
enabled = true
|
||||||
authoring = false
|
authoring = false
|
||||||
@@ -1883,6 +1924,9 @@ orchestration_control = true
|
|||||||
[feature.memory]
|
[feature.memory]
|
||||||
staging = true
|
staging = true
|
||||||
|
|
||||||
|
[feature.manage_workdir]
|
||||||
|
enabled = true
|
||||||
|
|
||||||
[feature.objective]
|
[feature.objective]
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
@@ -1918,6 +1962,7 @@ enabled = true
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(manifest.feature.memory.enabled);
|
assert!(manifest.feature.memory.enabled);
|
||||||
assert!(manifest.feature.memory.staging);
|
assert!(manifest.feature.memory.staging);
|
||||||
|
assert!(manifest.feature.manage_workdir.enabled);
|
||||||
assert!(manifest.feature.ticket.enabled);
|
assert!(manifest.feature.ticket.enabled);
|
||||||
assert!(!manifest.feature.ticket.authoring);
|
assert!(!manifest.feature.ticket.authoring);
|
||||||
assert!(manifest.feature.ticket.thread);
|
assert!(manifest.feature.ticket.thread);
|
||||||
@@ -1925,7 +1970,7 @@ enabled = true
|
|||||||
assert!(manifest.feature.ticket.orchestration_control);
|
assert!(manifest.feature.ticket.orchestration_control);
|
||||||
assert!(manifest.feature.objective.enabled);
|
assert!(manifest.feature.objective.enabled);
|
||||||
assert!(manifest.feature.web.enabled);
|
assert!(manifest.feature.web.enabled);
|
||||||
assert!(!manifest.feature.workers.enabled);
|
assert!(!manifest.feature.sub_worker.enabled);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -111,10 +111,14 @@ pub struct FeatureConfig {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub web: FeatureFlagConfig,
|
pub web: FeatureFlagConfig,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub workers: FeatureFlagConfig,
|
pub sub_worker: FeatureFlagConfig,
|
||||||
|
#[serde(default)]
|
||||||
|
pub worker: FeatureFlagConfig,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub objective: FeatureFlagConfig,
|
pub objective: FeatureFlagConfig,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
pub manage_workdir: FeatureFlagConfig,
|
||||||
|
#[serde(default)]
|
||||||
pub ticket: TicketFeatureConfig,
|
pub ticket: TicketFeatureConfig,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub plugins: FeatureFlagConfig,
|
pub plugins: FeatureFlagConfig,
|
||||||
@@ -126,8 +130,10 @@ impl Default for FeatureConfig {
|
|||||||
task: FeatureFlagConfig::disabled(),
|
task: FeatureFlagConfig::disabled(),
|
||||||
memory: MemoryFeatureConfig::disabled(),
|
memory: MemoryFeatureConfig::disabled(),
|
||||||
web: FeatureFlagConfig::disabled(),
|
web: FeatureFlagConfig::disabled(),
|
||||||
workers: FeatureFlagConfig::disabled(),
|
sub_worker: FeatureFlagConfig::disabled(),
|
||||||
|
worker: FeatureFlagConfig::disabled(),
|
||||||
objective: FeatureFlagConfig::disabled(),
|
objective: FeatureFlagConfig::disabled(),
|
||||||
|
manage_workdir: FeatureFlagConfig::disabled(),
|
||||||
ticket: TicketFeatureConfig::default(),
|
ticket: TicketFeatureConfig::default(),
|
||||||
plugins: FeatureFlagConfig::disabled(),
|
plugins: FeatureFlagConfig::disabled(),
|
||||||
}
|
}
|
||||||
@@ -402,7 +408,7 @@ pub struct MemoryConfig {
|
|||||||
/// system-prompt section. `None` ⇒ enabled.
|
/// system-prompt section. `None` ⇒ enabled.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub inject_summary: Option<bool>,
|
pub inject_summary: Option<bool>,
|
||||||
/// Language used by memory extraction / consolidation workers for durable
|
/// Language used by memory extraction / consolidation sub_worker for durable
|
||||||
/// memory text. Free-form so workspaces can use names like
|
/// memory text. Free-form so workspaces can use names like
|
||||||
/// `English`, `Japanese`, or locale tags. `None` ⇒
|
/// `English`, `Japanese`, or locale tags. `None` ⇒
|
||||||
/// [`defaults::MEMORY_LANGUAGE`].
|
/// [`defaults::MEMORY_LANGUAGE`].
|
||||||
|
|||||||
@@ -436,7 +436,7 @@ impl ProfileResolver {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// Resolve a registry/default selector against an already-discovered
|
/// Resolve a registry/default selector against an already-discovered
|
||||||
/// registry. Callers such as SpawnWorker use this to bind discovery to the
|
/// registry. Callers such as SubWorkerSpawn use this to bind discovery to the
|
||||||
/// Worker's cwd instead of the process current directory.
|
/// Worker's cwd instead of the process current directory.
|
||||||
pub fn resolve_from_registry(
|
pub fn resolve_from_registry(
|
||||||
&self,
|
&self,
|
||||||
@@ -899,7 +899,7 @@ fn builtin_profile_artifact(label: &str) -> Option<serde_json::Value> {
|
|||||||
true,
|
true,
|
||||||
true,
|
true,
|
||||||
true,
|
true,
|
||||||
true,
|
false,
|
||||||
);
|
);
|
||||||
Some(value)
|
Some(value)
|
||||||
}
|
}
|
||||||
@@ -936,7 +936,7 @@ fn builtin_profile_artifact(label: &str) -> Option<serde_json::Value> {
|
|||||||
value["feature"]["task"] = serde_json::json!({ "enabled": false });
|
value["feature"]["task"] = serde_json::json!({ "enabled": false });
|
||||||
value["feature"]["memory"] = serde_json::json!({ "enabled": true, "staging": true });
|
value["feature"]["memory"] = serde_json::json!({ "enabled": true, "staging": true });
|
||||||
value["feature"]["web"] = serde_json::json!({ "enabled": false });
|
value["feature"]["web"] = serde_json::json!({ "enabled": false });
|
||||||
value["feature"]["workers"] = serde_json::json!({ "enabled": false });
|
value["feature"]["sub_worker"] = serde_json::json!({ "enabled": false });
|
||||||
value["feature"]["objective"] = serde_json::json!({ "enabled": false });
|
value["feature"]["objective"] = serde_json::json!({ "enabled": false });
|
||||||
value["feature"]["ticket"] = serde_json::json!({ "enabled": false, "thread": false });
|
value["feature"]["ticket"] = serde_json::json!({ "enabled": false, "thread": false });
|
||||||
Some(value)
|
Some(value)
|
||||||
@@ -962,7 +962,8 @@ fn builtin_base_profile_artifact() -> serde_json::Value {
|
|||||||
"task": { "enabled": true },
|
"task": { "enabled": true },
|
||||||
"memory": { "enabled": true },
|
"memory": { "enabled": true },
|
||||||
"web": { "enabled": true },
|
"web": { "enabled": true },
|
||||||
"workers": { "enabled": true },
|
"sub_worker": { "enabled": true },
|
||||||
|
"worker": { "enabled": false },
|
||||||
"objective": { "enabled": true },
|
"objective": { "enabled": true },
|
||||||
"ticket": { "enabled": true, "authoring": true, "thread": true }
|
"ticket": { "enabled": true, "authoring": true, "thread": true }
|
||||||
},
|
},
|
||||||
@@ -990,14 +991,17 @@ fn apply_role_profile(
|
|||||||
task: bool,
|
task: bool,
|
||||||
memory: bool,
|
memory: bool,
|
||||||
web: bool,
|
web: bool,
|
||||||
workers: bool,
|
sub_worker: bool,
|
||||||
) {
|
) {
|
||||||
value["slug"] = serde_json::Value::String(slug.to_string());
|
value["slug"] = serde_json::Value::String(slug.to_string());
|
||||||
value["description"] = serde_json::Value::String(description.to_string());
|
value["description"] = serde_json::Value::String(description.to_string());
|
||||||
value["feature"]["task"] = serde_json::json!({ "enabled": task });
|
value["feature"]["task"] = serde_json::json!({ "enabled": task });
|
||||||
value["feature"]["memory"] = serde_json::json!({ "enabled": memory });
|
value["feature"]["memory"] = serde_json::json!({ "enabled": memory });
|
||||||
value["feature"]["web"] = serde_json::json!({ "enabled": web });
|
value["feature"]["web"] = serde_json::json!({ "enabled": web });
|
||||||
value["feature"]["workers"] = serde_json::json!({ "enabled": workers });
|
value["feature"]["sub_worker"] = serde_json::json!({ "enabled": sub_worker });
|
||||||
|
value["feature"]["worker"] =
|
||||||
|
serde_json::json!({ "enabled": matches!(slug, "companion" | "orchestrator") });
|
||||||
|
value["feature"]["manage_workdir"] = serde_json::json!({ "enabled": slug == "orchestrator" });
|
||||||
let ticket = match slug {
|
let ticket = match slug {
|
||||||
"companion" => serde_json::json!({ "enabled": true, "authoring": true, "thread": true }),
|
"companion" => serde_json::json!({ "enabled": true, "authoring": true, "thread": true }),
|
||||||
"intake" => {
|
"intake" => {
|
||||||
@@ -1455,7 +1459,8 @@ mod tests {
|
|||||||
|
|
||||||
let companion = resolve("companion");
|
let companion = resolve("companion");
|
||||||
assert!(companion.feature.task.enabled);
|
assert!(companion.feature.task.enabled);
|
||||||
assert!(companion.feature.workers.enabled);
|
assert!(companion.feature.sub_worker.enabled);
|
||||||
|
assert!(companion.feature.worker.enabled);
|
||||||
assert!(companion.scope.allow.is_empty());
|
assert!(companion.scope.allow.is_empty());
|
||||||
assert!(companion.scope.deny.is_empty());
|
assert!(companion.scope.deny.is_empty());
|
||||||
assert!(companion.delegation_scope.allow.is_empty());
|
assert!(companion.delegation_scope.allow.is_empty());
|
||||||
@@ -1486,12 +1491,14 @@ mod tests {
|
|||||||
|
|
||||||
let intake = resolve("intake");
|
let intake = resolve("intake");
|
||||||
assert!(intake.feature.task.enabled);
|
assert!(intake.feature.task.enabled);
|
||||||
assert!(!intake.feature.workers.enabled);
|
assert!(!intake.feature.sub_worker.enabled);
|
||||||
|
assert!(!intake.feature.worker.enabled);
|
||||||
assert!(intake.feature.ticket.enabled);
|
assert!(intake.feature.ticket.enabled);
|
||||||
assert!(intake.feature.ticket.enabled);
|
assert!(intake.feature.ticket.enabled);
|
||||||
assert!(intake.feature.ticket.authoring);
|
assert!(intake.feature.ticket.authoring);
|
||||||
assert!(intake.feature.ticket.thread);
|
assert!(intake.feature.ticket.thread);
|
||||||
assert!(intake.feature.objective.enabled);
|
assert!(intake.feature.objective.enabled);
|
||||||
|
assert!(!intake.feature.manage_workdir.enabled);
|
||||||
assert!(intake.feature.ticket.intake);
|
assert!(intake.feature.ticket.intake);
|
||||||
assert!(!intake.feature.ticket.orchestration_control);
|
assert!(!intake.feature.ticket.orchestration_control);
|
||||||
assert!(intake.scope.allow.is_empty());
|
assert!(intake.scope.allow.is_empty());
|
||||||
@@ -1502,12 +1509,14 @@ mod tests {
|
|||||||
|
|
||||||
let orchestrator = resolve("orchestrator");
|
let orchestrator = resolve("orchestrator");
|
||||||
assert!(orchestrator.feature.task.enabled);
|
assert!(orchestrator.feature.task.enabled);
|
||||||
assert!(orchestrator.feature.workers.enabled);
|
assert!(!orchestrator.feature.sub_worker.enabled);
|
||||||
|
assert!(orchestrator.feature.worker.enabled);
|
||||||
assert!(orchestrator.feature.ticket.enabled);
|
assert!(orchestrator.feature.ticket.enabled);
|
||||||
assert!(orchestrator.feature.ticket.enabled);
|
assert!(orchestrator.feature.ticket.enabled);
|
||||||
assert!(!orchestrator.feature.ticket.authoring);
|
assert!(!orchestrator.feature.ticket.authoring);
|
||||||
assert!(orchestrator.feature.ticket.thread);
|
assert!(orchestrator.feature.ticket.thread);
|
||||||
assert!(orchestrator.feature.objective.enabled);
|
assert!(orchestrator.feature.objective.enabled);
|
||||||
|
assert!(orchestrator.feature.manage_workdir.enabled);
|
||||||
assert!(!orchestrator.feature.ticket.intake);
|
assert!(!orchestrator.feature.ticket.intake);
|
||||||
assert!(orchestrator.feature.ticket.orchestration_control);
|
assert!(orchestrator.feature.ticket.orchestration_control);
|
||||||
assert!(orchestrator.scope.allow.is_empty());
|
assert!(orchestrator.scope.allow.is_empty());
|
||||||
@@ -1521,7 +1530,8 @@ mod tests {
|
|||||||
|
|
||||||
let coder = resolve("coder");
|
let coder = resolve("coder");
|
||||||
assert!(coder.feature.task.enabled);
|
assert!(coder.feature.task.enabled);
|
||||||
assert!(!coder.feature.workers.enabled);
|
assert!(!coder.feature.sub_worker.enabled);
|
||||||
|
assert!(!coder.feature.worker.enabled);
|
||||||
assert!(coder.scope.allow.is_empty());
|
assert!(coder.scope.allow.is_empty());
|
||||||
assert!(coder.delegation_scope.allow.is_empty());
|
assert!(coder.delegation_scope.allow.is_empty());
|
||||||
assert_eq!(coder.model.ref_.as_deref(), Some("codex-oauth/gpt-5.5"));
|
assert_eq!(coder.model.ref_.as_deref(), Some("codex-oauth/gpt-5.5"));
|
||||||
@@ -1532,16 +1542,19 @@ mod tests {
|
|||||||
assert!(!coder.feature.ticket.authoring);
|
assert!(!coder.feature.ticket.authoring);
|
||||||
assert!(coder.feature.ticket.thread);
|
assert!(coder.feature.ticket.thread);
|
||||||
assert!(coder.feature.objective.enabled);
|
assert!(coder.feature.objective.enabled);
|
||||||
|
assert!(!coder.feature.manage_workdir.enabled);
|
||||||
assert!(!coder.feature.ticket.intake);
|
assert!(!coder.feature.ticket.intake);
|
||||||
assert!(!coder.feature.ticket.orchestration_control);
|
assert!(!coder.feature.ticket.orchestration_control);
|
||||||
let reviewer = resolve("reviewer");
|
let reviewer = resolve("reviewer");
|
||||||
assert!(reviewer.feature.task.enabled);
|
assert!(reviewer.feature.task.enabled);
|
||||||
assert!(!reviewer.feature.workers.enabled);
|
assert!(!reviewer.feature.sub_worker.enabled);
|
||||||
|
assert!(!reviewer.feature.worker.enabled);
|
||||||
assert!(reviewer.feature.ticket.enabled);
|
assert!(reviewer.feature.ticket.enabled);
|
||||||
assert!(reviewer.feature.ticket.enabled);
|
assert!(reviewer.feature.ticket.enabled);
|
||||||
assert!(!reviewer.feature.ticket.authoring);
|
assert!(!reviewer.feature.ticket.authoring);
|
||||||
assert!(reviewer.feature.ticket.thread);
|
assert!(reviewer.feature.ticket.thread);
|
||||||
assert!(reviewer.feature.objective.enabled);
|
assert!(reviewer.feature.objective.enabled);
|
||||||
|
assert!(!reviewer.feature.manage_workdir.enabled);
|
||||||
assert!(!reviewer.feature.ticket.intake);
|
assert!(!reviewer.feature.ticket.intake);
|
||||||
assert!(!reviewer.feature.ticket.orchestration_control);
|
assert!(!reviewer.feature.ticket.orchestration_control);
|
||||||
assert!(reviewer.scope.allow.is_empty());
|
assert!(reviewer.scope.allow.is_empty());
|
||||||
@@ -1687,7 +1700,7 @@ enabled = false
|
|||||||
[feature.web]
|
[feature.web]
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
[feature.workers]
|
[feature.sub_worker]
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
[feature.ticket]
|
[feature.ticket]
|
||||||
@@ -1711,7 +1724,7 @@ orchestration_control = false
|
|||||||
assert!(resolved.manifest.feature.task.enabled);
|
assert!(resolved.manifest.feature.task.enabled);
|
||||||
assert!(!resolved.manifest.feature.memory.enabled);
|
assert!(!resolved.manifest.feature.memory.enabled);
|
||||||
assert!(resolved.manifest.feature.web.enabled);
|
assert!(resolved.manifest.feature.web.enabled);
|
||||||
assert!(resolved.manifest.feature.workers.enabled);
|
assert!(resolved.manifest.feature.sub_worker.enabled);
|
||||||
assert!(resolved.manifest.feature.ticket.enabled);
|
assert!(resolved.manifest.feature.ticket.enabled);
|
||||||
assert!(!resolved.manifest.feature.ticket.authoring);
|
assert!(!resolved.manifest.feature.ticket.authoring);
|
||||||
assert!(!resolved.manifest.feature.ticket.thread);
|
assert!(!resolved.manifest.feature.ticket.thread);
|
||||||
|
|||||||
@@ -331,7 +331,7 @@ impl Scope {
|
|||||||
|
|
||||||
/// Build a new [`Scope`] equal to `self` with `extra_deny` appended
|
/// Build a new [`Scope`] equal to `self` with `extra_deny` appended
|
||||||
/// to the deny set. Used by dynamic-scope shrink paths
|
/// to the deny set. Used by dynamic-scope shrink paths
|
||||||
/// (e.g. SpawnWorker-style delegation that strips Write from the
|
/// (e.g. SubWorkerSpawn-style delegation that strips Write from the
|
||||||
/// spawner without touching its allow rules).
|
/// spawner without touching its allow rules).
|
||||||
pub fn with_added_deny_rules(
|
pub fn with_added_deny_rules(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -125,7 +125,7 @@ pub enum WorkerEvent {
|
|||||||
/// Child has stopped (controller loop is exiting).
|
/// Child has stopped (controller loop is exiting).
|
||||||
ShutDown { worker_name: String },
|
ShutDown { worker_name: String },
|
||||||
|
|
||||||
/// Child sub-delegated scope to a grandchild Worker via `SpawnWorker`.
|
/// Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.
|
||||||
///
|
///
|
||||||
/// Control-plane only: receivers apply registry side effects and
|
/// Control-plane only: receivers apply registry side effects and
|
||||||
/// propagate upward, but do not expose this as an agent notification.
|
/// propagate upward, but do not expose this as an agent notification.
|
||||||
|
|||||||
@@ -1598,8 +1598,22 @@ fn tool_kind(name: &str) -> &'static str {
|
|||||||
"Read" | "Write" | "Edit" | "Glob" | "Grep" => "filesystem",
|
"Read" | "Write" | "Edit" | "Glob" | "Grep" => "filesystem",
|
||||||
"Bash" => "shell",
|
"Bash" => "shell",
|
||||||
"WebFetch" | "WebSearch" => "web",
|
"WebFetch" | "WebSearch" => "web",
|
||||||
"SpawnWorker" | "SendToWorker" | "SendToPeerWorker" | "ReadWorkerOutput"
|
"SubWorkerSpawn"
|
||||||
| "ListWorkers" | "StopWorker" | "RestoreWorker" => "worker",
|
| "SubWorkerSend"
|
||||||
|
| "SubWorkerReadOutput"
|
||||||
|
| "SubWorkerList"
|
||||||
|
| "SubWorkerStop"
|
||||||
|
| "WorkerList"
|
||||||
|
| "WorkerSpawn"
|
||||||
|
| "WorkerStop"
|
||||||
|
| "WorkerRestore"
|
||||||
|
| "SpawnWorker"
|
||||||
|
| "SendToWorker"
|
||||||
|
| "SendToPeerWorker"
|
||||||
|
| "ReadWorkerOutput"
|
||||||
|
| "ListWorkers"
|
||||||
|
| "StopWorker"
|
||||||
|
| "RestoreWorker" => "worker",
|
||||||
// Legacy session logs used the pre-rename peer tool name; keep analytics classification only.
|
// Legacy session logs used the pre-rename peer tool name; keep analytics classification only.
|
||||||
/* legacy session-log tool name only */
|
/* legacy session-log tool name only */
|
||||||
LEGACY_SEND_TO_PEER_POD_TOOL => "worker",
|
LEGACY_SEND_TO_PEER_POD_TOOL => "worker",
|
||||||
|
|||||||
@@ -20,17 +20,23 @@ use crate::segment_log::LogEntry;
|
|||||||
use crate::store::{Store, StoreError};
|
use crate::store::{Store, StoreError};
|
||||||
use crate::{SegmentId, SessionId};
|
use crate::{SegmentId, SessionId};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::io::Write;
|
use std::io::{Read, Seek, SeekFrom, Write};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
|
|
||||||
/// Filesystem-backed JSONL store.
|
/// Filesystem-backed JSONL store.
|
||||||
///
|
///
|
||||||
/// Each segment is stored as a single `.jsonl` file with one [`LogEntry`]
|
/// Each segment is stored as a single `.jsonl` file with one [`LogEntry`]
|
||||||
/// per line. Writes use append mode for crash safety.
|
/// per line. A trailing line is committed only once its newline has been
|
||||||
|
/// written; readers ignore an unterminated tail and the next append removes it.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct FsStore {
|
pub struct FsStore {
|
||||||
root: PathBuf,
|
root: PathBuf,
|
||||||
|
/// Serialises append repair + write + rollback across clones. A failed
|
||||||
|
/// `write_all` may have extended the file, so rollback is safe only while
|
||||||
|
/// no sibling writer can append behind it.
|
||||||
|
append_lock: Arc<Mutex<()>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FsStore {
|
impl FsStore {
|
||||||
@@ -39,7 +45,10 @@ impl FsStore {
|
|||||||
pub fn new(root: impl Into<PathBuf>) -> Result<Self, StoreError> {
|
pub fn new(root: impl Into<PathBuf>) -> Result<Self, StoreError> {
|
||||||
let root = root.into();
|
let root = root.into();
|
||||||
fs::create_dir_all(&root)?;
|
fs::create_dir_all(&root)?;
|
||||||
Ok(Self { root })
|
Ok(Self {
|
||||||
|
root,
|
||||||
|
append_lock: Arc::new(Mutex::new(())),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return the filesystem root used by this store.
|
/// Return the filesystem root used by this store.
|
||||||
@@ -101,22 +110,62 @@ impl FsStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn append_line(&self, path: &Path, line: &str) -> Result<(), StoreError> {
|
fn append_line(&self, path: &Path, line: &str) -> Result<(), StoreError> {
|
||||||
|
let _guard = self
|
||||||
|
.append_lock
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| std::io::Error::other("session store append lock was poisoned"))?;
|
||||||
if let Some(parent) = path.parent() {
|
if let Some(parent) = path.parent() {
|
||||||
fs::create_dir_all(parent)?;
|
fs::create_dir_all(parent)?;
|
||||||
}
|
}
|
||||||
let mut file = fs::OpenOptions::new()
|
let mut file = fs::OpenOptions::new()
|
||||||
.create(true)
|
.create(true)
|
||||||
|
.read(true)
|
||||||
|
.write(true)
|
||||||
.append(true)
|
.append(true)
|
||||||
.open(path)?;
|
.open(path)?;
|
||||||
file.write_all(line.as_bytes())?;
|
let committed_len = Self::truncate_uncommitted_tail(&mut file)?;
|
||||||
file.write_all(b"\n")?;
|
let mut record = Vec::with_capacity(line.len() + 1);
|
||||||
// Append-mode write is the durability boundary; an explicit
|
record.extend_from_slice(line.as_bytes());
|
||||||
// `sync_all` here would multiply latency by ~10× for no gain
|
record.push(b'\n');
|
||||||
// since the kernel already orders concurrent `O_APPEND` writes.
|
|
||||||
|
if let Err(write_error) = file.write_all(&record) {
|
||||||
|
return match file.set_len(committed_len) {
|
||||||
|
Ok(()) => Err(write_error.into()),
|
||||||
|
Err(rollback_error) => Err(std::io::Error::new(
|
||||||
|
rollback_error.kind(),
|
||||||
|
format!(
|
||||||
|
"session append failed ({write_error}) and partial-write rollback failed: {rollback_error}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.into()),
|
||||||
|
};
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_jsonl<T: serde::de::DeserializeOwned>(content: &str) -> Result<Vec<T>, StoreError> {
|
/// Return only newline-terminated records. A process interruption or
|
||||||
|
/// ENOSPC can leave the final UTF-8 code point / JSON object incomplete;
|
||||||
|
/// without a newline that record never crossed the commit boundary.
|
||||||
|
fn complete_jsonl_prefix(content: &[u8]) -> &[u8] {
|
||||||
|
if content.last() == Some(&b'\n') {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
match content.iter().rposition(|byte| *byte == b'\n') {
|
||||||
|
Some(index) => &content[..=index],
|
||||||
|
None => &[],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_jsonl<T: serde::de::DeserializeOwned>(content: &[u8]) -> Result<Vec<T>, StoreError> {
|
||||||
|
let complete = Self::complete_jsonl_prefix(content);
|
||||||
|
let content = std::str::from_utf8(complete).map_err(|error| StoreError::Corrupt {
|
||||||
|
line: complete[..error.valid_up_to()]
|
||||||
|
.iter()
|
||||||
|
.filter(|byte| **byte == b'\n')
|
||||||
|
.count()
|
||||||
|
+ 1,
|
||||||
|
message: error.to_string(),
|
||||||
|
})?;
|
||||||
let mut entries = Vec::new();
|
let mut entries = Vec::new();
|
||||||
for (i, line) in content.lines().enumerate() {
|
for (i, line) in content.lines().enumerate() {
|
||||||
if line.trim().is_empty() {
|
if line.trim().is_empty() {
|
||||||
@@ -130,6 +179,43 @@ impl FsStore {
|
|||||||
}
|
}
|
||||||
Ok(entries)
|
Ok(entries)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Remove a prior unterminated record and return the committed file size.
|
||||||
|
/// Scans backwards in bounded chunks so repairing a large session does not
|
||||||
|
/// require loading it into memory.
|
||||||
|
fn truncate_uncommitted_tail(file: &mut fs::File) -> std::io::Result<u64> {
|
||||||
|
const SCAN_BYTES: usize = 8 * 1024;
|
||||||
|
|
||||||
|
let len = file.metadata()?.len();
|
||||||
|
if len == 0 {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
file.seek(SeekFrom::End(-1))?;
|
||||||
|
let mut last = [0_u8; 1];
|
||||||
|
file.read_exact(&mut last)?;
|
||||||
|
if last[0] == b'\n' {
|
||||||
|
return Ok(len);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut end = len;
|
||||||
|
let mut buffer = [0_u8; SCAN_BYTES];
|
||||||
|
while end > 0 {
|
||||||
|
let start = end.saturating_sub(SCAN_BYTES as u64);
|
||||||
|
let chunk_len = (end - start) as usize;
|
||||||
|
file.seek(SeekFrom::Start(start))?;
|
||||||
|
file.read_exact(&mut buffer[..chunk_len])?;
|
||||||
|
if let Some(index) = buffer[..chunk_len].iter().rposition(|byte| *byte == b'\n') {
|
||||||
|
let committed_len = start + index as u64 + 1;
|
||||||
|
file.set_len(committed_len)?;
|
||||||
|
return Ok(committed_len);
|
||||||
|
}
|
||||||
|
end = start;
|
||||||
|
}
|
||||||
|
|
||||||
|
file.set_len(0)?;
|
||||||
|
Ok(0)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Store for FsStore {
|
impl Store for FsStore {
|
||||||
@@ -152,7 +238,7 @@ impl Store for FsStore {
|
|||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
return Err(StoreError::NotFound(segment_id));
|
return Err(StoreError::NotFound(segment_id));
|
||||||
}
|
}
|
||||||
let content = fs::read_to_string(&path)?;
|
let content = fs::read(&path)?;
|
||||||
Self::parse_jsonl(&content)
|
Self::parse_jsonl(&content)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,8 +337,17 @@ impl Store for FsStore {
|
|||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
return Err(StoreError::NotFound(segment_id));
|
return Err(StoreError::NotFound(segment_id));
|
||||||
}
|
}
|
||||||
let content = fs::read_to_string(&path)?;
|
let content = fs::read(&path)?;
|
||||||
Ok(content.lines().filter(|l| !l.trim().is_empty()).count())
|
let complete = Self::complete_jsonl_prefix(&content);
|
||||||
|
let complete = std::str::from_utf8(complete).map_err(|error| StoreError::Corrupt {
|
||||||
|
line: complete[..error.valid_up_to()]
|
||||||
|
.iter()
|
||||||
|
.filter(|byte| **byte == b'\n')
|
||||||
|
.count()
|
||||||
|
+ 1,
|
||||||
|
message: error.to_string(),
|
||||||
|
})?;
|
||||||
|
Ok(complete.lines().filter(|l| !l.trim().is_empty()).count())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn append_trace(
|
fn append_trace(
|
||||||
|
|||||||
@@ -69,7 +69,10 @@ pub enum LogEntry {
|
|||||||
/// Field name is `trigger` (not `kind`) because the LogEntry
|
/// Field name is `trigger` (not `kind`) because the LogEntry
|
||||||
/// serde tag already occupies `"kind"`.
|
/// serde tag already occupies `"kind"`.
|
||||||
///
|
///
|
||||||
/// Marker only — replay does not mutate `RestoredState`.
|
/// Replay marks the run interrupted until a terminal `RunCompleted`,
|
||||||
|
/// `RunErrored`, or `PausedTurnAbandoned` entry proves how it ended. This
|
||||||
|
/// makes a process/disk failure between Invoke and its terminal record
|
||||||
|
/// restore conservatively instead of re-running a dangling tool call.
|
||||||
Invoke { ts: u64, trigger: InvokeKind },
|
Invoke { ts: u64, trigger: InvokeKind },
|
||||||
|
|
||||||
/// User input accepted at submit time. Carries the original typed
|
/// User input accepted at submit time. Carries the original typed
|
||||||
@@ -236,9 +239,9 @@ pub fn collect_state(entries: &[LogEntry]) -> RestoredState {
|
|||||||
state.history = history.iter().cloned().map(Item::from).collect();
|
state.history = history.iter().cloned().map(Item::from).collect();
|
||||||
}
|
}
|
||||||
LogEntry::Invoke { .. } => {
|
LogEntry::Invoke { .. } => {
|
||||||
// Marker only; no state mutation. The trailing
|
// A terminal run record below clears or refines this. If the
|
||||||
// UserInput / SystemItem / TurnEnd entries carry all
|
// log ends first, restore must treat the turn as interrupted.
|
||||||
// replay-relevant data.
|
state.last_run_interrupted = true;
|
||||||
}
|
}
|
||||||
LogEntry::UserInput { segments, .. } => {
|
LogEntry::UserInput { segments, .. } => {
|
||||||
let text = Segment::flatten_to_text(segments);
|
let text = Segment::flatten_to_text(segments);
|
||||||
@@ -368,6 +371,35 @@ mod tests {
|
|||||||
assert!(!state.last_run_interrupted);
|
assert!(!state.last_run_interrupted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_incomplete_invoke_is_interrupted() {
|
||||||
|
let state = collect_state(&[
|
||||||
|
LogEntry::SegmentStart {
|
||||||
|
ts: 1000,
|
||||||
|
session_id: uuid::Uuid::nil(),
|
||||||
|
system_prompt: None,
|
||||||
|
config: RequestConfig::default(),
|
||||||
|
history: vec![],
|
||||||
|
forked_from: None,
|
||||||
|
compacted_from: None,
|
||||||
|
},
|
||||||
|
LogEntry::Invoke {
|
||||||
|
ts: 2000,
|
||||||
|
trigger: InvokeKind::UserSend,
|
||||||
|
},
|
||||||
|
LogEntry::UserInput {
|
||||||
|
ts: 2001,
|
||||||
|
segments: vec![Segment::text("run a tool")],
|
||||||
|
},
|
||||||
|
LogEntry::AssistantItem {
|
||||||
|
ts: 3000,
|
||||||
|
item: Item::tool_call("call_1", "side_effect", "{}").into(),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert!(state.last_run_interrupted);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replay_with_tool_calls() {
|
fn replay_with_tool_calls() {
|
||||||
let state = collect_state(&[
|
let state = collect_state(&[
|
||||||
@@ -546,7 +578,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replay_invoke_marker_does_not_mutate_state() {
|
fn replay_invoke_marker_only_mutates_interrupted_state() {
|
||||||
let state = collect_state(&[
|
let state = collect_state(&[
|
||||||
LogEntry::SegmentStart {
|
LogEntry::SegmentStart {
|
||||||
ts: 0,
|
ts: 0,
|
||||||
@@ -576,6 +608,7 @@ mod tests {
|
|||||||
]);
|
]);
|
||||||
assert_eq!(state.history.len(), 1);
|
assert_eq!(state.history.len(), 1);
|
||||||
assert_eq!(state.turn_count, 1);
|
assert_eq!(state.turn_count, 1);
|
||||||
|
assert!(state.last_run_interrupted);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -40,8 +40,8 @@ pub enum StoreError {
|
|||||||
pub trait Store: Send + Sync {
|
pub trait Store: Send + Sync {
|
||||||
/// Append a single log entry to the segment log.
|
/// Append a single log entry to the segment log.
|
||||||
///
|
///
|
||||||
/// One line per call. The kernel orders concurrent `O_APPEND` writes
|
/// One committed line per successful call. Implementations must not expose
|
||||||
/// for lines < `PIPE_BUF`, so user-space serialization is unnecessary.
|
/// a failed call's partial record as committed data on later reads.
|
||||||
fn append(
|
fn append(
|
||||||
&self,
|
&self,
|
||||||
session_id: SessionId,
|
session_id: SessionId,
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ use llm_engine::llm_client::types::{Item, RequestConfig};
|
|||||||
use session_store::{
|
use session_store::{
|
||||||
FsStore, LogEntry, Store, TraceEntry, collect_state, new_segment_id, new_session_id,
|
FsStore, LogEntry, Store, TraceEntry, collect_state, new_segment_id, new_session_id,
|
||||||
};
|
};
|
||||||
|
use std::io::Write;
|
||||||
|
|
||||||
fn nil_session_start(ts: u64, session_id: uuid::Uuid) -> LogEntry {
|
fn nil_session_start(ts: u64, session_id: uuid::Uuid) -> LogEntry {
|
||||||
LogEntry::SegmentStart {
|
LogEntry::SegmentStart {
|
||||||
@@ -224,6 +225,71 @@ fn read_entry_count_matches_append_tally() {
|
|||||||
assert_eq!(store.read_entry_count(sid, segid).unwrap(), entries.len());
|
assert_eq!(store.read_entry_count(sid, segid).unwrap(), entries.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unterminated_utf8_tail_is_ignored_and_replaced_on_append() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let store = FsStore::new(dir.path()).unwrap();
|
||||||
|
let sid = new_session_id();
|
||||||
|
let segid = new_segment_id();
|
||||||
|
let path = dir
|
||||||
|
.path()
|
||||||
|
.join(sid.to_string())
|
||||||
|
.join(format!("{segid}.jsonl"));
|
||||||
|
|
||||||
|
store
|
||||||
|
.append(sid, segid, &nil_session_start(1, sid))
|
||||||
|
.unwrap();
|
||||||
|
std::fs::OpenOptions::new()
|
||||||
|
.append(true)
|
||||||
|
.open(&path)
|
||||||
|
.unwrap()
|
||||||
|
// First byte of a three-byte UTF-8 code point, matching an ENOSPC
|
||||||
|
// partial write observed in a real session log.
|
||||||
|
.write_all(&[0xe3])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(store.read_all(sid, segid).unwrap().len(), 1);
|
||||||
|
assert_eq!(store.read_entry_count(sid, segid).unwrap(), 1);
|
||||||
|
|
||||||
|
let next = LogEntry::UserInput {
|
||||||
|
ts: 2,
|
||||||
|
segments: vec![protocol::Segment::text("recovered")],
|
||||||
|
};
|
||||||
|
store.append(sid, segid, &next).unwrap();
|
||||||
|
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
assert!(std::str::from_utf8(&bytes).is_ok());
|
||||||
|
assert_eq!(store.read_all(sid, segid).unwrap().len(), 2);
|
||||||
|
assert_eq!(store.read_entry_count(sid, segid).unwrap(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn newline_terminated_invalid_utf8_is_reported_as_corruption() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let store = FsStore::new(dir.path()).unwrap();
|
||||||
|
let sid = new_session_id();
|
||||||
|
let segid = new_segment_id();
|
||||||
|
let path = dir
|
||||||
|
.path()
|
||||||
|
.join(sid.to_string())
|
||||||
|
.join(format!("{segid}.jsonl"));
|
||||||
|
|
||||||
|
store
|
||||||
|
.append(sid, segid, &nil_session_start(1, sid))
|
||||||
|
.unwrap();
|
||||||
|
std::fs::OpenOptions::new()
|
||||||
|
.append(true)
|
||||||
|
.open(path)
|
||||||
|
.unwrap()
|
||||||
|
.write_all(&[0xe3, b'\n'])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
store.read_all(sid, segid),
|
||||||
|
Err(session_store::StoreError::Corrupt { line: 2, .. })
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn lookup_session_of_finds_owning_session() {
|
fn lookup_session_of_finds_owning_session() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -4966,7 +4966,7 @@ fn orchestrator_queue_notification_message(
|
|||||||
) -> String {
|
) -> String {
|
||||||
let title = ticket.title.replace(['\r', '\n'], " ");
|
let title = ticket.title.replace(['\r', '\n'], " ");
|
||||||
format!(
|
format!(
|
||||||
"Workspace Dashboard Queue for Ticket `{}`, title `{}`: human authorized Orchestrator routing; this is not an unattended scheduler. Read the Ticket and inspect current Orchestrator workspace state. If unblocked, record routing and transition state queued -> inprogress before any worktree/SpawnWorker implementation side effects. After inprogress acceptance, create the delegated implementation worktree with tracked `.yoi` project records visible and generated/local/runtime/log/lock/secret-like `.yoi` paths excluded, then run sibling coder/reviewer Workers through typed Ticket role launch surfaces. After reviewer approval and blocker resolution, integrate the implementation branch into the orchestration branch automatically, validate in the Orchestrator worktree, record the outcome, and clean up only child implementation worktrees/branches. Do not read, write, validate, merge, clean up, or run git operations in the root/original workspace. If blocked, record a concise reason and leave the Ticket queued or return it to planning with the missing-information reason.",
|
"Workspace Dashboard Queue for Ticket `{}`, title `{}`: human authorized Orchestrator routing; this is not an unattended scheduler. Read the Ticket and inspect current Orchestrator workspace state. If unblocked, record routing and transition state queued -> inprogress before any worktree/WorkerSpawn implementation side effects. After inprogress acceptance, create the delegated implementation worktree with tracked `.yoi` project records visible and generated/local/runtime/log/lock/secret-like `.yoi` paths excluded, then run sibling coder/reviewer Workers through typed Ticket role launch surfaces. After reviewer approval and blocker resolution, integrate the implementation branch into the orchestration branch automatically, validate in the Orchestrator worktree, record the outcome, and clean up only child implementation worktrees/branches. Do not read, write, validate, merge, clean up, or run git operations in the root/original workspace. If blocked, record a concise reason and leave the Ticket queued or return it to planning with the missing-information reason.",
|
||||||
ticket.id,
|
ticket.id,
|
||||||
title.trim()
|
title.trim()
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -823,7 +823,7 @@ fn ticket_queue_notification_message_carries_routing_contract() {
|
|||||||
assert!(message.contains("Read the Ticket"));
|
assert!(message.contains("Read the Ticket"));
|
||||||
assert!(message.contains("inspect current Orchestrator workspace state"));
|
assert!(message.contains("inspect current Orchestrator workspace state"));
|
||||||
assert!(message.contains("transition state queued -> inprogress"));
|
assert!(message.contains("transition state queued -> inprogress"));
|
||||||
assert!(message.contains("before any worktree/SpawnWorker implementation side effects"));
|
assert!(message.contains("before any worktree/WorkerSpawn implementation side effects"));
|
||||||
assert!(message.contains("After inprogress acceptance"));
|
assert!(message.contains("After inprogress acceptance"));
|
||||||
assert!(message.contains("implementation worktree"));
|
assert!(message.contains("implementation worktree"));
|
||||||
assert!(message.contains("tracked `.yoi` project records visible"));
|
assert!(message.contains("tracked `.yoi` project records visible"));
|
||||||
|
|||||||
@@ -233,7 +233,7 @@ enabled = true
|
|||||||
[feature.web]
|
[feature.web]
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
[feature.workers]
|
[feature.sub_worker]
|
||||||
enabled = false
|
enabled = false
|
||||||
|
|
||||||
[feature.ticket]
|
[feature.ticket]
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use crate::identity::{WorkerId, WorkerRef};
|
use crate::identity::{RuntimeWorkerRef, WorkerId, WorkerRef};
|
||||||
use crate::interaction::WorkerInput;
|
use crate::interaction::WorkerInput;
|
||||||
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef};
|
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -132,9 +132,8 @@ pub struct WorkingDirectoryCleanupTarget {
|
|||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct WorkingDirectoryOccupancy {
|
pub struct WorkingDirectoryOccupancy {
|
||||||
pub runtime_id: String,
|
#[serde(flatten)]
|
||||||
pub runtime_worker_id: u64,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub worker_id: String,
|
|
||||||
pub display_name: String,
|
pub display_name: String,
|
||||||
pub linked_at: String,
|
pub linked_at: String,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,6 +30,32 @@ impl fmt::Display for WorkerId {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Backend-visible Worker identity, namespaced by the Runtime that owns the Worker record.
|
||||||
|
///
|
||||||
|
/// This is intentionally distinct from [`WorkerRef`], which is meaningful only inside one
|
||||||
|
/// Runtime. Do not flatten this reference into a concatenated string for authority decisions.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||||
|
pub struct RuntimeWorkerRef {
|
||||||
|
pub runtime_id: String,
|
||||||
|
pub worker_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RuntimeWorkerRef {
|
||||||
|
pub fn new(runtime_id: impl Into<String>, worker_id: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
runtime_id: runtime_id.into(),
|
||||||
|
worker_id: worker_id.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn local_worker_ref(&self) -> Result<WorkerRef, std::num::ParseIntError> {
|
||||||
|
self.worker_id
|
||||||
|
.parse::<u64>()
|
||||||
|
.map(WorkerId::new)
|
||||||
|
.map(WorkerRef::new)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Runtime-local authority reference for Worker operations.
|
/// Runtime-local authority reference for Worker operations.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||||
pub struct WorkerRef {
|
pub struct WorkerRef {
|
||||||
@@ -41,3 +67,29 @@ impl WorkerRef {
|
|||||||
Self { worker_id }
|
Self { worker_id }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_worker_ref_preserves_structured_identity_and_json_fields() {
|
||||||
|
let worker = RuntimeWorkerRef::new("arcadia", "30");
|
||||||
|
assert_eq!(worker.runtime_id, "arcadia");
|
||||||
|
assert_eq!(worker.worker_id, "30");
|
||||||
|
assert_eq!(
|
||||||
|
worker.local_worker_ref().unwrap(),
|
||||||
|
WorkerRef::new(WorkerId::new(30))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&worker).unwrap(),
|
||||||
|
serde_json::json!({"runtime_id": "arcadia", "worker_id": "30"})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_worker_ref_does_not_treat_composite_text_as_local_worker_id() {
|
||||||
|
let worker = RuntimeWorkerRef::new("arcadia", "embedded-worker-runtime-5");
|
||||||
|
assert!(worker.local_worker_ref().is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum WorkerInputKind {
|
pub enum WorkerInputKind {
|
||||||
User,
|
User,
|
||||||
System,
|
Notify,
|
||||||
Compact,
|
Compact,
|
||||||
ListRewindTargets,
|
ListRewindTargets,
|
||||||
RegisterPeer,
|
RegisterPeer,
|
||||||
@@ -38,15 +38,35 @@ impl WorkerInput {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn system(content: impl Into<String>) -> Self {
|
pub fn notify(content: impl Into<String>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
kind: WorkerInputKind::System,
|
kind: WorkerInputKind::Notify,
|
||||||
content: content.into(),
|
content: content.into(),
|
||||||
segments: None,
|
segments: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::WorkerInput;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn notify_is_an_operation_and_legacy_system_kind_is_rejected() {
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(WorkerInput::notify("message")).unwrap(),
|
||||||
|
serde_json::json!({ "kind": "notify", "content": "message" })
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<WorkerInput>(serde_json::json!({
|
||||||
|
"kind": "system",
|
||||||
|
"content": "message"
|
||||||
|
}))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Acknowledgement returned after input is accepted into the Worker.
|
/// Acknowledgement returned after input is accepted into the Worker.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct WorkerInteractionAck {
|
pub struct WorkerInteractionAck {
|
||||||
|
|||||||
@@ -360,8 +360,11 @@ impl Runtime {
|
|||||||
self.annotate_working_directory_status(status)
|
self.annotate_working_directory_status(status)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Open a fresh Workdir operation session after proving that the persisted
|
/// Open a fresh Workdir operation session in this Runtime's authorized Workspace.
|
||||||
/// materialization is assigned to a Worker in the authorized workspace.
|
///
|
||||||
|
/// A same-Runtime owner Worker can be supplied as an additional persisted-binding check.
|
||||||
|
/// Cross-Runtime callers rely on the Runtime's Workspace capability scope and the existence
|
||||||
|
/// of the Runtime-owned materialization; Backend attachment remains occupancy authority.
|
||||||
pub fn open_workdir_session_scoped(
|
pub fn open_workdir_session_scoped(
|
||||||
&self,
|
&self,
|
||||||
scope: &RuntimeWorkspaceScope,
|
scope: &RuntimeWorkspaceScope,
|
||||||
@@ -373,33 +376,34 @@ impl Runtime {
|
|||||||
state.ensure_running()?;
|
state.ensure_running()?;
|
||||||
state.ensure_workspace_owner(scope, false)?;
|
state.ensure_workspace_owner(scope, false)?;
|
||||||
|
|
||||||
let owns_workdir = |worker: &WorkerRecord| {
|
if let Some(worker_ref) = owner_worker_ref {
|
||||||
|
let owns_workdir = state
|
||||||
|
.workers
|
||||||
|
.get(&worker_ref.worker_id)
|
||||||
|
.is_some_and(|worker| {
|
||||||
worker.belongs_to_workspace(&scope.workspace_id)
|
worker.belongs_to_workspace(&scope.workspace_id)
|
||||||
&& worker.working_directory.as_ref().is_some_and(|status| {
|
&& worker.working_directory.as_ref().is_some_and(|status| {
|
||||||
status.summary.working_directory_id == working_directory_id
|
status.summary.working_directory_id == working_directory_id
|
||||||
})
|
})
|
||||||
};
|
});
|
||||||
let authorized = match owner_worker_ref {
|
if !owns_workdir {
|
||||||
Some(worker_ref) => state
|
|
||||||
.workers
|
|
||||||
.get(&worker_ref.worker_id)
|
|
||||||
.is_some_and(owns_workdir),
|
|
||||||
None => state.workers.values().any(owns_workdir),
|
|
||||||
};
|
|
||||||
if !authorized {
|
|
||||||
return Err(RuntimeError::WorkingDirectory(
|
return Err(RuntimeError::WorkingDirectory(
|
||||||
crate::working_directory::WorkingDirectoryDiagnostic::rejected(
|
crate::working_directory::WorkingDirectoryDiagnostic::rejected(
|
||||||
"working_directory_not_found",
|
"working_directory_not_found",
|
||||||
"working directory was not found in the authorized workspace",
|
"working directory was not assigned to the authorized owner Worker",
|
||||||
),
|
),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
state.execution_backend.clone().ok_or_else(|| {
|
state.execution_backend.clone().ok_or_else(|| {
|
||||||
RuntimeError::ExecutionBackendUnavailable {
|
RuntimeError::ExecutionBackendUnavailable {
|
||||||
message: "opening a Workdir session requires an execution backend".to_string(),
|
message: "opening a Workdir session requires an execution backend".to_string(),
|
||||||
}
|
}
|
||||||
})?
|
})?
|
||||||
};
|
};
|
||||||
|
backend
|
||||||
|
.working_directory(working_directory_id)
|
||||||
|
.map_err(RuntimeError::WorkingDirectory)?;
|
||||||
backend
|
backend
|
||||||
.open_workdir_session(working_directory_id)
|
.open_workdir_session(working_directory_id)
|
||||||
.map_err(RuntimeError::WorkingDirectory)
|
.map_err(RuntimeError::WorkingDirectory)
|
||||||
@@ -2392,9 +2396,9 @@ fn input_protocol_event(input: &WorkerInput) -> protocol::Event {
|
|||||||
}]
|
}]
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
WorkerInputKind::System => protocol::Event::SystemItem {
|
WorkerInputKind::Notify => protocol::Event::SystemItem {
|
||||||
item: serde_json::json!({
|
item: serde_json::json!({
|
||||||
"kind": "embedded_worker_system_input",
|
"kind": "embedded_worker_notification",
|
||||||
"content": input.content.clone(),
|
"content": input.content.clone(),
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
@@ -3148,7 +3152,7 @@ mod tests {
|
|||||||
fn create_worker_rejects_system_initial_input_without_persisting_worker() {
|
fn create_worker_rejects_system_initial_input_without_persisting_worker() {
|
||||||
let runtime = runtime_with_backend();
|
let runtime = runtime_with_backend();
|
||||||
let mut request = task_request("system initial input");
|
let mut request = task_request("system initial input");
|
||||||
request.initial_input = Some(WorkerInput::system("role/system belongs in config bundle"));
|
request.initial_input = Some(WorkerInput::notify("role/system belongs in config bundle"));
|
||||||
|
|
||||||
let error = runtime.create_worker(request).unwrap_err();
|
let error = runtime.create_worker(request).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
@@ -3390,7 +3394,7 @@ mod tests {
|
|||||||
.send_input(&detail.worker_ref, WorkerInput::user("hello"))
|
.send_input(&detail.worker_ref, WorkerInput::user("hello"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime
|
runtime
|
||||||
.send_input(&detail.worker_ref, WorkerInput::system("note"))
|
.send_input(&detail.worker_ref, WorkerInput::notify("note"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let observations = runtime
|
let observations = runtime
|
||||||
@@ -3550,7 +3554,7 @@ mod tests {
|
|||||||
.send_input(&worker.worker_ref, WorkerInput::user("first"))
|
.send_input(&worker.worker_ref, WorkerInput::user("first"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime
|
runtime
|
||||||
.send_input(&worker.worker_ref, WorkerInput::system("second"))
|
.send_input(&worker.worker_ref, WorkerInput::notify("second"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime
|
runtime
|
||||||
.stop_worker(&worker.worker_ref, Some("finished".to_string()))
|
.stop_worker(&worker.worker_ref, Some("finished".to_string()))
|
||||||
|
|||||||
@@ -793,6 +793,14 @@ fn method_starts_turn(method: &Method) -> bool {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExecutionRunState {
|
||||||
|
match status {
|
||||||
|
WorkerStatus::Running => WorkerExecutionRunState::Busy,
|
||||||
|
WorkerStatus::Idle if auto_run => WorkerExecutionRunState::Busy,
|
||||||
|
WorkerStatus::Idle | WorkerStatus::Paused => WorkerExecutionRunState::Idle,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
|
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
|
||||||
match method {
|
match method {
|
||||||
Method::Run { .. }
|
Method::Run { .. }
|
||||||
@@ -1098,6 +1106,29 @@ where
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if input.kind == WorkerInputKind::Notify {
|
||||||
|
let status = worker.shared_state.get_status();
|
||||||
|
let accepted_run_state = accepted_notify_run_state(status, true);
|
||||||
|
let claimed_here = status == WorkerStatus::Idle
|
||||||
|
&& busy
|
||||||
|
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||||
|
.is_ok();
|
||||||
|
let result = self.send_method(
|
||||||
|
WorkerExecutionOperation::Input,
|
||||||
|
worker,
|
||||||
|
Method::Notify {
|
||||||
|
message: input.content,
|
||||||
|
auto_run: true,
|
||||||
|
},
|
||||||
|
accepted_run_state,
|
||||||
|
);
|
||||||
|
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
|
||||||
|
{
|
||||||
|
busy.store(false, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
if worker.shared_state.get_status() != WorkerStatus::Idle
|
if worker.shared_state.get_status() != WorkerStatus::Idle
|
||||||
|| busy
|
|| busy
|
||||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||||
@@ -1115,10 +1146,9 @@ where
|
|||||||
.segments
|
.segments
|
||||||
.unwrap_or_else(|| vec![Segment::text(input.content.trim().to_string())]),
|
.unwrap_or_else(|| vec![Segment::text(input.content.trim().to_string())]),
|
||||||
},
|
},
|
||||||
WorkerInputKind::System => Method::Notify {
|
WorkerInputKind::Notify => {
|
||||||
message: input.content,
|
unreachable!("Notify input is dispatched before the turn-start busy guard")
|
||||||
auto_run: true,
|
}
|
||||||
},
|
|
||||||
WorkerInputKind::Compact => Method::Compact,
|
WorkerInputKind::Compact => Method::Compact,
|
||||||
WorkerInputKind::ListRewindTargets => Method::ListRewindTargets,
|
WorkerInputKind::ListRewindTargets => Method::ListRewindTargets,
|
||||||
WorkerInputKind::RegisterPeer => Method::RegisterPeer {
|
WorkerInputKind::RegisterPeer => Method::RegisterPeer {
|
||||||
@@ -1159,6 +1189,28 @@ where
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if let Method::Notify { auto_run, .. } = &method {
|
||||||
|
let auto_run = *auto_run;
|
||||||
|
let status = worker.shared_state.get_status();
|
||||||
|
let accepted_run_state = accepted_notify_run_state(status, auto_run);
|
||||||
|
let claimed_here = status == WorkerStatus::Idle
|
||||||
|
&& auto_run
|
||||||
|
&& busy
|
||||||
|
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||||
|
.is_ok();
|
||||||
|
let result = self.send_method(
|
||||||
|
WorkerExecutionOperation::ProtocolMethod,
|
||||||
|
worker,
|
||||||
|
method,
|
||||||
|
accepted_run_state,
|
||||||
|
);
|
||||||
|
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
|
||||||
|
{
|
||||||
|
busy.store(false, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
let starts_turn = method_starts_turn(&method);
|
let starts_turn = method_starts_turn(&method);
|
||||||
if starts_turn
|
if starts_turn
|
||||||
&& (worker.shared_state.get_status() != WorkerStatus::Idle
|
&& (worker.shared_state.get_status() != WorkerStatus::Idle
|
||||||
@@ -1298,6 +1350,26 @@ mod tests {
|
|||||||
use manifest::{Scope, WorkerManifest};
|
use manifest::{Scope, WorkerManifest};
|
||||||
use session_store::WorkerMetadataStore;
|
use session_store::WorkerMetadataStore;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn notify_run_state_allows_running_worker_inbox_delivery() {
|
||||||
|
assert_eq!(
|
||||||
|
accepted_notify_run_state(WorkerStatus::Running, true),
|
||||||
|
WorkerExecutionRunState::Busy
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
accepted_notify_run_state(WorkerStatus::Idle, true),
|
||||||
|
WorkerExecutionRunState::Busy
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
accepted_notify_run_state(WorkerStatus::Idle, false),
|
||||||
|
WorkerExecutionRunState::Idle
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
accepted_notify_run_state(WorkerStatus::Paused, true),
|
||||||
|
WorkerExecutionRunState::Idle
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct MockClient {
|
struct MockClient {
|
||||||
responses: Arc<Vec<Vec<LlmEvent>>>,
|
responses: Arc<Vec<Vec<LlmEvent>>>,
|
||||||
|
|||||||
+117
-64
@@ -8,9 +8,7 @@ use session_store::WorkerMetadataStore;
|
|||||||
use session_store::{LogEntry, Store};
|
use session_store::{LogEntry, Store};
|
||||||
use tokio::sync::{broadcast, mpsc, oneshot};
|
use tokio::sync::{broadcast, mpsc, oneshot};
|
||||||
|
|
||||||
use crate::discovery::{
|
use crate::discovery::WorkerDiscovery;
|
||||||
WorkerDiscovery, list_workers_tool, restore_worker_tool, send_to_peer_worker_tool,
|
|
||||||
};
|
|
||||||
use crate::feature::FeatureRegistryBuilder;
|
use crate::feature::FeatureRegistryBuilder;
|
||||||
use crate::in_flight::{InFlightEvents, snapshot_from_guard};
|
use crate::in_flight::{InFlightEvents, snapshot_from_guard};
|
||||||
use crate::ipc::alerter::Alerter;
|
use crate::ipc::alerter::Alerter;
|
||||||
@@ -23,9 +21,11 @@ use crate::shutdown_after_idle::{
|
|||||||
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
||||||
take_shutdown_request_after_status,
|
take_shutdown_request_after_status,
|
||||||
};
|
};
|
||||||
use crate::spawn::comm_tools::{read_worker_output_tool, send_to_worker_tool, stop_worker_tool};
|
use crate::spawn::comm_tools::{
|
||||||
|
sub_worker_list_tool, sub_worker_read_output_tool, sub_worker_send_tool, sub_worker_stop_tool,
|
||||||
|
};
|
||||||
use crate::spawn::registry::SpawnedWorkerRegistry;
|
use crate::spawn::registry::SpawnedWorkerRegistry;
|
||||||
use crate::spawn::tool::spawn_worker_tool;
|
use crate::spawn::tool::sub_worker_spawn_tool;
|
||||||
use crate::worker::{SystemItemCommitter, Worker, WorkerError, WorkerRunResult};
|
use crate::worker::{SystemItemCommitter, Worker, WorkerError, WorkerRunResult};
|
||||||
use protocol::{
|
use protocol::{
|
||||||
AlertLevel, AlertSource, ErrorCode, Event, Method, RewindTargetId, RunResult, Segment,
|
AlertLevel, AlertSource, ErrorCode, Event, Method, RewindTargetId, RunResult, Segment,
|
||||||
@@ -285,6 +285,7 @@ impl WorkerController {
|
|||||||
worker.push_notify(
|
worker.push_notify(
|
||||||
"Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
"Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
||||||
.to_string(),
|
.to_string(),
|
||||||
|
false,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -335,7 +336,6 @@ impl WorkerController {
|
|||||||
let fs_for_view = register_worker_tools(
|
let fs_for_view = register_worker_tools(
|
||||||
&mut worker,
|
&mut worker,
|
||||||
bash_output_dir,
|
bash_output_dir,
|
||||||
runtime_dir.socket_path(),
|
|
||||||
runtime_base.to_path_buf(),
|
runtime_base.to_path_buf(),
|
||||||
spawned_registry.clone(),
|
spawned_registry.clone(),
|
||||||
)
|
)
|
||||||
@@ -581,13 +581,12 @@ fn wire_event_bridges_on_engine<C, St>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Register the builtin file-manipulation tools, optional memory tools,
|
/// Register the builtin file-manipulation tools, optional memory tools,
|
||||||
/// and the Worker-orchestration tools (SpawnWorker + comm) on the Worker's
|
/// and the Worker-orchestration tools (SubWorkerSpawn + comm) on the Worker's
|
||||||
/// Engine. Returns the WorkdirSession handle used to attach a `WorkerFsView` to
|
/// Engine. Returns the WorkdirSession handle used to attach a `WorkerFsView` to
|
||||||
/// the shared state.
|
/// the shared state.
|
||||||
async fn register_worker_tools<C, St>(
|
pub(crate) async fn register_worker_tools<C, St>(
|
||||||
worker: &mut Worker<C, St>,
|
worker: &mut Worker<C, St>,
|
||||||
bash_output_dir: PathBuf,
|
bash_output_dir: PathBuf,
|
||||||
spawner_socket: PathBuf,
|
|
||||||
runtime_base: PathBuf,
|
runtime_base: PathBuf,
|
||||||
spawned_registry: Arc<SpawnedWorkerRegistry>,
|
spawned_registry: Arc<SpawnedWorkerRegistry>,
|
||||||
) -> std::io::Result<Option<workdir::WorkdirSessionHandle>>
|
) -> std::io::Result<Option<workdir::WorkdirSessionHandle>>
|
||||||
@@ -598,6 +597,18 @@ where
|
|||||||
// Worker-immutable snapshots taken before the mutable worker borrow
|
// Worker-immutable snapshots taken before the mutable worker borrow
|
||||||
// below so the worker borrow doesn't conflict with reads on `worker`.
|
// below so the worker borrow doesn't conflict with reads on `worker`.
|
||||||
let scope_handle = worker.scope().clone();
|
let scope_handle = worker.scope().clone();
|
||||||
|
let feature_config = worker.manifest().feature.clone();
|
||||||
|
if feature_config.manage_workdir.enabled {
|
||||||
|
if let Some(existing) = worker.workdir_session().cloned() {
|
||||||
|
existing.close().await.map_err(std::io::Error::other)?;
|
||||||
|
}
|
||||||
|
let workspace_client = worker.workspace_client_handle();
|
||||||
|
worker.bind_workdir_session(Some(
|
||||||
|
crate::feature::builtin::manage_workdir::WorkspaceAttachedWorkdirSession::handle(
|
||||||
|
workspace_client,
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
let worker_workdir = worker.workdir_session().cloned();
|
let worker_workdir = worker.workdir_session().cloned();
|
||||||
let local_filesystem = worker.local_working_directory().cloned();
|
let local_filesystem = worker.local_working_directory().cloned();
|
||||||
let local_workspace_root = local_filesystem.as_ref().map(|local| local.root.clone());
|
let local_workspace_root = local_filesystem.as_ref().map(|local| local.root.clone());
|
||||||
@@ -605,13 +616,11 @@ where
|
|||||||
let memory_config = worker.manifest().memory.clone();
|
let memory_config = worker.manifest().memory.clone();
|
||||||
let web_config = worker.manifest().web.clone();
|
let web_config = worker.manifest().web.clone();
|
||||||
let mcp_config = worker.manifest().mcp.clone();
|
let mcp_config = worker.manifest().mcp.clone();
|
||||||
let feature_config = worker.manifest().feature.clone();
|
|
||||||
let spawner_name = worker.manifest().worker.name.clone();
|
let spawner_name = worker.manifest().worker.name.clone();
|
||||||
let spawner_manifest = worker.manifest().clone();
|
let spawner_manifest = worker.manifest().clone();
|
||||||
|
let spawner_workspace_context = worker.workspace_context_handle();
|
||||||
|
let parent_notifies = worker.notify_buffer_handle();
|
||||||
let prompts = worker.prompts().clone();
|
let prompts = worker.prompts().clone();
|
||||||
let worker_metadata_store = worker.store().clone();
|
|
||||||
let self_parent_socket = worker.callback_socket().cloned();
|
|
||||||
|
|
||||||
// Resolve the existing Worker–Workdir binding into the domain provider.
|
// Resolve the existing Worker–Workdir binding into the domain provider.
|
||||||
// Tools only consume the provider handle; they do not own its root, cwd,
|
// Tools only consume the provider handle; they do not own its root, cwd,
|
||||||
// scope, or lifecycle. No-workdir Workers expose no local tools.
|
// scope, or lifecycle. No-workdir Workers expose no local tools.
|
||||||
@@ -665,6 +674,39 @@ where
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if feature_config.manage_workdir.enabled {
|
||||||
|
// Workdir lifecycle is Workspace control-plane authority. The Worker
|
||||||
|
// receives only the injected WorkspaceClient and never Runtime URLs,
|
||||||
|
// repository paths, materializer handles, or cleanup sessions.
|
||||||
|
let workspace_client = worker.workspace_client_handle();
|
||||||
|
let has_workspace_identity = workspace_client.workspace_id().is_some_and(|workspace_id| {
|
||||||
|
!workspace_id.is_empty() && !workspace_id.chars().any(char::is_control)
|
||||||
|
});
|
||||||
|
if !workspace_client.is_available() || !has_workspace_identity {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidInput,
|
||||||
|
"manage Workdir tools require Backend Workspace API authority",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
feature_registry.add_module(
|
||||||
|
crate::feature::builtin::manage_workdir::manage_workdir_feature(workspace_client),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if feature_config.worker.enabled {
|
||||||
|
let workspace_client = worker.workspace_client_handle();
|
||||||
|
let has_workspace_identity = workspace_client.workspace_id().is_some_and(|workspace_id| {
|
||||||
|
!workspace_id.is_empty() && !workspace_id.chars().any(char::is_control)
|
||||||
|
});
|
||||||
|
if !workspace_client.is_available() || !has_workspace_identity {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidInput,
|
||||||
|
"Worker tools require Backend Workspace API authority",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
feature_registry.add_module(
|
||||||
|
crate::feature::builtin::manage_worker::manage_worker_feature(workspace_client),
|
||||||
|
);
|
||||||
|
}
|
||||||
for module in crate::feature::plugin::plugin_tool_features_if_enabled(
|
for module in crate::feature::plugin::plugin_tool_features_if_enabled(
|
||||||
feature_config.plugins.enabled,
|
feature_config.plugins.enabled,
|
||||||
&worker.manifest().plugins,
|
&worker.manifest().plugins,
|
||||||
@@ -679,7 +721,7 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if feature_config.workers.enabled {
|
if feature_config.sub_worker.enabled {
|
||||||
worker.register_worker_orchestration_instruction();
|
worker.register_worker_orchestration_instruction();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -737,18 +779,12 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Worker-orchestration tools (SpawnWorker + the four comm tools) share
|
// Worker-orchestration tools (SubWorkerSpawn + the four comm tools) share
|
||||||
// the Worker-scoped `SpawnedWorkerRegistry` (also consumed by the main
|
// the Worker-scoped `SpawnedWorkerRegistry` (also consumed by the main
|
||||||
// loop's `WorkerEvent` handler). Expose them only behind the explicit
|
// loop's `WorkerEvent` handler). Expose them only behind the explicit
|
||||||
// profile feature and require delegation authority up front so enabling
|
// profile feature and require delegation authority up front so enabling
|
||||||
// the surface cannot imply broad child scope by accident.
|
// the surface cannot imply broad child scope by accident.
|
||||||
if feature_config.workers.enabled {
|
if feature_config.sub_worker.enabled {
|
||||||
if spawner_manifest.delegation_scope.allow.is_empty() {
|
|
||||||
return Err(std::io::Error::new(
|
|
||||||
std::io::ErrorKind::InvalidInput,
|
|
||||||
"[feature.workers].enabled = true requires non-empty [[delegation_scope.allow]]",
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let spawner_cwd = local_filesystem
|
let spawner_cwd = local_filesystem
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|local| local.cwd.clone())
|
.map(|local| local.cwd.clone())
|
||||||
@@ -764,31 +800,22 @@ where
|
|||||||
"worker spawn tools require local Worker filesystem authority",
|
"worker spawn tools require local Worker filesystem authority",
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
engine.register_tool(spawn_worker_tool(
|
engine.register_tool(sub_worker_spawn_tool(
|
||||||
spawner_name.clone(),
|
spawner_name.clone(),
|
||||||
spawner_socket,
|
spawner_workspace_context,
|
||||||
|
parent_notifies,
|
||||||
runtime_base.clone(),
|
runtime_base.clone(),
|
||||||
spawner_workspace_root,
|
spawner_workspace_root,
|
||||||
spawner_cwd.clone(),
|
spawner_cwd.clone(),
|
||||||
spawned_registry.clone(),
|
spawned_registry.clone(),
|
||||||
self_parent_socket,
|
|
||||||
spawner_manifest,
|
spawner_manifest,
|
||||||
scope_handle,
|
scope_handle,
|
||||||
prompts,
|
prompts,
|
||||||
));
|
));
|
||||||
engine.register_tool(send_to_worker_tool(spawned_registry.clone()));
|
engine.register_tool(sub_worker_list_tool(spawned_registry.clone()));
|
||||||
engine.register_tool(read_worker_output_tool(spawned_registry.clone()));
|
engine.register_tool(sub_worker_send_tool(spawned_registry.clone()));
|
||||||
engine.register_tool(stop_worker_tool(spawned_registry.clone()));
|
engine.register_tool(sub_worker_read_output_tool(spawned_registry.clone()));
|
||||||
let discovery = WorkerDiscovery::new(
|
engine.register_tool(sub_worker_stop_tool(spawned_registry));
|
||||||
worker_metadata_store,
|
|
||||||
spawner_name,
|
|
||||||
runtime_base,
|
|
||||||
Some(spawner_cwd),
|
|
||||||
spawned_registry,
|
|
||||||
);
|
|
||||||
engine.register_tool(list_workers_tool(discovery.clone()));
|
|
||||||
engine.register_tool(restore_worker_tool(discovery.clone()));
|
|
||||||
engine.register_tool(send_to_peer_worker_tool(discovery));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let _feature_install_report = worker.install_features(feature_registry);
|
let _feature_install_report = worker.install_features(feature_registry);
|
||||||
@@ -865,7 +892,7 @@ async fn controller_loop<C, St>(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let parent_originated = run.is_parent_originated();
|
let parent_originated = run.is_parent_originated();
|
||||||
let (new_status, shutdown) = match run {
|
let (mut new_status, shutdown) = match run {
|
||||||
PendingRun::Run(input) => {
|
PendingRun::Run(input) => {
|
||||||
drive_turn(
|
drive_turn(
|
||||||
worker.run(input),
|
worker.run(input),
|
||||||
@@ -912,6 +939,11 @@ async fn controller_loop<C, St>(
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if !shutdown && new_status == WorkerStatus::Idle && notify_buffer.has_auto_run_pending()
|
||||||
|
{
|
||||||
|
pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify));
|
||||||
|
new_status = WorkerStatus::Running;
|
||||||
|
}
|
||||||
finish_controller_run(
|
finish_controller_run(
|
||||||
&mut worker,
|
&mut worker,
|
||||||
&shared_state,
|
&shared_state,
|
||||||
@@ -967,13 +999,13 @@ async fn controller_loop<C, St>(
|
|||||||
// `LogEntry::SystemItem` entry — drained out of the
|
// `LogEntry::SystemItem` entry — drained out of the
|
||||||
// notify buffer + broadcast through the sink. No
|
// notify buffer + broadcast through the sink. No
|
||||||
// separate echo here.
|
// separate echo here.
|
||||||
worker.push_notify(message);
|
worker.push_notify(message, auto_run);
|
||||||
// RUNNING / Paused: the buffer push is the entire
|
// RUNNING: the in-flight turn drains the buffer at its next
|
||||||
// operation; an in-flight turn (or the next
|
// pending_history_appends; if an auto-run notification remains
|
||||||
// Resume/Run) will drain it at its next
|
// at turn end, the Controller stages a follow-up notification
|
||||||
// pending_history_appends. IDLE: only `auto_run`
|
// turn. Paused notifications remain queued until Resume/Run.
|
||||||
// notifications stage RunForNotification; weak progress
|
// IDLE: `auto_run` notifications stage RunForNotification;
|
||||||
// notices stay queued until an explicit run/resume.
|
// weak progress notices stay queued until an explicit run.
|
||||||
if should_auto_run_notification(shared_state.get_status(), auto_run) {
|
if should_auto_run_notification(shared_state.get_status(), auto_run) {
|
||||||
pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify));
|
pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify));
|
||||||
}
|
}
|
||||||
@@ -1367,11 +1399,11 @@ where
|
|||||||
.into(),
|
.into(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Some(Method::Notify { message, .. }) => {
|
Some(Method::Notify { message, auto_run }) => {
|
||||||
// Live echo arrives via `Event::SystemItem` once
|
// Live echo arrives via `Event::SystemItem` once
|
||||||
// the in-flight turn's next `pending_history_appends`
|
// the in-flight turn's next `pending_history_appends`
|
||||||
// drains this entry through the interceptor.
|
// drains this entry through the interceptor.
|
||||||
notify_buffer.push_notify(message);
|
notify_buffer.push_notify(message, auto_run);
|
||||||
}
|
}
|
||||||
Some(Method::ListCompletions { .. }) => {}
|
Some(Method::ListCompletions { .. }) => {}
|
||||||
Some(Method::ListWorkers | Method::RestoreWorker { .. } | Method::RegisterPeer { .. }) => {
|
Some(Method::ListWorkers | Method::RestoreWorker { .. } | Method::RegisterPeer { .. }) => {
|
||||||
@@ -1545,7 +1577,6 @@ fn worker_error_code(e: &WorkerError) -> ErrorCode {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::runtime::dir::SpawnedWorkerRecord;
|
|
||||||
use protocol::WorkerEvent;
|
use protocol::WorkerEvent;
|
||||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -1802,17 +1833,8 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn running_scope_sub_delegated_applies_side_effects_without_notify_buffer() {
|
async fn running_legacy_scope_callback_has_no_registry_authority_or_notify() {
|
||||||
let mut env = make_env().await;
|
let mut env = make_env().await;
|
||||||
env.spawned_registry
|
|
||||||
.add(SpawnedWorkerRecord {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
socket_path: "/tmp/child.sock".into(),
|
|
||||||
scope_delegated: vec![],
|
|
||||||
callback_address: "/tmp/parent.sock".into(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.expect("seed child record");
|
|
||||||
env._method_tx
|
env._method_tx
|
||||||
.send(Method::WorkerEvent(WorkerEvent::ScopeSubDelegated {
|
.send(Method::WorkerEvent(WorkerEvent::ScopeSubDelegated {
|
||||||
parent_worker: "child".into(),
|
parent_worker: "child".into(),
|
||||||
@@ -1843,13 +1865,9 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(status, WorkerStatus::Idle);
|
assert_eq!(status, WorkerStatus::Idle);
|
||||||
assert!(!shutdown);
|
assert!(!shutdown);
|
||||||
assert!(
|
|
||||||
env.spawned_registry.get("grandchild").await.is_some(),
|
|
||||||
"ScopeSubDelegated side effects must still register the grandchild"
|
|
||||||
);
|
|
||||||
assert!(
|
assert!(
|
||||||
env.notify_buffer.is_empty(),
|
env.notify_buffer.is_empty(),
|
||||||
"control-plane-only ScopeSubDelegated must not enter the agent-visible notify buffer"
|
"legacy ScopeSubDelegated must not enter the agent-visible notify buffer"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1886,6 +1904,41 @@ mod tests {
|
|||||||
assert_eq!(env.notify_buffer.len(), 1);
|
assert_eq!(env.notify_buffer.len(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn running_auto_run_notify_remains_staged_for_followup_turn() {
|
||||||
|
let mut env = make_env().await;
|
||||||
|
env._method_tx
|
||||||
|
.send(Method::Notify {
|
||||||
|
message: "continue".into(),
|
||||||
|
auto_run: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("send notify");
|
||||||
|
|
||||||
|
let worker_future = async {
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
Ok::<_, WorkerError>(WorkerRunResult::Finished)
|
||||||
|
};
|
||||||
|
let (status, shutdown) = drive_turn(
|
||||||
|
worker_future,
|
||||||
|
&mut env.method_rx,
|
||||||
|
&env.event_tx,
|
||||||
|
&env.cancel_tx,
|
||||||
|
&env.shared_state,
|
||||||
|
&env.notify_buffer,
|
||||||
|
Some(&env.parent_socket_path),
|
||||||
|
"parent",
|
||||||
|
&env.spawned_registry,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(status, WorkerStatus::Idle);
|
||||||
|
assert!(!shutdown);
|
||||||
|
assert_eq!(env.notify_buffer.len(), 1);
|
||||||
|
assert!(env.notify_buffer.has_auto_run_pending());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn compact_method_is_rejected_while_running() {
|
async fn compact_method_is_rejected_while_running() {
|
||||||
let mut env = make_env().await;
|
let mut env = make_env().await;
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ use session_store::{
|
|||||||
use tokio::net::UnixStream;
|
use tokio::net::UnixStream;
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
|
|
||||||
use crate::runtime::dir::SpawnedWorkerRecord;
|
|
||||||
use crate::runtime::worker_allocation;
|
use crate::runtime::worker_allocation;
|
||||||
use crate::spawn::comm_tools::connect_and_send;
|
use crate::spawn::comm_tools::connect_and_send;
|
||||||
use crate::spawn::registry::SpawnedWorkerRegistry;
|
use crate::spawn::registry::SpawnedWorkerRegistry;
|
||||||
@@ -44,7 +43,6 @@ pub struct WorkerDiscovery<St> {
|
|||||||
runtime_base: PathBuf,
|
runtime_base: PathBuf,
|
||||||
cwd: Option<PathBuf>,
|
cwd: Option<PathBuf>,
|
||||||
store_dir: Option<PathBuf>,
|
store_dir: Option<PathBuf>,
|
||||||
spawned_registry: Arc<SpawnedWorkerRegistry>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<St> WorkerDiscovery<St>
|
impl<St> WorkerDiscovery<St>
|
||||||
@@ -56,7 +54,7 @@ where
|
|||||||
self_worker_name: String,
|
self_worker_name: String,
|
||||||
runtime_base: PathBuf,
|
runtime_base: PathBuf,
|
||||||
cwd: Option<PathBuf>,
|
cwd: Option<PathBuf>,
|
||||||
spawned_registry: Arc<SpawnedWorkerRegistry>,
|
_spawned_registry: Arc<SpawnedWorkerRegistry>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let store_dir = store.root_dir();
|
let store_dir = store.root_dir();
|
||||||
Self {
|
Self {
|
||||||
@@ -65,7 +63,6 @@ where
|
|||||||
runtime_base,
|
runtime_base,
|
||||||
cwd,
|
cwd,
|
||||||
store_dir,
|
store_dir,
|
||||||
spawned_registry,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -250,20 +247,6 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The live in-memory registry covers just-spawned children even if a
|
|
||||||
// state write failed after the process became reachable. It is an
|
|
||||||
// additive visibility hint, not the source of Worker metadata.
|
|
||||||
for record in self.spawned_registry.list().await {
|
|
||||||
visible
|
|
||||||
.entry(record.worker_name.clone())
|
|
||||||
.or_insert(VisibilityReason::SpawnedChild);
|
|
||||||
child_sockets.insert(record.worker_name.clone(), record.socket_path.clone());
|
|
||||||
comm_registry.insert(
|
|
||||||
record.worker_name.clone(),
|
|
||||||
CommRegistryInfo::from_record(&record),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(VisibilitySet {
|
Ok(VisibilitySet {
|
||||||
visible,
|
visible,
|
||||||
child_sockets,
|
child_sockets,
|
||||||
@@ -569,14 +552,6 @@ impl CommRegistryInfo {
|
|||||||
scope_delegated: Vec::new(),
|
scope_delegated: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn from_record(record: &SpawnedWorkerRecord) -> Self {
|
|
||||||
Self {
|
|
||||||
registered: true,
|
|
||||||
socket_path: Some(record.socket_path.clone()),
|
|
||||||
scope_delegated: record.scope_delegated.clone(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ struct Cli {
|
|||||||
|
|
||||||
/// Claim a scope allocation pre-registered by a spawning Worker, rather
|
/// Claim a scope allocation pre-registered by a spawning Worker, rather
|
||||||
/// than installing a new top-level allocation. Used only when this
|
/// than installing a new top-level allocation. Used only when this
|
||||||
/// process is launched by `SpawnWorker`; end users should never pass it.
|
/// process is launched by `SubWorkerSpawn`; end users should never pass it.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
adopt: bool,
|
adopt: bool,
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
//! same descriptor-approved registry path used by feature modules. They are not
|
//! same descriptor-approved registry path used by feature modules. They are not
|
||||||
//! an external plugin-loading surface.
|
//! an external plugin-loading surface.
|
||||||
|
|
||||||
|
pub mod manage_workdir;
|
||||||
|
pub mod manage_worker;
|
||||||
pub mod memory;
|
pub mod memory;
|
||||||
pub mod objective;
|
pub mod objective;
|
||||||
pub mod session_explore;
|
pub mod session_explore;
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,361 @@
|
|||||||
|
//! Workspace-authority-backed Worker session management tools.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use llm_engine::tool::{
|
||||||
|
Tool, ToolDefinition, ToolError, ToolExecutionContext, ToolMeta, ToolOutput,
|
||||||
|
};
|
||||||
|
use schemars::JsonSchema;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::feature::{
|
||||||
|
FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureModule, ToolContribution,
|
||||||
|
ToolDeclaration,
|
||||||
|
};
|
||||||
|
use crate::worker::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod};
|
||||||
|
|
||||||
|
const FEATURE_ID: &str = "worker";
|
||||||
|
const FEATURE_NAME: &str = "Worker";
|
||||||
|
const FEATURE_DESCRIPTION: &str =
|
||||||
|
"Workspace-authority tools for managing Workdir-bound Backend/Runtime Worker sessions.";
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct ManageWorkerFeature {
|
||||||
|
client: Arc<dyn WorkspaceClient>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn manage_worker_feature(client: Arc<dyn WorkspaceClient>) -> ManageWorkerFeature {
|
||||||
|
ManageWorkerFeature { client }
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FeatureModule for ManageWorkerFeature {
|
||||||
|
fn descriptor(&self) -> FeatureDescriptor {
|
||||||
|
let mut descriptor = FeatureDescriptor::builtin(FEATURE_ID, FEATURE_NAME)
|
||||||
|
.with_description(FEATURE_DESCRIPTION);
|
||||||
|
for operation in WorkerOperation::ALL {
|
||||||
|
descriptor = descriptor.with_tool(ToolDeclaration::new(
|
||||||
|
operation.tool_name(),
|
||||||
|
operation.description(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
descriptor
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> {
|
||||||
|
let workspace_id = self
|
||||||
|
.client
|
||||||
|
.workspace_id()
|
||||||
|
.filter(|value| !value.trim().is_empty())
|
||||||
|
.ok_or_else(|| {
|
||||||
|
FeatureInstallError::InvalidDescriptor(
|
||||||
|
"worker feature requires a Workspace id".to_string(),
|
||||||
|
)
|
||||||
|
})?
|
||||||
|
.to_string();
|
||||||
|
for operation in WorkerOperation::ALL {
|
||||||
|
let definition = match operation {
|
||||||
|
WorkerOperation::List => definition::<WorkerListInput>(
|
||||||
|
operation,
|
||||||
|
self.client.clone(),
|
||||||
|
workspace_id.clone(),
|
||||||
|
),
|
||||||
|
WorkerOperation::Spawn => definition::<WorkerSpawnInput>(
|
||||||
|
operation,
|
||||||
|
self.client.clone(),
|
||||||
|
workspace_id.clone(),
|
||||||
|
),
|
||||||
|
WorkerOperation::Stop => definition::<WorkerStopInput>(
|
||||||
|
operation,
|
||||||
|
self.client.clone(),
|
||||||
|
workspace_id.clone(),
|
||||||
|
),
|
||||||
|
WorkerOperation::Restore => definition::<WorkerTargetInput>(
|
||||||
|
operation,
|
||||||
|
self.client.clone(),
|
||||||
|
workspace_id.clone(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
context
|
||||||
|
.tools()
|
||||||
|
.register(ToolContribution::new(operation.tool_name(), definition))?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct WorkerListInput {}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct WorkerSpawnInput {
|
||||||
|
runtime_id: String,
|
||||||
|
working_directory_id: String,
|
||||||
|
profile: String,
|
||||||
|
#[serde(default)]
|
||||||
|
display_name: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
initial_text: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
relative_cwd: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct WorkerSpawnRequest {
|
||||||
|
runtime_id: String,
|
||||||
|
display_name: String,
|
||||||
|
profile: String,
|
||||||
|
initial_text: String,
|
||||||
|
working_directory: WorkerWorkingDirectorySelection,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct WorkerWorkingDirectorySelection {
|
||||||
|
working_directory_id: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
relative_cwd: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct WorkerTargetInput {
|
||||||
|
runtime_id: String,
|
||||||
|
worker_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct WorkerStopInput {
|
||||||
|
runtime_id: String,
|
||||||
|
worker_id: String,
|
||||||
|
#[serde(default)]
|
||||||
|
reason: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct WorkspaceWorkerTool {
|
||||||
|
operation: WorkerOperation,
|
||||||
|
client: Arc<dyn WorkspaceClient>,
|
||||||
|
workspace_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
enum WorkerOperation {
|
||||||
|
List,
|
||||||
|
Spawn,
|
||||||
|
Stop,
|
||||||
|
Restore,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerOperation {
|
||||||
|
const ALL: [Self; 4] = [Self::List, Self::Spawn, Self::Stop, Self::Restore];
|
||||||
|
|
||||||
|
fn tool_name(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::List => "WorkerList",
|
||||||
|
Self::Spawn => "WorkerSpawn",
|
||||||
|
Self::Stop => "WorkerStop",
|
||||||
|
Self::Restore => "WorkerRestore",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn description(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::List => {
|
||||||
|
"List Backend/Runtime Worker sessions in the current Workspace. SubWorkers are excluded."
|
||||||
|
}
|
||||||
|
Self::Spawn => {
|
||||||
|
"Spawn a Backend/Runtime Worker session in an existing Workspace Workdir. The Workdir id is authority; filesystem paths and Runtime URLs are not accepted."
|
||||||
|
}
|
||||||
|
Self::Stop => "Stop a Backend/Runtime Worker session in the current Workspace.",
|
||||||
|
Self::Restore => {
|
||||||
|
"Restore a stopped Backend/Runtime Worker session in the current Workspace."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Tool for WorkspaceWorkerTool {
|
||||||
|
async fn execute(
|
||||||
|
&self,
|
||||||
|
input_json: &str,
|
||||||
|
_ctx: ToolExecutionContext,
|
||||||
|
) -> Result<ToolOutput, ToolError> {
|
||||||
|
let request = match self.operation {
|
||||||
|
WorkerOperation::List => {
|
||||||
|
parse::<WorkerListInput>(input_json, "WorkerList")?;
|
||||||
|
WorkspaceRequest::get(format!("/api/w/{}/workers", self.workspace_id))
|
||||||
|
}
|
||||||
|
WorkerOperation::Spawn => {
|
||||||
|
let input = parse::<WorkerSpawnInput>(input_json, "WorkerSpawn")?;
|
||||||
|
let request = WorkerSpawnRequest {
|
||||||
|
runtime_id: authority_id(&input.runtime_id, "runtime_id")?,
|
||||||
|
display_name: input
|
||||||
|
.display_name
|
||||||
|
.filter(|value| !value.trim().is_empty())
|
||||||
|
.unwrap_or_else(|| "Workspace Worker".to_string()),
|
||||||
|
profile: non_empty(input.profile, "profile")?,
|
||||||
|
initial_text: input.initial_text.unwrap_or_default(),
|
||||||
|
working_directory: WorkerWorkingDirectorySelection {
|
||||||
|
working_directory_id: authority_id(
|
||||||
|
&input.working_directory_id,
|
||||||
|
"working_directory_id",
|
||||||
|
)?,
|
||||||
|
relative_cwd: input
|
||||||
|
.relative_cwd
|
||||||
|
.map(|value| validate_relative_cwd(&value))
|
||||||
|
.transpose()?,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
WorkspaceRequest::json(
|
||||||
|
WorkspaceRequestMethod::Post,
|
||||||
|
format!("/api/w/{}/workers", self.workspace_id),
|
||||||
|
serde_json::to_string(&request)
|
||||||
|
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
WorkerOperation::Stop => {
|
||||||
|
let input = parse::<WorkerStopInput>(input_json, "WorkerStop")?;
|
||||||
|
let runtime_id = authority_id(&input.runtime_id, "runtime_id")?;
|
||||||
|
let worker_id = authority_id(&input.worker_id, "worker_id")?;
|
||||||
|
WorkspaceRequest::json(
|
||||||
|
WorkspaceRequestMethod::Post,
|
||||||
|
format!(
|
||||||
|
"/api/w/{}/runtimes/{runtime_id}/workers/{worker_id}/stop",
|
||||||
|
self.workspace_id
|
||||||
|
),
|
||||||
|
serde_json::json!({ "reason": input.reason }).to_string(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
WorkerOperation::Restore => {
|
||||||
|
let input = parse::<WorkerTargetInput>(input_json, "WorkerRestore")?;
|
||||||
|
let runtime_id = authority_id(&input.runtime_id, "runtime_id")?;
|
||||||
|
let worker_id = authority_id(&input.worker_id, "worker_id")?;
|
||||||
|
WorkspaceRequest::json(
|
||||||
|
WorkspaceRequestMethod::Post,
|
||||||
|
format!(
|
||||||
|
"/api/w/{}/runtimes/{runtime_id}/workers/{worker_id}/restore",
|
||||||
|
self.workspace_id
|
||||||
|
),
|
||||||
|
"{}",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let response = self
|
||||||
|
.client
|
||||||
|
.execute(request)
|
||||||
|
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||||
|
if !response.is_success() {
|
||||||
|
return Err(ToolError::ExecutionFailed(format!(
|
||||||
|
"Workspace Worker operation returned HTTP {}: {}",
|
||||||
|
response.status, response.body
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(ToolOutput {
|
||||||
|
summary: format!("{} completed", self.operation.tool_name()),
|
||||||
|
content: Some(response.body),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn definition<I: JsonSchema + 'static>(
|
||||||
|
operation: WorkerOperation,
|
||||||
|
client: Arc<dyn WorkspaceClient>,
|
||||||
|
workspace_id: String,
|
||||||
|
) -> ToolDefinition {
|
||||||
|
Arc::new(move || {
|
||||||
|
let schema = schemars::schema_for!(I);
|
||||||
|
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||||
|
let meta = ToolMeta::new(operation.tool_name())
|
||||||
|
.description(operation.description())
|
||||||
|
.input_schema(schema_value);
|
||||||
|
let tool: Arc<dyn Tool> = Arc::new(WorkspaceWorkerTool {
|
||||||
|
operation,
|
||||||
|
client: client.clone(),
|
||||||
|
workspace_id: workspace_id.clone(),
|
||||||
|
});
|
||||||
|
(meta, tool)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse<T: for<'de> Deserialize<'de>>(input: &str, tool: &str) -> Result<T, ToolError> {
|
||||||
|
serde_json::from_str(input)
|
||||||
|
.map_err(|error| ToolError::InvalidArgument(format!("invalid {tool} input: {error}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn authority_id(value: &str, field: &str) -> Result<String, ToolError> {
|
||||||
|
let value = non_empty(value.to_string(), field)?;
|
||||||
|
if value.contains('/') || value.contains('?') || value.contains('#') {
|
||||||
|
return Err(ToolError::InvalidArgument(format!(
|
||||||
|
"{field} must be an authority id, not a path or URL"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn non_empty(value: String, field: &str) -> Result<String, ToolError> {
|
||||||
|
let value = value.trim().to_string();
|
||||||
|
if value.is_empty() {
|
||||||
|
return Err(ToolError::InvalidArgument(format!(
|
||||||
|
"{field} must not be empty"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_relative_cwd(value: &str) -> Result<String, ToolError> {
|
||||||
|
let value = value.trim();
|
||||||
|
if value.is_empty()
|
||||||
|
|| value.starts_with('/')
|
||||||
|
|| value.split('/').any(|part| matches!(part, "" | "." | ".."))
|
||||||
|
{
|
||||||
|
return Err(ToolError::InvalidArgument(
|
||||||
|
"relative_cwd must be a normalized relative path inside the Workdir".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(value.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_tool_family_is_distinct_from_sub_worker_tools() {
|
||||||
|
assert_eq!(
|
||||||
|
WorkerOperation::ALL.map(WorkerOperation::tool_name),
|
||||||
|
["WorkerList", "WorkerSpawn", "WorkerStop", "WorkerRestore"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_spawn_request_uses_authority_ids_without_runtime_paths() {
|
||||||
|
let request = WorkerSpawnRequest {
|
||||||
|
runtime_id: "runtime-1".to_string(),
|
||||||
|
display_name: "Coder".to_string(),
|
||||||
|
profile: "builtin:coder".to_string(),
|
||||||
|
initial_text: "Implement the Ticket".to_string(),
|
||||||
|
working_directory: WorkerWorkingDirectorySelection {
|
||||||
|
working_directory_id: "wd-1".to_string(),
|
||||||
|
relative_cwd: Some("repo".to_string()),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let value = serde_json::to_value(request).unwrap();
|
||||||
|
assert_eq!(value["runtime_id"], "runtime-1");
|
||||||
|
assert_eq!(value["working_directory"]["working_directory_id"], "wd-1");
|
||||||
|
assert!(value.get("cwd").is_none());
|
||||||
|
assert!(value.get("runtime_url").is_none());
|
||||||
|
assert!(value["working_directory"].get("mode").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_inputs_reject_paths_and_parent_traversal() {
|
||||||
|
assert!(authority_id("https://runtime.example", "runtime_id").is_err());
|
||||||
|
assert!(authority_id("runtime/id", "runtime_id").is_err());
|
||||||
|
assert!(validate_relative_cwd("../repo").is_err());
|
||||||
|
assert!(validate_relative_cwd("/repo").is_err());
|
||||||
|
assert_eq!(validate_relative_cwd("repo/src").unwrap(), "repo/src");
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -26,9 +26,8 @@
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use protocol::{Method, ScopeRule, WorkerEvent};
|
use protocol::{Method, WorkerEvent};
|
||||||
|
|
||||||
use crate::runtime::dir::SpawnedWorkerRecord;
|
|
||||||
use crate::spawn::comm_tools::connect_and_send;
|
use crate::spawn::comm_tools::connect_and_send;
|
||||||
use crate::spawn::registry::SpawnedWorkerRegistry;
|
use crate::spawn::registry::SpawnedWorkerRegistry;
|
||||||
|
|
||||||
@@ -59,7 +58,7 @@ pub fn fire_and_forget(socket: Option<PathBuf>, event: WorkerEvent) {
|
|||||||
/// Only events classified by `WorkerEvent::should_notify_agent` are injected
|
/// Only events classified by `WorkerEvent::should_notify_agent` are injected
|
||||||
/// into the parent's LLM context as system messages; control-plane-only events
|
/// into the parent's LLM context as system messages; control-plane-only events
|
||||||
/// keep this renderer for diagnostics/tests. Agent-visible summaries are kept
|
/// keep this renderer for diagnostics/tests. Agent-visible summaries are kept
|
||||||
/// deliberately short — the LLM can always call `ReadWorkerOutput` to fetch more
|
/// deliberately short — the LLM can always call `SubWorkerReadOutput` to fetch more
|
||||||
/// detail if the event summary is not enough.
|
/// detail if the event summary is not enough.
|
||||||
pub fn render_event(event: &WorkerEvent) -> String {
|
pub fn render_event(event: &WorkerEvent) -> String {
|
||||||
match event {
|
match event {
|
||||||
@@ -85,86 +84,15 @@ pub fn render_event(event: &WorkerEvent) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Apply the variant-specific side effect on the parent side.
|
/// Legacy process callback events have no SubWorker registry authority.
|
||||||
///
|
///
|
||||||
/// All operations are idempotent so that out-of-order delivery (e.g.
|
/// Internal SubWorker lifecycle is applied directly through typed session handles. A callback from
|
||||||
/// `TurnEnded` arriving after `ShutDown`) does not produce errors:
|
/// an externally adopted Worker may still be rendered for diagnostics, but it cannot add/remove
|
||||||
///
|
/// Internal children or transfer filesystem authority.
|
||||||
/// - `TurnEnded` / `Errored`: no system work; the LLM handles the
|
|
||||||
/// semantic response.
|
|
||||||
/// - `ShutDown`: remove the child from `spawned_workers.json`, Worker state,
|
|
||||||
/// and reclaim its delegated scope/allocation. Missing entries are swallowed.
|
|
||||||
/// - `ScopeSubDelegated`: register the grandchild locally and re-emit
|
|
||||||
/// upward to our own parent if we have one. Duplicate grandchild
|
|
||||||
/// entries (re-delivery) are swallowed.
|
|
||||||
pub async fn apply_event_side_effects(
|
pub async fn apply_event_side_effects(
|
||||||
event: &WorkerEvent,
|
_event: &WorkerEvent,
|
||||||
registry: &Arc<SpawnedWorkerRegistry>,
|
_registry: &Arc<SpawnedWorkerRegistry>,
|
||||||
self_name: &str,
|
_self_name: &str,
|
||||||
self_parent_socket: &Option<PathBuf>,
|
_self_parent_socket: &Option<PathBuf>,
|
||||||
) {
|
) {
|
||||||
match event {
|
|
||||||
WorkerEvent::TurnEnded { .. } | WorkerEvent::Errored { .. } => {}
|
|
||||||
|
|
||||||
WorkerEvent::ShutDown { worker_name } => {
|
|
||||||
if let Err(e) = registry.remove(worker_name).await {
|
|
||||||
tracing::warn!(error = %e, worker = %worker_name, "registry remove on ShutDown failed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
WorkerEvent::ScopeSubDelegated {
|
|
||||||
parent_worker,
|
|
||||||
sub_worker,
|
|
||||||
sub_socket,
|
|
||||||
scope,
|
|
||||||
} => {
|
|
||||||
if registry.get(sub_worker).await.is_some() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let callback_address = registry
|
|
||||||
.get(parent_worker)
|
|
||||||
.await
|
|
||||||
.map(|r| r.socket_path)
|
|
||||||
.unwrap_or_else(PathBuf::new);
|
|
||||||
let record = SpawnedWorkerRecord {
|
|
||||||
worker_name: sub_worker.clone(),
|
|
||||||
socket_path: sub_socket.clone(),
|
|
||||||
scope_delegated: scope.clone(),
|
|
||||||
callback_address,
|
|
||||||
};
|
|
||||||
if let Err(e) = registry.add(record).await {
|
|
||||||
tracing::warn!(
|
|
||||||
error = %e,
|
|
||||||
sub_worker = %sub_worker,
|
|
||||||
"registry add on ScopeSubDelegated failed"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
reemit_scope_sub_delegated(
|
|
||||||
self_parent_socket,
|
|
||||||
self_name,
|
|
||||||
sub_worker.clone(),
|
|
||||||
sub_socket.clone(),
|
|
||||||
scope.clone(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn reemit_scope_sub_delegated(
|
|
||||||
self_parent_socket: &Option<PathBuf>,
|
|
||||||
self_name: &str,
|
|
||||||
sub_worker: String,
|
|
||||||
sub_socket: PathBuf,
|
|
||||||
scope: Vec<ScopeRule>,
|
|
||||||
) {
|
|
||||||
let Some(parent_socket) = self_parent_socket.clone() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
let event = WorkerEvent::ScopeSubDelegated {
|
|
||||||
parent_worker: self_name.to_string(),
|
|
||||||
sub_worker,
|
|
||||||
sub_socket,
|
|
||||||
scope,
|
|
||||||
};
|
|
||||||
fire_and_forget(Some(parent_socket), event);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -112,13 +112,14 @@ impl WorkerInterceptor {
|
|||||||
/// `Item::system_message`s reach the worker via
|
/// `Item::system_message`s reach the worker via
|
||||||
/// `ContinueWith` / `pending_history_appends`, so on-disk order
|
/// `ContinueWith` / `pending_history_appends`, so on-disk order
|
||||||
/// matches worker-history order.
|
/// matches worker-history order.
|
||||||
fn commit_system_items(&self, items: &[SystemItem]) {
|
fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> {
|
||||||
let Some(writer) = self.log_writer.as_ref() else {
|
let Some(writer) = self.log_writer.as_ref() else {
|
||||||
return;
|
return Ok(());
|
||||||
};
|
};
|
||||||
for item in items {
|
for item in items {
|
||||||
writer.commit_system_item(item.clone());
|
writer.commit_system_item(item.clone())?;
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn current_turn_index(&self) -> usize {
|
fn current_turn_index(&self) -> usize {
|
||||||
@@ -194,15 +195,17 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
// `Item::system_message`s, so on-disk order matches
|
// `Item::system_message`s, so on-disk order matches
|
||||||
// worker-history order.
|
// worker-history order.
|
||||||
let items: Vec<Item> = extras.iter().map(SystemItem::to_history_item).collect();
|
let items: Vec<Item> = extras.iter().map(SystemItem::to_history_item).collect();
|
||||||
self.commit_system_items(&extras);
|
match self.commit_system_items(&extras) {
|
||||||
PromptAction::ContinueWith(items)
|
Ok(()) => PromptAction::ContinueWith(items),
|
||||||
|
Err(error) => PromptAction::Cancel(format!("session persistence failed: {error}")),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pending_history_appends(&self) -> Vec<Item> {
|
async fn pending_history_appends(&self) -> Result<Vec<Item>, String> {
|
||||||
let drained = self.pending_notifies.drain();
|
let drained = self.pending_notifies.drain();
|
||||||
if drained.is_empty() {
|
if drained.is_empty() {
|
||||||
return Vec::new();
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut system_items: Vec<SystemItem> = Vec::with_capacity(drained.len());
|
let mut system_items: Vec<SystemItem> = Vec::with_capacity(drained.len());
|
||||||
@@ -220,7 +223,9 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
// simply be skipped from the SystemItem batch.
|
// simply be skipped from the SystemItem batch.
|
||||||
warn!(error = %e, "failed to render notify_wrapper; using raw message");
|
warn!(error = %e, "failed to render notify_wrapper; using raw message");
|
||||||
let fallback = match &entry {
|
let fallback = match &entry {
|
||||||
super::notify_buffer::PendingNotify::Notify { message } => message.clone(),
|
super::notify_buffer::PendingNotify::Notify { message, .. } => {
|
||||||
|
message.clone()
|
||||||
|
}
|
||||||
super::notify_buffer::PendingNotify::WorkerEvent { event } => {
|
super::notify_buffer::PendingNotify::WorkerEvent { event } => {
|
||||||
session_store::render_worker_event(event)
|
session_store::render_worker_event(event)
|
||||||
}
|
}
|
||||||
@@ -229,8 +234,9 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.commit_system_items(&system_items);
|
self.commit_system_items(&system_items)
|
||||||
items
|
.map_err(|error| format!("session persistence failed: {error}"))?;
|
||||||
|
Ok(items)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pre_llm_request(&self, context: &mut Vec<Item>) -> PreRequestAction {
|
async fn pre_llm_request(&self, context: &mut Vec<Item>) -> PreRequestAction {
|
||||||
@@ -276,7 +282,9 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
let current_tokens = self.estimated_tokens(effective_context.as_ref());
|
let current_tokens = self.estimated_tokens(effective_context.as_ref());
|
||||||
|
|
||||||
if self.request_threshold_exceeded(current_tokens, effective_context.as_ref()) {
|
if self.request_threshold_exceeded(current_tokens, effective_context.as_ref()) {
|
||||||
self.commit_system_items(&system_items);
|
if let Err(error) = self.commit_system_items(&system_items) {
|
||||||
|
return PreRequestAction::Cancel(format!("session persistence failed: {error}"));
|
||||||
|
}
|
||||||
return if appended_items.is_empty() {
|
return if appended_items.is_empty() {
|
||||||
PreRequestAction::Yield
|
PreRequestAction::Yield
|
||||||
} else {
|
} else {
|
||||||
@@ -290,8 +298,10 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
if system_items.is_empty() {
|
if system_items.is_empty() {
|
||||||
return PreRequestAction::Continue;
|
return PreRequestAction::Continue;
|
||||||
}
|
}
|
||||||
self.commit_system_items(&system_items);
|
match self.commit_system_items(&system_items) {
|
||||||
PreRequestAction::ContinueWith(appended_items)
|
Ok(()) => PreRequestAction::ContinueWith(appended_items),
|
||||||
|
Err(error) => PreRequestAction::Cancel(format!("session persistence failed: {error}")),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pre_tool_call(&self, info: &mut ToolCallInfo) -> PreToolAction {
|
async fn pre_tool_call(&self, info: &mut ToolCallInfo) -> PreToolAction {
|
||||||
@@ -449,13 +459,17 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SystemItemCommitter for RecordingSystemItemCommitter {
|
impl SystemItemCommitter for RecordingSystemItemCommitter {
|
||||||
fn commit_log_entry(&self, entry: session_store::LogEntry) {
|
fn commit_log_entry(
|
||||||
|
&self,
|
||||||
|
entry: session_store::LogEntry,
|
||||||
|
) -> Result<(), session_store::StoreError> {
|
||||||
if let session_store::LogEntry::SystemItem { item, .. } = entry {
|
if let session_store::LogEntry::SystemItem { item, .. } = entry {
|
||||||
self.committed
|
self.committed
|
||||||
.lock()
|
.lock()
|
||||||
.expect("committed system-item list poisoned")
|
.expect("committed system-item list poisoned")
|
||||||
.push(item);
|
.push(item);
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1019,8 +1033,8 @@ mod tests {
|
|||||||
async fn pending_history_appends_drains_buffer_into_items() {
|
async fn pending_history_appends_drains_buffer_into_items() {
|
||||||
let registry = Arc::new(HookRegistryBuilder::new().build());
|
let registry = Arc::new(HookRegistryBuilder::new().build());
|
||||||
let buffer = NotifyBuffer::new();
|
let buffer = NotifyBuffer::new();
|
||||||
buffer.push_notify("first".into());
|
buffer.push_notify("first".into(), false);
|
||||||
buffer.push_notify("second".into());
|
buffer.push_notify("second".into(), false);
|
||||||
|
|
||||||
let interceptor = WorkerInterceptor::new(
|
let interceptor = WorkerInterceptor::new(
|
||||||
registry,
|
registry,
|
||||||
@@ -1032,7 +1046,7 @@ mod tests {
|
|||||||
None,
|
None,
|
||||||
);
|
);
|
||||||
|
|
||||||
let items = interceptor.pending_history_appends().await;
|
let items = interceptor.pending_history_appends().await.unwrap();
|
||||||
assert_eq!(items.len(), 2);
|
assert_eq!(items.len(), 2);
|
||||||
let first = items[0].as_text().unwrap_or_default();
|
let first = items[0].as_text().unwrap_or_default();
|
||||||
let second = items[1].as_text().unwrap_or_default();
|
let second = items[1].as_text().unwrap_or_default();
|
||||||
@@ -1046,7 +1060,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Empty buffer → empty Vec (no synthesised items).
|
// Empty buffer → empty Vec (no synthesised items).
|
||||||
let again = interceptor.pending_history_appends().await;
|
let again = interceptor.pending_history_appends().await.unwrap();
|
||||||
assert!(again.is_empty());
|
assert!(again.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1057,7 +1071,7 @@ mod tests {
|
|||||||
// anything itself.
|
// anything itself.
|
||||||
let registry = Arc::new(HookRegistryBuilder::new().build());
|
let registry = Arc::new(HookRegistryBuilder::new().build());
|
||||||
let buffer = NotifyBuffer::new();
|
let buffer = NotifyBuffer::new();
|
||||||
buffer.push_notify("msg".into());
|
buffer.push_notify("msg".into(), false);
|
||||||
|
|
||||||
let interceptor = WorkerInterceptor::new(
|
let interceptor = WorkerInterceptor::new(
|
||||||
registry,
|
registry,
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ const CAPACITY: usize = 128;
|
|||||||
/// is available.
|
/// is available.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub enum PendingNotify {
|
pub enum PendingNotify {
|
||||||
Notify { message: String },
|
Notify { message: String, auto_run: bool },
|
||||||
WorkerEvent { event: WorkerEvent },
|
WorkerEvent { event: WorkerEvent },
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,8 +61,8 @@ impl NotifyBuffer {
|
|||||||
/// Push a notify entry onto the queue. If the queue is full, the
|
/// Push a notify entry onto the queue. If the queue is full, the
|
||||||
/// oldest entry is dropped and a `tracing::warn` is emitted — the
|
/// oldest entry is dropped and a `tracing::warn` is emitted — the
|
||||||
/// caller should never hit this in normal operation.
|
/// caller should never hit this in normal operation.
|
||||||
pub fn push_notify(&self, message: String) {
|
pub fn push_notify(&self, message: String, auto_run: bool) {
|
||||||
self.push_entry(PendingNotify::Notify { message });
|
self.push_entry(PendingNotify::Notify { message, auto_run });
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Push a typed worker-event entry onto the queue.
|
/// Push a typed worker-event entry onto the queue.
|
||||||
@@ -89,6 +89,15 @@ impl NotifyBuffer {
|
|||||||
q.drain(..).collect()
|
q.drain(..).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether an undrained `Method::Notify { auto_run: true }` remains.
|
||||||
|
pub fn has_auto_run_pending(&self) -> bool {
|
||||||
|
self.inner
|
||||||
|
.lock()
|
||||||
|
.expect("notify buffer poisoned")
|
||||||
|
.iter()
|
||||||
|
.any(|entry| matches!(entry, PendingNotify::Notify { auto_run: true, .. }))
|
||||||
|
}
|
||||||
|
|
||||||
/// Number of pending entries. Primarily for tests.
|
/// Number of pending entries. Primarily for tests.
|
||||||
pub fn len(&self) -> usize {
|
pub fn len(&self) -> usize {
|
||||||
self.inner.lock().expect("notify buffer poisoned").len()
|
self.inner.lock().expect("notify buffer poisoned").len()
|
||||||
@@ -107,7 +116,7 @@ pub(crate) fn build_system_item(
|
|||||||
prompts: &PromptCatalog,
|
prompts: &PromptCatalog,
|
||||||
) -> Result<SystemItem, CatalogError> {
|
) -> Result<SystemItem, CatalogError> {
|
||||||
match entry {
|
match entry {
|
||||||
PendingNotify::Notify { message } => {
|
PendingNotify::Notify { message, .. } => {
|
||||||
let body = prompts.notify_wrapper(message)?;
|
let body = prompts.notify_wrapper(message)?;
|
||||||
Ok(SystemItem::Notification {
|
Ok(SystemItem::Notification {
|
||||||
message: message.clone(),
|
message: message.clone(),
|
||||||
@@ -132,12 +141,15 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn push_then_drain_preserves_order() {
|
fn push_then_drain_preserves_order() {
|
||||||
let buf = NotifyBuffer::new();
|
let buf = NotifyBuffer::new();
|
||||||
buf.push_notify("one".into());
|
buf.push_notify("one".into(), false);
|
||||||
buf.push_notify("two".into());
|
assert!(!buf.has_auto_run_pending());
|
||||||
|
buf.push_notify("two".into(), true);
|
||||||
|
assert!(buf.has_auto_run_pending());
|
||||||
let drained = buf.drain();
|
let drained = buf.drain();
|
||||||
|
assert!(!buf.has_auto_run_pending());
|
||||||
assert_eq!(drained.len(), 2);
|
assert_eq!(drained.len(), 2);
|
||||||
match &drained[0] {
|
match &drained[0] {
|
||||||
PendingNotify::Notify { message } => assert_eq!(message, "one"),
|
PendingNotify::Notify { message, .. } => assert_eq!(message, "one"),
|
||||||
other => panic!("unexpected: {other:?}"),
|
other => panic!("unexpected: {other:?}"),
|
||||||
}
|
}
|
||||||
assert!(buf.is_empty());
|
assert!(buf.is_empty());
|
||||||
@@ -147,12 +159,12 @@ mod tests {
|
|||||||
fn capacity_drops_oldest() {
|
fn capacity_drops_oldest() {
|
||||||
let buf = NotifyBuffer::new();
|
let buf = NotifyBuffer::new();
|
||||||
for i in 0..(CAPACITY + 5) {
|
for i in 0..(CAPACITY + 5) {
|
||||||
buf.push_notify(format!("msg{i}"));
|
buf.push_notify(format!("msg{i}"), false);
|
||||||
}
|
}
|
||||||
let drained = buf.drain();
|
let drained = buf.drain();
|
||||||
assert_eq!(drained.len(), CAPACITY);
|
assert_eq!(drained.len(), CAPACITY);
|
||||||
match &drained[0] {
|
match &drained[0] {
|
||||||
PendingNotify::Notify { message } => assert_eq!(message, "msg5"),
|
PendingNotify::Notify { message, .. } => assert_eq!(message, "msg5"),
|
||||||
other => panic!("unexpected: {other:?}"),
|
other => panic!("unexpected: {other:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -161,6 +173,7 @@ mod tests {
|
|||||||
fn build_system_item_for_notify_carries_wrapper_body() {
|
fn build_system_item_for_notify_carries_wrapper_body() {
|
||||||
let entry = PendingNotify::Notify {
|
let entry = PendingNotify::Notify {
|
||||||
message: "hello".into(),
|
message: "hello".into(),
|
||||||
|
auto_run: false,
|
||||||
};
|
};
|
||||||
let catalog = PromptCatalog::builtins_only().unwrap();
|
let catalog = PromptCatalog::builtins_only().unwrap();
|
||||||
let item = build_system_item(&entry, &catalog).unwrap();
|
let item = build_system_item(&entry, &catalog).unwrap();
|
||||||
|
|||||||
@@ -88,9 +88,9 @@ pub enum WorkerPrompt {
|
|||||||
WorkerOrchestrationGuidanceSection,
|
WorkerOrchestrationGuidanceSection,
|
||||||
/// Weak Companion Notify payload for explicit Orchestrator Ticket events.
|
/// Weak Companion Notify payload for explicit Orchestrator Ticket events.
|
||||||
TicketEventCompanionNotice,
|
TicketEventCompanionNotice,
|
||||||
/// LLM-facing description for the SpawnWorker tool, including discovered
|
/// LLM-facing description for the SubWorkerSpawn tool, including discovered
|
||||||
/// profile selectors.
|
/// profile selectors.
|
||||||
SpawnWorkerToolDescription,
|
SubWorkerSpawnToolDescription,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerPrompt {
|
impl WorkerPrompt {
|
||||||
@@ -107,7 +107,7 @@ impl WorkerPrompt {
|
|||||||
Self::ResidentMemorySummarySection => "resident_memory_summary_section",
|
Self::ResidentMemorySummarySection => "resident_memory_summary_section",
|
||||||
Self::WorkerOrchestrationGuidanceSection => "worker_orchestration_guidance_section",
|
Self::WorkerOrchestrationGuidanceSection => "worker_orchestration_guidance_section",
|
||||||
Self::TicketEventCompanionNotice => "ticket_event_companion_notice",
|
Self::TicketEventCompanionNotice => "ticket_event_companion_notice",
|
||||||
Self::SpawnWorkerToolDescription => "spawn_worker_tool_description",
|
Self::SubWorkerSpawnToolDescription => "sub_worker_spawn_tool_description",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -126,7 +126,7 @@ impl WorkerPrompt {
|
|||||||
WorkerPrompt::ResidentMemorySummarySection,
|
WorkerPrompt::ResidentMemorySummarySection,
|
||||||
WorkerPrompt::WorkerOrchestrationGuidanceSection,
|
WorkerPrompt::WorkerOrchestrationGuidanceSection,
|
||||||
WorkerPrompt::TicketEventCompanionNotice,
|
WorkerPrompt::TicketEventCompanionNotice,
|
||||||
WorkerPrompt::SpawnWorkerToolDescription,
|
WorkerPrompt::SubWorkerSpawnToolDescription,
|
||||||
];
|
];
|
||||||
|
|
||||||
pub const KEYS: &'static [&'static str] = &[
|
pub const KEYS: &'static [&'static str] = &[
|
||||||
@@ -141,7 +141,7 @@ impl WorkerPrompt {
|
|||||||
"resident_memory_summary_section",
|
"resident_memory_summary_section",
|
||||||
"worker_orchestration_guidance_section",
|
"worker_orchestration_guidance_section",
|
||||||
"ticket_event_companion_notice",
|
"ticket_event_companion_notice",
|
||||||
"spawn_worker_tool_description",
|
"sub_worker_spawn_tool_description",
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,6 +251,7 @@ struct PackFile {
|
|||||||
/// `$yoi` / `$user` / `$workspace`.
|
/// `$yoi` / `$user` / `$workspace`.
|
||||||
pub struct PromptCatalog {
|
pub struct PromptCatalog {
|
||||||
env: Environment<'static>,
|
env: Environment<'static>,
|
||||||
|
loader: PromptLoader,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for PromptCatalog {
|
impl std::fmt::Debug for PromptCatalog {
|
||||||
@@ -260,6 +261,10 @@ impl std::fmt::Debug for PromptCatalog {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl PromptCatalog {
|
impl PromptCatalog {
|
||||||
|
pub(crate) fn loader(&self) -> PromptLoader {
|
||||||
|
self.loader.clone()
|
||||||
|
}
|
||||||
|
|
||||||
/// Builtin-only catalog. All `{% include %}` references must resolve
|
/// Builtin-only catalog. All `{% include %}` references must resolve
|
||||||
/// through `$yoi` (user/workspace prefixes are unavailable).
|
/// through `$yoi` (user/workspace prefixes are unavailable).
|
||||||
pub fn builtins_only() -> Result<Arc<Self>, CatalogError> {
|
pub fn builtins_only() -> Result<Arc<Self>, CatalogError> {
|
||||||
@@ -384,8 +389,8 @@ impl PromptCatalog {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Render `WorkerPrompt::SpawnWorkerToolDescription`.
|
/// Render `WorkerPrompt::SubWorkerSpawnToolDescription`.
|
||||||
pub fn spawn_worker_tool_description(
|
pub fn sub_worker_spawn_tool_description(
|
||||||
&self,
|
&self,
|
||||||
available_profiles: &str,
|
available_profiles: &str,
|
||||||
default_profile: &str,
|
default_profile: &str,
|
||||||
@@ -396,7 +401,7 @@ impl PromptCatalog {
|
|||||||
m.insert("available_profiles", Value::from(available_profiles));
|
m.insert("available_profiles", Value::from(available_profiles));
|
||||||
m.insert("default_profile", Value::from(default_profile));
|
m.insert("default_profile", Value::from(default_profile));
|
||||||
m.insert("profile_diagnostic", Value::from(profile_diagnostic));
|
m.insert("profile_diagnostic", Value::from(profile_diagnostic));
|
||||||
self.render(WorkerPrompt::SpawnWorkerToolDescription, Value::from(m))
|
self.render(WorkerPrompt::SubWorkerSpawnToolDescription, Value::from(m))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -483,7 +488,7 @@ fn build_catalog(
|
|||||||
})?;
|
})?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(PromptCatalog { env })
|
Ok(PromptCatalog { env, loader })
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -722,8 +727,8 @@ compact_system = "PREFIX\n{% include \"$yoi/internal/compact_system\" %}"
|
|||||||
fn worker_orchestration_guidance_section_renders_resource_body() {
|
fn worker_orchestration_guidance_section_renders_resource_body() {
|
||||||
let cat = PromptCatalog::builtins_only().unwrap();
|
let cat = PromptCatalog::builtins_only().unwrap();
|
||||||
let rendered = cat.worker_orchestration_guidance_section().unwrap();
|
let rendered = cat.worker_orchestration_guidance_section().unwrap();
|
||||||
assert!(rendered.contains("## Worker orchestration"));
|
assert!(rendered.contains("## SubWorker orchestration"));
|
||||||
assert!(rendered.contains("spawned Worker notifications are background signals"));
|
assert!(rendered.contains("SubWorker notifications are background signals"));
|
||||||
assert!(rendered.contains("does not need to keep a turn open"));
|
assert!(rendered.contains("does not need to keep a turn open"));
|
||||||
assert!(rendered.contains("Do not use `sleep` or polling loops"));
|
assert!(rendered.contains("Do not use `sleep` or polling loops"));
|
||||||
assert!(rendered.contains("worktree state, diff, and test results"));
|
assert!(rendered.contains("worktree state, diff, and test results"));
|
||||||
@@ -732,10 +737,10 @@ compact_system = "PREFIX\n{% include \"$yoi/internal/compact_system\" %}"
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn spawn_worker_tool_description_renders_profile_block() {
|
fn sub_worker_spawn_tool_description_renders_profile_block() {
|
||||||
let cat = PromptCatalog::builtins_only().unwrap();
|
let cat = PromptCatalog::builtins_only().unwrap();
|
||||||
let rendered = cat
|
let rendered = cat
|
||||||
.spawn_worker_tool_description(
|
.sub_worker_spawn_tool_description(
|
||||||
"- `project:coder` — Coder\n- `project:reviewer` — Reviewer",
|
"- `project:coder` — Coder\n- `project:reviewer` — Reviewer",
|
||||||
"project:coder",
|
"project:coder",
|
||||||
"",
|
"",
|
||||||
|
|||||||
@@ -206,12 +206,12 @@ struct ToolCapabilities {
|
|||||||
memory_query: bool,
|
memory_query: bool,
|
||||||
memory_read_document: bool,
|
memory_read_document: bool,
|
||||||
memory_update_document: bool,
|
memory_update_document: bool,
|
||||||
worker_spawn: bool,
|
sub_worker_spawn: bool,
|
||||||
worker_send: bool,
|
sub_worker_send: bool,
|
||||||
worker_read_output: bool,
|
sub_worker_read_output: bool,
|
||||||
worker_stop: bool,
|
sub_worker_stop: bool,
|
||||||
worker_list: bool,
|
sub_worker_list: bool,
|
||||||
worker_restore: bool,
|
sub_worker_restore: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ToolCapabilities {
|
impl ToolCapabilities {
|
||||||
@@ -222,12 +222,11 @@ impl ToolCapabilities {
|
|||||||
"MemoryQuery" => capabilities.memory_query = true,
|
"MemoryQuery" => capabilities.memory_query = true,
|
||||||
"MemoryReadDocument" => capabilities.memory_read_document = true,
|
"MemoryReadDocument" => capabilities.memory_read_document = true,
|
||||||
"MemoryUpdateDocument" => capabilities.memory_update_document = true,
|
"MemoryUpdateDocument" => capabilities.memory_update_document = true,
|
||||||
"SpawnWorker" => capabilities.worker_spawn = true,
|
"SubWorkerSpawn" => capabilities.sub_worker_spawn = true,
|
||||||
"SendToWorker" => capabilities.worker_send = true,
|
"SubWorkerSend" => capabilities.sub_worker_send = true,
|
||||||
"ReadWorkerOutput" => capabilities.worker_read_output = true,
|
"SubWorkerReadOutput" => capabilities.sub_worker_read_output = true,
|
||||||
"StopWorker" => capabilities.worker_stop = true,
|
"SubWorkerStop" => capabilities.sub_worker_stop = true,
|
||||||
"ListWorkers" => capabilities.worker_list = true,
|
"SubWorkerList" => capabilities.sub_worker_list = true,
|
||||||
"RestoreWorker" => capabilities.worker_restore = true,
|
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -246,13 +245,13 @@ impl ToolCapabilities {
|
|||||||
self.memory_update_document
|
self.memory_update_document
|
||||||
}
|
}
|
||||||
|
|
||||||
fn worker_management(self) -> bool {
|
fn sub_worker_management(self) -> bool {
|
||||||
self.worker_spawn
|
self.sub_worker_spawn
|
||||||
|| self.worker_send
|
|| self.sub_worker_send
|
||||||
|| self.worker_read_output
|
|| self.sub_worker_read_output
|
||||||
|| self.worker_stop
|
|| self.sub_worker_stop
|
||||||
|| self.worker_list
|
|| self.sub_worker_list
|
||||||
|| self.worker_restore
|
|| self.sub_worker_restore
|
||||||
}
|
}
|
||||||
|
|
||||||
fn to_minijinja_value(self) -> Value {
|
fn to_minijinja_value(self) -> Value {
|
||||||
@@ -269,7 +268,10 @@ impl ToolCapabilities {
|
|||||||
Value::from(self.memory_update_document),
|
Value::from(self.memory_update_document),
|
||||||
);
|
);
|
||||||
map.insert("memory_mutation", Value::from(self.memory_mutation()));
|
map.insert("memory_mutation", Value::from(self.memory_mutation()));
|
||||||
map.insert("worker_management", Value::from(self.worker_management()));
|
map.insert(
|
||||||
|
"sub_worker_management",
|
||||||
|
Value::from(self.sub_worker_management()),
|
||||||
|
);
|
||||||
Value::from(map)
|
Value::from(map)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -419,7 +421,7 @@ mod tests {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn worker_orchestration_instruction() -> FeatureInstructionDeclaration {
|
fn sub_worker_orchestration_instruction() -> FeatureInstructionDeclaration {
|
||||||
FeatureInstructionDeclaration::new(
|
FeatureInstructionDeclaration::new(
|
||||||
crate::feature::FeatureInstructionId::builtin("worker.orchestration"),
|
crate::feature::FeatureInstructionId::builtin("worker.orchestration"),
|
||||||
"$yoi/common/worker-orchestration",
|
"$yoi/common/worker-orchestration",
|
||||||
@@ -595,13 +597,13 @@ mod tests {
|
|||||||
let tmpl = SystemPromptTemplate::parse("$yoi/default", loader).unwrap();
|
let tmpl = SystemPromptTemplate::parse("$yoi/default", loader).unwrap();
|
||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
let scope = build_scope(dir.path());
|
let scope = build_scope(dir.path());
|
||||||
let instructions = [worker_orchestration_instruction()];
|
let instructions = [sub_worker_orchestration_instruction()];
|
||||||
let mut ctx = ctx(dir.path(), &scope, vec!["Read".into()], None);
|
let mut ctx = ctx(dir.path(), &scope, vec!["Read".into()], None);
|
||||||
ctx.feature_instructions = &instructions;
|
ctx.feature_instructions = &instructions;
|
||||||
let rendered = tmpl.render(&ctx).unwrap();
|
let rendered = tmpl.render(&ctx).unwrap();
|
||||||
|
|
||||||
assert!(rendered.contains("## Worker orchestration"));
|
assert!(rendered.contains("## SubWorker orchestration"));
|
||||||
assert!(rendered.contains("spawned Worker notifications are background signals"));
|
assert!(rendered.contains("SubWorker notifications are background signals"));
|
||||||
assert!(rendered.contains("does not need to keep a turn open"));
|
assert!(rendered.contains("does not need to keep a turn open"));
|
||||||
assert!(rendered.contains("Do not use `sleep` or polling loops"));
|
assert!(rendered.contains("Do not use `sleep` or polling loops"));
|
||||||
assert!(rendered.contains("worktree state, diff, and test results"));
|
assert!(rendered.contains("worktree state, diff, and test results"));
|
||||||
@@ -610,7 +612,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn worker_orchestration_guidance_is_omitted_without_worker_management_tools() {
|
fn worker_orchestration_guidance_is_omitted_without_sub_worker_management_tools() {
|
||||||
let loader = PromptLoader::builtins_only();
|
let loader = PromptLoader::builtins_only();
|
||||||
let tmpl = SystemPromptTemplate::parse("$yoi/default", loader).unwrap();
|
let tmpl = SystemPromptTemplate::parse("$yoi/default", loader).unwrap();
|
||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
|
|||||||
@@ -1,13 +1,9 @@
|
|||||||
//! Worker-to-Worker communication tools.
|
//! Parent-facing tools for in-process Internal SubWorker sessions.
|
||||||
//!
|
//!
|
||||||
//! Three tools in one module: `SendToWorker`, `ReadWorkerOutput`, `StopWorker`,
|
//! All five tools share the same parent-owned `SpawnedWorkerRegistry` of typed session handles.
|
||||||
//! all built on the same `SpawnedWorkerRegistry` handed in by
|
//! There is no Runtime catalog lookup or child socket transport, so a Worker can operate only on
|
||||||
//! the controller. Each operation is request-response: connect to the
|
//! its direct Internal children. The socket helper at the bottom remains solely for the legacy
|
||||||
//! target's Unix socket, perform one method exchange, disconnect.
|
//! top-level Worker callback protocol and is not part of SubWorker communication.
|
||||||
//!
|
|
||||||
//! These tools only touch Workers listed in the spawner's
|
|
||||||
//! `SpawnedWorkerRegistry`; there is no machine-wide directory lookup, so
|
|
||||||
//! the spawner can only reach its own descendants.
|
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -17,12 +13,11 @@ use async_trait::async_trait;
|
|||||||
use llm_engine::llm_client::types::{ContentPart, Item, Role};
|
use llm_engine::llm_client::types::{ContentPart, Item, Role};
|
||||||
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
|
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
|
||||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||||
use protocol::{ErrorCode, Event, InvokeKind, Method};
|
use protocol::{Event, Method};
|
||||||
use serde::Deserialize;
|
use serde::{Deserialize, Serialize};
|
||||||
use session_store::LogEntry;
|
use session_store::LogEntry;
|
||||||
use tokio::net::UnixStream;
|
use tokio::net::UnixStream;
|
||||||
|
|
||||||
use crate::runtime::dir::SpawnedWorkerRecord;
|
|
||||||
use crate::spawn::registry::SpawnedWorkerRegistry;
|
use crate::spawn::registry::SpawnedWorkerRegistry;
|
||||||
|
|
||||||
/// Timeout applied to each socket-level operation — connect, write,
|
/// Timeout applied to each socket-level operation — connect, write,
|
||||||
@@ -35,77 +30,59 @@ const SOCKET_OP_TIMEOUT: Duration = Duration::from_secs(5);
|
|||||||
|
|
||||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||||
struct NameInput {
|
struct NameInput {
|
||||||
/// Name of a previously spawned Worker.
|
/// Name of a previously spawned SubWorker.
|
||||||
name: String,
|
name: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// SendToWorker
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
const SEND_TO_POD_DESCRIPTION: &str = "Send a text message to a previously spawned Worker. The spawned Worker \
|
|
||||||
processes it as a user turn. Fails if the Worker is already executing a \
|
|
||||||
turn — retry after it finishes. Does not wait for the turn to complete; \
|
|
||||||
use `ReadWorkerOutput` to fetch results afterwards.";
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||||
struct SendToWorkerInput {
|
#[serde(deny_unknown_fields)]
|
||||||
/// Target Worker name.
|
struct SubWorkerListInput {}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct SubWorkerListItem {
|
||||||
name: String,
|
name: String,
|
||||||
/// Text delivered to the Worker as the next user message.
|
|
||||||
message: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
struct SendToWorkerTool {
|
struct SubWorkerListTool {
|
||||||
registry: Arc<SpawnedWorkerRegistry>,
|
registry: Arc<SpawnedWorkerRegistry>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl Tool for SendToWorkerTool {
|
impl Tool for SubWorkerListTool {
|
||||||
async fn execute(
|
async fn execute(
|
||||||
&self,
|
&self,
|
||||||
input_json: &str,
|
input_json: &str,
|
||||||
_ctx: llm_engine::tool::ToolExecutionContext,
|
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let input: SendToWorkerInput = serde_json::from_str(input_json)
|
let _input: SubWorkerListInput = serde_json::from_str(input_json).map_err(|error| {
|
||||||
.map_err(|e| ToolError::InvalidArgument(format!("invalid SendToWorker input: {e}")))?;
|
ToolError::InvalidArgument(format!("invalid SubWorkerList input: {error}"))
|
||||||
let record = self
|
|
||||||
.registry
|
|
||||||
.get(&input.name)
|
|
||||||
.await
|
|
||||||
.ok_or_else(|| unknown_worker_err(&input.name))?;
|
|
||||||
|
|
||||||
send_run_and_confirm(&record.socket_path, input.message)
|
|
||||||
.await
|
|
||||||
.map_err(|e| match e {
|
|
||||||
SendRunError::AlreadyRunning => ToolError::ExecutionFailed(format!(
|
|
||||||
"worker `{}` is already running a turn; wait for it to finish and retry",
|
|
||||||
input.name
|
|
||||||
)),
|
|
||||||
SendRunError::Rejected { code, message } => ToolError::ExecutionFailed(format!(
|
|
||||||
"worker `{}` rejected the run with {code:?}: {message}",
|
|
||||||
input.name
|
|
||||||
)),
|
|
||||||
SendRunError::Io(msg) => {
|
|
||||||
ToolError::ExecutionFailed(format!("send to `{}`: {msg}", input.name))
|
|
||||||
}
|
|
||||||
})?;
|
})?;
|
||||||
|
let items = self
|
||||||
|
.registry
|
||||||
|
.list_internal()
|
||||||
|
.into_iter()
|
||||||
|
.map(|record| SubWorkerListItem {
|
||||||
|
name: record.worker_name,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let count = items.len();
|
||||||
|
let content = serde_json::to_string_pretty(&serde_json::json!({ "sub_workers": items }))
|
||||||
|
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("sent message to `{}`", input.name),
|
summary: format!("listed {count} child SubWorker(s)"),
|
||||||
content: None,
|
content: Some(content),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn send_to_worker_tool(registry: Arc<SpawnedWorkerRegistry>) -> ToolDefinition {
|
pub fn sub_worker_list_tool(registry: Arc<SpawnedWorkerRegistry>) -> ToolDefinition {
|
||||||
Arc::new(move || {
|
Arc::new(move || {
|
||||||
let schema = schemars::schema_for!(SendToWorkerInput);
|
let schema = schemars::schema_for!(SubWorkerListInput);
|
||||||
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||||
let meta = ToolMeta::new("SendToWorker")
|
let meta = ToolMeta::new("SubWorkerList")
|
||||||
.description(SEND_TO_POD_DESCRIPTION)
|
.description("List child SubWorkers owned by this Worker. Peer Workers and general Runtime Workers are excluded.")
|
||||||
.input_schema(schema_value);
|
.input_schema(schema_value);
|
||||||
let tool: Arc<dyn Tool> = Arc::new(SendToWorkerTool {
|
let tool: Arc<dyn Tool> = Arc::new(SubWorkerListTool {
|
||||||
registry: registry.clone(),
|
registry: registry.clone(),
|
||||||
});
|
});
|
||||||
(meta, tool)
|
(meta, tool)
|
||||||
@@ -113,79 +90,126 @@ pub fn send_to_worker_tool(registry: Arc<SpawnedWorkerRegistry>) -> ToolDefiniti
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// ReadWorkerOutput
|
// SubWorkerSend
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const READ_POD_OUTPUT_DESCRIPTION: &str = "Fetch new assistant text from a spawned Worker since the last read. \
|
const SEND_TO_POD_DESCRIPTION: &str = "Send a text message to a previously spawned SubWorker. The SubWorker \
|
||||||
Uses an internal cursor per-Worker so consecutive calls return only \
|
processes it as a user turn. Fails if the SubWorker is already executing a \
|
||||||
newly-produced output. Returns the Worker's current status and the new \
|
turn — retry after it finishes. Does not wait for the turn to complete; \
|
||||||
text, or reports `stopped` if the Worker can no longer be reached.";
|
use `SubWorkerReadOutput` to fetch results afterwards.";
|
||||||
|
|
||||||
struct ReadWorkerOutputTool {
|
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||||
|
struct SubWorkerSendInput {
|
||||||
|
/// Target SubWorker name.
|
||||||
|
name: String,
|
||||||
|
/// Text delivered to the SubWorker as the next user message.
|
||||||
|
message: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct SubWorkerSendTool {
|
||||||
registry: Arc<SpawnedWorkerRegistry>,
|
registry: Arc<SpawnedWorkerRegistry>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl Tool for ReadWorkerOutputTool {
|
impl Tool for SubWorkerSendTool {
|
||||||
|
async fn execute(
|
||||||
|
&self,
|
||||||
|
input_json: &str,
|
||||||
|
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||||
|
) -> Result<ToolOutput, ToolError> {
|
||||||
|
let input: SubWorkerSendInput = serde_json::from_str(input_json)
|
||||||
|
.map_err(|e| ToolError::InvalidArgument(format!("invalid SubWorkerSend input: {e}")))?;
|
||||||
|
if let Some(record) = self.registry.get_internal(&input.name) {
|
||||||
|
record.session.send(input.message).await.map_err(|error| {
|
||||||
|
ToolError::ExecutionFailed(format!("send to `{}`: {error}", input.name))
|
||||||
|
})?;
|
||||||
|
return Ok(ToolOutput {
|
||||||
|
summary: format!("sent message to `{}`", input.name),
|
||||||
|
content: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(unknown_worker_err(&input.name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sub_worker_send_tool(registry: Arc<SpawnedWorkerRegistry>) -> ToolDefinition {
|
||||||
|
Arc::new(move || {
|
||||||
|
let schema = schemars::schema_for!(SubWorkerSendInput);
|
||||||
|
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||||
|
let meta = ToolMeta::new("SubWorkerSend")
|
||||||
|
.description(SEND_TO_POD_DESCRIPTION)
|
||||||
|
.input_schema(schema_value);
|
||||||
|
let tool: Arc<dyn Tool> = Arc::new(SubWorkerSendTool {
|
||||||
|
registry: registry.clone(),
|
||||||
|
});
|
||||||
|
(meta, tool)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// SubWorkerReadOutput
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const READ_POD_OUTPUT_DESCRIPTION: &str = "Fetch new assistant text from a SubWorker since the last read. \
|
||||||
|
Uses an internal cursor per-SubWorker so consecutive calls return only \
|
||||||
|
newly-produced output. Returns the SubWorker's current status and the new \
|
||||||
|
text, or reports `stopped` if the SubWorker can no longer be reached.";
|
||||||
|
|
||||||
|
struct SubWorkerReadOutputTool {
|
||||||
|
registry: Arc<SpawnedWorkerRegistry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Tool for SubWorkerReadOutputTool {
|
||||||
async fn execute(
|
async fn execute(
|
||||||
&self,
|
&self,
|
||||||
input_json: &str,
|
input_json: &str,
|
||||||
_ctx: llm_engine::tool::ToolExecutionContext,
|
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let input: NameInput = serde_json::from_str(input_json).map_err(|e| {
|
let input: NameInput = serde_json::from_str(input_json).map_err(|e| {
|
||||||
ToolError::InvalidArgument(format!("invalid ReadWorkerOutput input: {e}"))
|
ToolError::InvalidArgument(format!("invalid SubWorkerReadOutput input: {e}"))
|
||||||
})?;
|
})?;
|
||||||
let record = self
|
if let Some(record) = self.registry.get_internal(&input.name) {
|
||||||
.registry
|
let entries = record.session.entries();
|
||||||
.get(&input.name)
|
|
||||||
.await
|
|
||||||
.ok_or_else(|| unknown_worker_err(&input.name))?;
|
|
||||||
|
|
||||||
let items = match fetch_history(&record.socket_path).await {
|
|
||||||
Ok(items) => items,
|
|
||||||
Err(_) => {
|
|
||||||
return Ok(ToolOutput {
|
|
||||||
summary: format!("worker `{}` is stopped (unreachable)", input.name),
|
|
||||||
content: None,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let cursor = self.registry.cursor(&input.name).await;
|
let cursor = self.registry.cursor(&input.name).await;
|
||||||
let new_items = if cursor >= items.len() {
|
let new_entries = if cursor >= entries.len() {
|
||||||
&[] as &[serde_json::Value]
|
&[] as &[LogEntry]
|
||||||
} else {
|
} else {
|
||||||
&items[cursor..]
|
&entries[cursor..]
|
||||||
};
|
};
|
||||||
let new_text = extract_assistant_text(new_items);
|
let values = new_entries
|
||||||
self.registry.set_cursor(&input.name, items.len()).await;
|
.iter()
|
||||||
|
.filter_map(|entry| serde_json::to_value(entry).ok())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let new_text = extract_assistant_text(&values);
|
||||||
|
self.registry.set_cursor(&input.name, entries.len()).await;
|
||||||
|
let status = format!("{:?}", record.session.status()).to_lowercase();
|
||||||
let summary = if new_text.is_empty() {
|
let summary = if new_text.is_empty() {
|
||||||
format!("worker `{}` running; no new assistant text", input.name)
|
format!("worker `{}` {status}; no new assistant text", input.name)
|
||||||
} else {
|
} else {
|
||||||
let lines = new_text.lines().count();
|
|
||||||
format!(
|
format!(
|
||||||
"worker `{}`: {lines} new line(s) of assistant text",
|
"worker `{}` {status}: {} new line(s) of assistant text",
|
||||||
input.name
|
input.name,
|
||||||
|
new_text.lines().count()
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
let content = if new_text.is_empty() {
|
return Ok(ToolOutput {
|
||||||
None
|
summary,
|
||||||
} else {
|
content: (!new_text.is_empty()).then_some(new_text),
|
||||||
Some(new_text)
|
});
|
||||||
};
|
}
|
||||||
Ok(ToolOutput { summary, content })
|
Err(unknown_worker_err(&input.name))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn read_worker_output_tool(registry: Arc<SpawnedWorkerRegistry>) -> ToolDefinition {
|
pub fn sub_worker_read_output_tool(registry: Arc<SpawnedWorkerRegistry>) -> ToolDefinition {
|
||||||
Arc::new(move || {
|
Arc::new(move || {
|
||||||
let schema = schemars::schema_for!(NameInput);
|
let schema = schemars::schema_for!(NameInput);
|
||||||
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||||
let meta = ToolMeta::new("ReadWorkerOutput")
|
let meta = ToolMeta::new("SubWorkerReadOutput")
|
||||||
.description(READ_POD_OUTPUT_DESCRIPTION)
|
.description(READ_POD_OUTPUT_DESCRIPTION)
|
||||||
.input_schema(schema_value);
|
.input_schema(schema_value);
|
||||||
let tool: Arc<dyn Tool> = Arc::new(ReadWorkerOutputTool {
|
let tool: Arc<dyn Tool> = Arc::new(SubWorkerReadOutputTool {
|
||||||
registry: registry.clone(),
|
registry: registry.clone(),
|
||||||
});
|
});
|
||||||
(meta, tool)
|
(meta, tool)
|
||||||
@@ -193,65 +217,52 @@ pub fn read_worker_output_tool(registry: Arc<SpawnedWorkerRegistry>) -> ToolDefi
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// StopWorker
|
// SubWorkerStop
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const STOP_POD_DESCRIPTION: &str = "Terminate a spawned Worker and reclaim the delegated scope. The Worker \
|
const STOP_POD_DESCRIPTION: &str = "Cancel and stop a spawned Internal SubWorker session, remove it from the parent's direct-child registry, and reclaim delegated Write scope.";
|
||||||
receives `Shutdown`; its scope entry is released in the machine-wide \
|
|
||||||
registry so the spawner can spawn a new Worker over the same paths.";
|
|
||||||
|
|
||||||
struct StopWorkerTool {
|
struct SubWorkerStopTool {
|
||||||
registry: Arc<SpawnedWorkerRegistry>,
|
registry: Arc<SpawnedWorkerRegistry>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl Tool for StopWorkerTool {
|
impl Tool for SubWorkerStopTool {
|
||||||
async fn execute(
|
async fn execute(
|
||||||
&self,
|
&self,
|
||||||
input_json: &str,
|
input_json: &str,
|
||||||
_ctx: llm_engine::tool::ToolExecutionContext,
|
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let input: NameInput = serde_json::from_str(input_json)
|
let input: NameInput = serde_json::from_str(input_json)
|
||||||
.map_err(|e| ToolError::InvalidArgument(format!("invalid StopWorker input: {e}")))?;
|
.map_err(|e| ToolError::InvalidArgument(format!("invalid SubWorkerStop input: {e}")))?;
|
||||||
let record = self
|
if let Some(record) = self.registry.get_internal(&input.name) {
|
||||||
.registry
|
record.session.stop().await.map_err(|error| {
|
||||||
.get(&input.name)
|
ToolError::ExecutionFailed(format!("stop `{}`: {error}", input.name))
|
||||||
.await
|
|
||||||
.ok_or_else(|| unknown_worker_err(&input.name))?;
|
|
||||||
|
|
||||||
// Best-effort Shutdown. The child's own `ScopeAllocationGuard`
|
|
||||||
// releases its entry on clean exit; the parent reclaim below is the
|
|
||||||
// authoritative operation for removing the child record and returning
|
|
||||||
// delegated Write scope to the spawner.
|
|
||||||
let _ = connect_and_send(&record.socket_path, &Method::Shutdown).await;
|
|
||||||
|
|
||||||
let scope_summary = summarize_scope(&record);
|
|
||||||
|
|
||||||
self.registry
|
|
||||||
.remove(&record.worker_name)
|
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
ToolError::ExecutionFailed(format!("update spawned worker registry: {e}"))
|
|
||||||
})?;
|
})?;
|
||||||
|
self.registry
|
||||||
Ok(ToolOutput {
|
.remove_internal(&input.name)
|
||||||
|
.await
|
||||||
|
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||||
|
return Ok(ToolOutput {
|
||||||
summary: format!(
|
summary: format!(
|
||||||
"stopped worker `{}`; reclaimed scope: {scope_summary}",
|
"stopped worker `{}` and reclaimed delegated scope",
|
||||||
record.worker_name
|
input.name
|
||||||
),
|
),
|
||||||
content: None,
|
content: None,
|
||||||
})
|
});
|
||||||
|
}
|
||||||
|
Err(unknown_worker_err(&input.name))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn stop_worker_tool(registry: Arc<SpawnedWorkerRegistry>) -> ToolDefinition {
|
pub fn sub_worker_stop_tool(registry: Arc<SpawnedWorkerRegistry>) -> ToolDefinition {
|
||||||
Arc::new(move || {
|
Arc::new(move || {
|
||||||
let schema = schemars::schema_for!(NameInput);
|
let schema = schemars::schema_for!(NameInput);
|
||||||
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||||
let meta = ToolMeta::new("StopWorker")
|
let meta = ToolMeta::new("SubWorkerStop")
|
||||||
.description(STOP_POD_DESCRIPTION)
|
.description(STOP_POD_DESCRIPTION)
|
||||||
.input_schema(schema_value);
|
.input_schema(schema_value);
|
||||||
let tool: Arc<dyn Tool> = Arc::new(StopWorkerTool {
|
let tool: Arc<dyn Tool> = Arc::new(SubWorkerStopTool {
|
||||||
registry: registry.clone(),
|
registry: registry.clone(),
|
||||||
});
|
});
|
||||||
(meta, tool)
|
(meta, tool)
|
||||||
@@ -266,29 +277,6 @@ fn unknown_worker_err(name: &str) -> ToolError {
|
|||||||
ToolError::InvalidArgument(format!("no spawned worker named `{name}`"))
|
ToolError::InvalidArgument(format!("no spawned worker named `{name}`"))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn summarize_scope(record: &SpawnedWorkerRecord) -> String {
|
|
||||||
if record.scope_delegated.is_empty() {
|
|
||||||
return "(none)".into();
|
|
||||||
}
|
|
||||||
let parts: Vec<String> = record
|
|
||||||
.scope_delegated
|
|
||||||
.iter()
|
|
||||||
.map(|rule| {
|
|
||||||
let perm = match rule.permission {
|
|
||||||
manifest::Permission::Read => "read",
|
|
||||||
manifest::Permission::Write => "write",
|
|
||||||
};
|
|
||||||
let recursive = if rule.recursive {
|
|
||||||
""
|
|
||||||
} else {
|
|
||||||
" [non-recursive]"
|
|
||||||
};
|
|
||||||
format!("{perm}:{}{recursive}", rule.target.display())
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
parts.join(", ")
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Connect with a timeout, drain the server's connect-time snapshot,
|
/// Connect with a timeout, drain the server's connect-time snapshot,
|
||||||
/// write one `Method` line, flush, and close.
|
/// write one `Method` line, flush, and close.
|
||||||
///
|
///
|
||||||
@@ -335,125 +323,6 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Failure modes distinguished by `SendToWorker`.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub(crate) enum SendRunError {
|
|
||||||
/// Target Worker responded with `Error { AlreadyRunning }` — the
|
|
||||||
/// caller can retry once the current turn ends.
|
|
||||||
AlreadyRunning,
|
|
||||||
/// Target Worker explicitly rejected the run after delivery reached the
|
|
||||||
/// controller.
|
|
||||||
Rejected { code: ErrorCode, message: String },
|
|
||||||
/// Transport, protocol, timeout, or unexpected EOF before acceptance
|
|
||||||
/// evidence was observed.
|
|
||||||
Io(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Write `Method::Run` to the target and read back events until we see
|
|
||||||
/// evidence that the controller accepted the run (`UserMessage`,
|
|
||||||
/// `TurnStart`, or a user-send `InvokeStart`) or rejected it. The connect-time
|
|
||||||
/// event prelude is drained before sending the method so large Snapshots and
|
|
||||||
/// large Run payloads cannot block each other on the same socket. Times out
|
|
||||||
/// per operation so a stuck Worker doesn't hang the tool.
|
|
||||||
pub(crate) async fn send_run_and_confirm(socket: &Path, input: String) -> Result<(), SendRunError> {
|
|
||||||
let stream = tokio::time::timeout(SOCKET_OP_TIMEOUT, UnixStream::connect(socket))
|
|
||||||
.await
|
|
||||||
.map_err(|_| SendRunError::Io("connect timed out".into()))?
|
|
||||||
.map_err(|e| SendRunError::Io(format!("connect: {e}")))?;
|
|
||||||
let (r, w) = stream.into_split();
|
|
||||||
let mut writer = JsonLineWriter::new(w);
|
|
||||||
let mut reader = JsonLineReader::new(r);
|
|
||||||
|
|
||||||
loop {
|
|
||||||
let event = tokio::time::timeout(SOCKET_OP_TIMEOUT, reader.next::<Event>())
|
|
||||||
.await
|
|
||||||
.map_err(|_| SendRunError::Io("read initial Snapshot timed out".into()))?
|
|
||||||
.map_err(|e| SendRunError::Io(format!("read initial Snapshot: {e}")))?;
|
|
||||||
match event {
|
|
||||||
Some(Event::Snapshot { .. }) => break,
|
|
||||||
Some(Event::Alert(_)) => continue,
|
|
||||||
Some(Event::Error {
|
|
||||||
code: ErrorCode::AlreadyRunning,
|
|
||||||
..
|
|
||||||
}) => return Err(SendRunError::AlreadyRunning),
|
|
||||||
Some(Event::Error { code, message }) => {
|
|
||||||
return Err(SendRunError::Rejected { code, message });
|
|
||||||
}
|
|
||||||
Some(_) => continue,
|
|
||||||
None => {
|
|
||||||
return Err(SendRunError::Io(
|
|
||||||
"connection closed before initial Snapshot".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
tokio::time::timeout(
|
|
||||||
SOCKET_OP_TIMEOUT,
|
|
||||||
writer.write(&Method::Run {
|
|
||||||
input: vec![protocol::Segment::text(input)],
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(|_| SendRunError::Io("write timed out".into()))?
|
|
||||||
.map_err(|e| SendRunError::Io(format!("write: {e}")))?;
|
|
||||||
loop {
|
|
||||||
let event = tokio::time::timeout(SOCKET_OP_TIMEOUT, reader.next::<Event>())
|
|
||||||
.await
|
|
||||||
.map_err(|_| SendRunError::Io("read response timed out".into()))?
|
|
||||||
.map_err(|e| SendRunError::Io(format!("read response: {e}")))?;
|
|
||||||
match event {
|
|
||||||
Some(Event::Error {
|
|
||||||
code: ErrorCode::AlreadyRunning,
|
|
||||||
..
|
|
||||||
}) => return Err(SendRunError::AlreadyRunning),
|
|
||||||
Some(Event::Error { code, message }) => {
|
|
||||||
return Err(SendRunError::Rejected { code, message });
|
|
||||||
}
|
|
||||||
Some(Event::InvokeStart {
|
|
||||||
kind: InvokeKind::UserSend,
|
|
||||||
})
|
|
||||||
| Some(Event::UserMessage { .. })
|
|
||||||
| Some(Event::TurnStart { .. }) => return Ok(()),
|
|
||||||
// Other post-Snapshot events can race with the controller's
|
|
||||||
// response; keep reading until the Run is accepted or rejected.
|
|
||||||
Some(_) => continue,
|
|
||||||
None => return Err(SendRunError::Io("connection closed before response".into())),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Connect to a Worker's socket and read the connect-time `Event::Snapshot`.
|
|
||||||
///
|
|
||||||
/// Workers deliver the session-log mirror as the first non-Alert event on
|
|
||||||
/// every new connection, so consuming it is sufficient — no explicit
|
|
||||||
/// `GetHistory` method round trip. Returns the entries as raw JSON
|
|
||||||
/// values; callers deserialize as `session_store::LogEntry` if they
|
|
||||||
/// need typed access.
|
|
||||||
async fn fetch_history(socket: &Path) -> std::io::Result<Vec<serde_json::Value>> {
|
|
||||||
let stream = tokio::time::timeout(SOCKET_OP_TIMEOUT, UnixStream::connect(socket))
|
|
||||||
.await
|
|
||||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out"))??;
|
|
||||||
let (r, _w) = stream.into_split();
|
|
||||||
let mut reader = JsonLineReader::new(r);
|
|
||||||
|
|
||||||
loop {
|
|
||||||
let event = tokio::time::timeout(SOCKET_OP_TIMEOUT, reader.next::<Event>())
|
|
||||||
.await
|
|
||||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timed out"))??;
|
|
||||||
match event {
|
|
||||||
Some(Event::Snapshot { entries, .. }) => return Ok(entries),
|
|
||||||
Some(_) => continue,
|
|
||||||
None => {
|
|
||||||
return Err(std::io::Error::new(
|
|
||||||
std::io::ErrorKind::UnexpectedEof,
|
|
||||||
"worker closed connection before Snapshot event",
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn extract_assistant_text(entries: &[serde_json::Value]) -> String {
|
fn extract_assistant_text(entries: &[serde_json::Value]) -> String {
|
||||||
let mut out = String::new();
|
let mut out = String::new();
|
||||||
for value in entries {
|
for value in entries {
|
||||||
@@ -538,119 +407,6 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn serve_initial_events_then_run_ack(
|
|
||||||
listener: UnixListener,
|
|
||||||
initial_events: Vec<Event>,
|
|
||||||
ack: Event,
|
|
||||||
) -> JoinHandle<Option<Method>> {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let (stream, _) = listener.accept().await.ok()?;
|
|
||||||
let (r, w) = stream.into_split();
|
|
||||||
let mut reader = JsonLineReader::new(r);
|
|
||||||
let mut writer = JsonLineWriter::new(w);
|
|
||||||
for event in initial_events {
|
|
||||||
writer.write(&event).await.ok()?;
|
|
||||||
}
|
|
||||||
let method = reader.next::<Method>().await.ok().flatten()?;
|
|
||||||
writer.write(&ack).await.ok()?;
|
|
||||||
Some(method)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn send_run_and_confirm_keeps_connection_open_until_user_message_ack() {
|
|
||||||
let tmp = TempDir::new().unwrap();
|
|
||||||
let socket = tmp.path().join("worker.sock");
|
|
||||||
let listener = UnixListener::bind(&socket).unwrap();
|
|
||||||
let received = serve_initial_events_then_run_ack(
|
|
||||||
listener,
|
|
||||||
vec![
|
|
||||||
Event::Alert(Alert {
|
|
||||||
level: AlertLevel::Warn,
|
|
||||||
source: AlertSource::Worker,
|
|
||||||
message: "replayed alert".into(),
|
|
||||||
timestamp_ms: 0,
|
|
||||||
}),
|
|
||||||
snapshot(Vec::new()),
|
|
||||||
],
|
|
||||||
Event::UserMessage {
|
|
||||||
segments: vec![protocol::Segment::text("hello")],
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
send_run_and_confirm(&socket, "hello".into()).await.unwrap();
|
|
||||||
|
|
||||||
let method = received.await.unwrap().expect("expected method");
|
|
||||||
match method {
|
|
||||||
Method::Run { input } => {
|
|
||||||
assert_eq!(protocol::Segment::flatten_to_text(&input), "hello");
|
|
||||||
}
|
|
||||||
other => panic!("expected Run, got {other:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn send_run_and_confirm_drains_alert_and_large_snapshot_before_large_run() {
|
|
||||||
let tmp = TempDir::new().unwrap();
|
|
||||||
let socket = tmp.path().join("worker.sock");
|
|
||||||
let listener = UnixListener::bind(&socket).unwrap();
|
|
||||||
let large_snapshot_payload = "s".repeat(2 * 1024 * 1024);
|
|
||||||
let large_run_payload = "r".repeat(2 * 1024 * 1024);
|
|
||||||
let received = serve_initial_events_then_run_ack(
|
|
||||||
listener,
|
|
||||||
vec![
|
|
||||||
Event::Alert(Alert {
|
|
||||||
level: AlertLevel::Warn,
|
|
||||||
source: AlertSource::Worker,
|
|
||||||
message: "replayed alert".into(),
|
|
||||||
timestamp_ms: 0,
|
|
||||||
}),
|
|
||||||
snapshot(vec![
|
|
||||||
serde_json::json!({ "payload": large_snapshot_payload }),
|
|
||||||
]),
|
|
||||||
],
|
|
||||||
Event::InvokeStart {
|
|
||||||
kind: InvokeKind::UserSend,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
send_run_and_confirm(&socket, large_run_payload.clone())
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let method = received.await.unwrap().expect("expected method");
|
|
||||||
match method {
|
|
||||||
Method::Run { input } => {
|
|
||||||
assert_eq!(
|
|
||||||
protocol::Segment::flatten_to_text(&input),
|
|
||||||
large_run_payload
|
|
||||||
);
|
|
||||||
}
|
|
||||||
other => panic!("expected Run, got {other:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn send_run_and_confirm_reports_already_running() {
|
|
||||||
let tmp = TempDir::new().unwrap();
|
|
||||||
let socket = tmp.path().join("worker.sock");
|
|
||||||
let listener = UnixListener::bind(&socket).unwrap();
|
|
||||||
let received = serve_initial_events_then_run_ack(
|
|
||||||
listener,
|
|
||||||
vec![snapshot(Vec::new())],
|
|
||||||
Event::Error {
|
|
||||||
code: ErrorCode::AlreadyRunning,
|
|
||||||
message: "busy".into(),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
let err = send_run_and_confirm(&socket, "hello".into())
|
|
||||||
.await
|
|
||||||
.expect_err("expected AlreadyRunning");
|
|
||||||
assert!(matches!(err, SendRunError::AlreadyRunning));
|
|
||||||
assert!(matches!(received.await.unwrap(), Some(Method::Run { .. })));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn connect_and_send_drains_initial_alert_and_snapshot_before_method() {
|
async fn connect_and_send_drains_initial_alert_and_snapshot_before_method() {
|
||||||
let tmp = TempDir::new().unwrap();
|
let tmp = TempDir::new().unwrap();
|
||||||
|
|||||||
+160
-300
@@ -1,77 +1,66 @@
|
|||||||
//! Shared registry of Workers spawned by this Worker.
|
//! Parent-owned registry of direct Internal SubWorker sessions.
|
||||||
//!
|
//!
|
||||||
//! `SpawnWorker` writes here; the worker-comm tools (`SendToWorker`,
|
//! `SubWorkerSpawn` inserts typed `InternalWorkerSessionHandle`s; List/Send/ReadOutput/Stop use
|
||||||
//! `ReadWorkerOutput`, `StopWorker`) read and mutate the same instance. Discovery
|
//! the same in-memory authority. Internal children are not persisted, restored, discovered as
|
||||||
//! tools consult this registry together with durable Worker state. Runtime
|
//! Runtime Workers, or addressed through sockets. Restore consumes any legacy persisted process
|
||||||
//! write-through still materialises `spawned_workers.json`, but durable state lives
|
//! child records only to reclaim their delegated scope and clear obsolete metadata.
|
||||||
//! in the spawner's Worker metadata.
|
|
||||||
//!
|
//!
|
||||||
//! `ReadWorkerOutput` additionally owns a per-spawned-worker cursor here so
|
//! `SubWorkerReadOutput` owns a per-child, process-lifetime history cursor so consecutive reads
|
||||||
//! two consecutive reads yield only new assistant text. The cursor is
|
//! yield only new assistant text. Parent registry drop closes all session handles and synchronously
|
||||||
//! an item-index into the child's history; push-only history makes
|
//! returns delegated Write deny rules to the parent scope.
|
||||||
//! index stable across reads.
|
|
||||||
//!
|
|
||||||
//! Cursors intentionally do not persist; a restored registry starts with
|
|
||||||
//! fresh read positions.
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::io;
|
use std::io;
|
||||||
use std::path::Path;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use manifest::{Permission, ScopeRule, SharedScope};
|
use manifest::{Permission, ScopeRule, SharedScope};
|
||||||
use session_store::{
|
use session_store::{
|
||||||
WorkerMetadataStore, WorkerReclaimedChild, WorkerSpawnedChild, WorkerSpawnedScopeRule,
|
WorkerMetadataStore, WorkerReclaimedChild, WorkerSpawnedChild, WorkerStoreError,
|
||||||
WorkerStoreError,
|
|
||||||
};
|
};
|
||||||
use tokio::net::UnixStream;
|
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
|
use crate::internal_worker::InternalWorkerSessionHandle;
|
||||||
use crate::runtime::dir::{RuntimeDir, SpawnedWorkerRecord};
|
use crate::runtime::dir::{RuntimeDir, SpawnedWorkerRecord};
|
||||||
use crate::runtime::worker_allocation;
|
use crate::runtime::worker_allocation;
|
||||||
|
|
||||||
type RegistryStateWriter = Arc<dyn Fn(&[SpawnedWorkerRecord]) -> io::Result<()> + Send + Sync>;
|
#[derive(Clone)]
|
||||||
type RegistryReclaimWriter = Arc<dyn Fn(&SpawnedWorkerRecord) -> io::Result<()> + Send + Sync>;
|
pub(crate) struct InternalSpawnedWorkerRecord {
|
||||||
|
pub worker_name: String,
|
||||||
const RESTORE_REACHABILITY_TIMEOUT: Duration = Duration::from_millis(500);
|
pub scope_delegated: Vec<ScopeRule>,
|
||||||
const REGISTRY_CLEANUP_TIMEOUT: Duration = Duration::from_secs(15);
|
pub session: InternalWorkerSessionHandle,
|
||||||
|
}
|
||||||
|
|
||||||
pub struct SpawnedWorkerRegistry {
|
pub struct SpawnedWorkerRegistry {
|
||||||
records: Mutex<Vec<SpawnedWorkerRecord>>,
|
internal_records: std::sync::Mutex<Vec<InternalSpawnedWorkerRecord>>,
|
||||||
cursors: Mutex<HashMap<String, usize>>,
|
cursors: Mutex<HashMap<String, usize>>,
|
||||||
mutations: Mutex<()>,
|
|
||||||
runtime_dir: Arc<RuntimeDir>,
|
|
||||||
state_writer: Option<RegistryStateWriter>,
|
|
||||||
reclaim_writer: Option<RegistryReclaimWriter>,
|
|
||||||
parent_name: Option<String>,
|
|
||||||
parent_scope: Option<SharedScope>,
|
parent_scope: Option<SharedScope>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct SpawnedWorkerRegistryLoad {
|
pub struct SpawnedWorkerRegistryLoad {
|
||||||
pub registry: Arc<SpawnedWorkerRegistry>,
|
pub registry: Arc<SpawnedWorkerRegistry>,
|
||||||
|
/// True when obsolete process-child metadata was consumed and cleared.
|
||||||
pub reclaimed_unreachable: bool,
|
pub reclaimed_unreachable: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SpawnedWorkerRegistry {
|
impl SpawnedWorkerRegistry {
|
||||||
pub fn new(runtime_dir: Arc<RuntimeDir>) -> Arc<Self> {
|
/// Empty registry used by tests and non-spawning projections.
|
||||||
|
pub fn new(_runtime_dir: Arc<RuntimeDir>) -> Arc<Self> {
|
||||||
Arc::new(Self {
|
Arc::new(Self {
|
||||||
records: Mutex::new(Vec::new()),
|
internal_records: std::sync::Mutex::new(Vec::new()),
|
||||||
cursors: Mutex::new(HashMap::new()),
|
cursors: Mutex::new(HashMap::new()),
|
||||||
mutations: Mutex::new(()),
|
|
||||||
runtime_dir,
|
|
||||||
state_writer: None,
|
|
||||||
reclaim_writer: None,
|
|
||||||
parent_name: None,
|
|
||||||
parent_scope: None,
|
parent_scope: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a registry from the spawner's durable Worker state, pruning child
|
pub(crate) fn new_internal(_parent_name: String, parent_scope: SharedScope) -> Arc<Self> {
|
||||||
/// records whose socket path is already gone. The surviving list is
|
Arc::new(Self {
|
||||||
/// written through to both `spawned_workers.json` and Worker state so runtime
|
internal_records: std::sync::Mutex::new(Vec::new()),
|
||||||
/// and durable views start aligned.
|
cursors: Mutex::new(HashMap::new()),
|
||||||
|
parent_scope: Some(parent_scope),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn load_from_worker_state<St>(
|
pub async fn load_from_worker_state<St>(
|
||||||
runtime_dir: Arc<RuntimeDir>,
|
runtime_dir: Arc<RuntimeDir>,
|
||||||
store: St,
|
store: St,
|
||||||
@@ -80,12 +69,14 @@ impl SpawnedWorkerRegistry {
|
|||||||
where
|
where
|
||||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
let loaded =
|
Ok(
|
||||||
Self::load_from_worker_state_with_reclaim(runtime_dir, store, worker_name, None)
|
Self::load_from_worker_state_with_reclaim(runtime_dir, store, worker_name, None)
|
||||||
.await?;
|
.await?
|
||||||
Ok(loaded.registry)
|
.registry,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Clear obsolete process-child state instead of attempting socket reconnection.
|
||||||
pub async fn load_from_worker_state_with_reclaim<St>(
|
pub async fn load_from_worker_state_with_reclaim<St>(
|
||||||
runtime_dir: Arc<RuntimeDir>,
|
runtime_dir: Arc<RuntimeDir>,
|
||||||
store: St,
|
store: St,
|
||||||
@@ -100,231 +91,157 @@ impl SpawnedWorkerRegistry {
|
|||||||
.map_err(store_error_to_io)?;
|
.map_err(store_error_to_io)?;
|
||||||
let persisted_children = metadata
|
let persisted_children = metadata
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|m| m.spawned_children.clone())
|
.map(|metadata| metadata.spawned_children.clone())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
let mut valid_records = Vec::new();
|
||||||
let mut records = Vec::with_capacity(persisted_children.len());
|
|
||||||
let mut pruned_records = Vec::new();
|
|
||||||
for child in &persisted_children {
|
for child in &persisted_children {
|
||||||
let record = match record_from_worker_state(child) {
|
match record_from_worker_state(child) {
|
||||||
Ok(record) => record,
|
Ok(record) => {
|
||||||
Err(err) => {
|
|
||||||
warn!(
|
|
||||||
error = %err,
|
|
||||||
worker = %child.worker_name,
|
|
||||||
"dropping corrupt persisted spawned-worker record"
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if is_reachable(&record.socket_path).await {
|
|
||||||
records.push(record);
|
|
||||||
} else {
|
|
||||||
warn!(
|
warn!(
|
||||||
worker = %record.worker_name,
|
worker = %record.worker_name,
|
||||||
socket = %record.socket_path.display(),
|
"reclaiming legacy persisted process SubWorker during Internal session restore"
|
||||||
"dropping unreachable persisted spawned-worker record"
|
|
||||||
);
|
);
|
||||||
pruned_records.push(record);
|
valid_records.push(record);
|
||||||
|
}
|
||||||
|
Err(error) => warn!(
|
||||||
|
error = %error,
|
||||||
|
worker = %child.worker_name,
|
||||||
|
"clearing corrupt legacy persisted process SubWorker record"
|
||||||
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
runtime_dir.write_spawned_workers(&records).await?;
|
// Runtime projection is migration input only; the normal Internal registry is never
|
||||||
let state_writer = worker_state_writer(store.clone(), worker_name.clone());
|
// materialized into spawned_workers.json.
|
||||||
let reclaim_writer = worker_state_reclaim_writer(store.clone(), worker_name.clone());
|
let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists();
|
||||||
if metadata.is_none() {
|
if !persisted_children.is_empty() || legacy_projection_exists {
|
||||||
state_writer(&records)?;
|
runtime_dir.write_spawned_workers(&[]).await?;
|
||||||
}
|
}
|
||||||
|
if !persisted_children.is_empty() {
|
||||||
let mut reclaimed_unreachable = false;
|
let reclaimed = persisted_children
|
||||||
if !pruned_records.is_empty() {
|
|
||||||
let reclaimed = pruned_records
|
|
||||||
.iter()
|
.iter()
|
||||||
.map(|record| WorkerReclaimedChild {
|
.map(reclaimed_child_from_metadata)
|
||||||
worker_name: record.worker_name.clone(),
|
|
||||||
scope_delegated: record
|
|
||||||
.scope_delegated
|
|
||||||
.iter()
|
|
||||||
.map(|rule| WorkerSpawnedScopeRule {
|
|
||||||
target: rule.target.clone(),
|
|
||||||
permission: match rule.permission {
|
|
||||||
Permission::Read => "read".to_string(),
|
|
||||||
Permission::Write => "write".to_string(),
|
|
||||||
},
|
|
||||||
recursive: rule.recursive,
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
})
|
|
||||||
.collect();
|
.collect();
|
||||||
store
|
store
|
||||||
.reclaim_spawned_children(&worker_name, reclaimed)
|
.reclaim_spawned_children(&worker_name, reclaimed)
|
||||||
.map_err(store_error_to_io)?;
|
.map_err(store_error_to_io)?;
|
||||||
reclaimed_unreachable = true;
|
|
||||||
}
|
}
|
||||||
if parent_scope.is_some() {
|
for record in &valid_records {
|
||||||
for record in &pruned_records {
|
|
||||||
reclaim_record(&worker_name, parent_scope.as_ref(), record)?;
|
reclaim_record(&worker_name, parent_scope.as_ref(), record)?;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(SpawnedWorkerRegistryLoad {
|
Ok(SpawnedWorkerRegistryLoad {
|
||||||
registry: Arc::new(Self {
|
registry: Arc::new(Self {
|
||||||
records: Mutex::new(records),
|
internal_records: std::sync::Mutex::new(Vec::new()),
|
||||||
cursors: Mutex::new(HashMap::new()),
|
cursors: Mutex::new(HashMap::new()),
|
||||||
mutations: Mutex::new(()),
|
|
||||||
runtime_dir,
|
|
||||||
state_writer: Some(state_writer),
|
|
||||||
reclaim_writer: Some(reclaim_writer),
|
|
||||||
parent_name: Some(worker_name),
|
|
||||||
parent_scope,
|
parent_scope,
|
||||||
}),
|
}),
|
||||||
reclaimed_unreachable,
|
reclaimed_unreachable: !persisted_children.is_empty(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Append a new record and persist the full list. Returns an I/O
|
pub(crate) fn add_internal(&self, record: InternalSpawnedWorkerRecord) -> io::Result<()> {
|
||||||
/// error if either persisted write fails; the in-memory state is still
|
let mut records = self
|
||||||
/// updated in that case — the next successful write will reconcile.
|
.internal_records
|
||||||
pub async fn add(&self, record: SpawnedWorkerRecord) -> io::Result<()> {
|
.lock()
|
||||||
let _mutation = self.mutations.lock().await;
|
.map_err(|_| io::Error::other("internal spawned-worker registry lock poisoned"))?;
|
||||||
let snapshot = {
|
if records
|
||||||
let mut records = self.records.lock().await;
|
.iter()
|
||||||
|
.any(|existing| existing.worker_name == record.worker_name)
|
||||||
|
{
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::AlreadyExists,
|
||||||
|
format!(
|
||||||
|
"spawned worker `{}` is already registered",
|
||||||
|
record.worker_name
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
records.push(record);
|
records.push(record);
|
||||||
records.clone()
|
Ok(())
|
||||||
};
|
|
||||||
self.persist_records(&snapshot).await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Look up a record by worker name. Cloned so callers can drop the lock.
|
pub(crate) fn get_internal(&self, worker_name: &str) -> Option<InternalSpawnedWorkerRecord> {
|
||||||
pub async fn get(&self, worker_name: &str) -> Option<SpawnedWorkerRecord> {
|
self.internal_records
|
||||||
self.records
|
|
||||||
.lock()
|
.lock()
|
||||||
.await
|
.ok()?
|
||||||
.iter()
|
.iter()
|
||||||
.find(|r| r.worker_name == worker_name)
|
.find(|record| record.worker_name == worker_name)
|
||||||
.cloned()
|
.cloned()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list(&self) -> Vec<SpawnedWorkerRecord> {
|
pub(crate) fn list_internal(&self) -> Vec<InternalSpawnedWorkerRecord> {
|
||||||
self.records.lock().await.clone()
|
self.internal_records
|
||||||
|
.lock()
|
||||||
|
.map(|records| records.clone())
|
||||||
|
.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Remove the record for `worker_name`, persist, clear its cursor, and
|
pub(crate) async fn remove_internal(
|
||||||
/// reclaim any delegated Write scope owned by that child. Returns the
|
&self,
|
||||||
/// removed record (if any).
|
worker_name: &str,
|
||||||
pub async fn remove(&self, worker_name: &str) -> io::Result<Option<SpawnedWorkerRecord>> {
|
) -> io::Result<Option<InternalSpawnedWorkerRecord>> {
|
||||||
let _mutation = self.mutations.lock().await;
|
let removed = {
|
||||||
let (removed, snapshot) = {
|
let mut records = self
|
||||||
let mut records = self.records.lock().await;
|
.internal_records
|
||||||
let idx = records.iter().position(|r| r.worker_name == worker_name);
|
.lock()
|
||||||
let removed = idx.map(|i| records.remove(i));
|
.map_err(|_| io::Error::other("internal spawned-worker registry lock poisoned"))?;
|
||||||
let snapshot = records.clone();
|
records
|
||||||
(removed, snapshot)
|
.iter()
|
||||||
|
.position(|record| record.worker_name == worker_name)
|
||||||
|
.map(|index| records.remove(index))
|
||||||
};
|
};
|
||||||
self.persist_records(&snapshot).await?;
|
|
||||||
self.cursors.lock().await.remove(worker_name);
|
self.cursors.lock().await.remove(worker_name);
|
||||||
if let Some(record) = &removed {
|
if let (Some(record), Some(parent_scope)) = (&removed, &self.parent_scope) {
|
||||||
self.reclaim_removed_record(record.clone()).await?;
|
parent_scope
|
||||||
|
.update(|current| current.with_removed_deny_rules(delegated_write_rules(record)))
|
||||||
|
.map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?;
|
||||||
}
|
}
|
||||||
Ok(removed)
|
Ok(removed)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reclaim_removed_record(&self, record: SpawnedWorkerRecord) -> io::Result<()> {
|
|
||||||
let parent_name = self.parent_name.clone();
|
|
||||||
let parent_scope = self.parent_scope.clone();
|
|
||||||
let reclaim_writer = self.reclaim_writer.clone();
|
|
||||||
let worker_name = record.worker_name.clone();
|
|
||||||
let reclaim = tokio::task::spawn_blocking(move || {
|
|
||||||
reclaim_removed_record_blocking(parent_name, parent_scope, reclaim_writer, record)
|
|
||||||
});
|
|
||||||
tokio::time::timeout(REGISTRY_CLEANUP_TIMEOUT, reclaim)
|
|
||||||
.await
|
|
||||||
.map_err(|_| {
|
|
||||||
io::Error::new(
|
|
||||||
io::ErrorKind::TimedOut,
|
|
||||||
format!("timed out reclaiming spawned worker `{worker_name}`"),
|
|
||||||
)
|
|
||||||
})?
|
|
||||||
.map_err(|err| io::Error::other(format!("spawned-worker reclaim task failed: {err}")))?
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read-only cursor lookup. Returns 0 when no cursor has been set.
|
|
||||||
pub async fn cursor(&self, worker_name: &str) -> usize {
|
pub async fn cursor(&self, worker_name: &str) -> usize {
|
||||||
|
*self.cursors.lock().await.get(worker_name).unwrap_or(&0)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_cursor(&self, worker_name: &str, value: usize) {
|
||||||
self.cursors
|
self.cursors
|
||||||
.lock()
|
.lock()
|
||||||
.await
|
.await
|
||||||
.get(worker_name)
|
.insert(worker_name.to_owned(), value);
|
||||||
.copied()
|
|
||||||
.unwrap_or(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn set_cursor(&self, worker_name: &str, cursor: usize) {
|
|
||||||
self.cursors
|
|
||||||
.lock()
|
|
||||||
.await
|
|
||||||
.insert(worker_name.to_string(), cursor);
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn persist_records(&self, records: &[SpawnedWorkerRecord]) -> io::Result<()> {
|
|
||||||
self.runtime_dir.write_spawned_workers(records).await?;
|
|
||||||
if let Some(write_state) = &self.state_writer {
|
|
||||||
write_state(records)?;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn worker_state_writer<St>(store: St, worker_name: String) -> RegistryStateWriter
|
impl Drop for SpawnedWorkerRegistry {
|
||||||
where
|
fn drop(&mut self) {
|
||||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
let Some(parent_scope) = &self.parent_scope else {
|
||||||
{
|
return;
|
||||||
Arc::new(move |records| {
|
};
|
||||||
write_records_to_worker_state(&store, &worker_name, records).map_err(store_error_to_io)
|
let Ok(records) = self.internal_records.lock() else {
|
||||||
})
|
return;
|
||||||
|
};
|
||||||
|
let write_rules = records
|
||||||
|
.iter()
|
||||||
|
.flat_map(delegated_write_rules)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn worker_state_reclaim_writer<St>(store: St, worker_name: String) -> RegistryReclaimWriter
|
fn delegated_write_rules(record: &InternalSpawnedWorkerRecord) -> Vec<ScopeRule> {
|
||||||
where
|
record
|
||||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
|
||||||
{
|
|
||||||
Arc::new(move |record| {
|
|
||||||
let reclaimed = WorkerReclaimedChild {
|
|
||||||
worker_name: record.worker_name.clone(),
|
|
||||||
scope_delegated: record
|
|
||||||
.scope_delegated
|
.scope_delegated
|
||||||
.iter()
|
.iter()
|
||||||
.map(|rule| WorkerSpawnedScopeRule {
|
.filter(|rule| rule.permission == Permission::Write)
|
||||||
target: rule.target.clone(),
|
.cloned()
|
||||||
permission: match rule.permission {
|
.collect()
|
||||||
Permission::Read => "read".to_string(),
|
|
||||||
Permission::Write => "write".to_string(),
|
|
||||||
},
|
|
||||||
recursive: rule.recursive,
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
};
|
|
||||||
store
|
|
||||||
.reclaim_spawned_children(&worker_name, vec![reclaimed])
|
|
||||||
.map(|_| ())
|
|
||||||
.map_err(store_error_to_io)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reclaim_removed_record_blocking(
|
fn reclaimed_child_from_metadata(child: &WorkerSpawnedChild) -> WorkerReclaimedChild {
|
||||||
parent_name: Option<String>,
|
WorkerReclaimedChild {
|
||||||
parent_scope: Option<SharedScope>,
|
worker_name: child.worker_name.clone(),
|
||||||
reclaim_writer: Option<RegistryReclaimWriter>,
|
scope_delegated: child.scope_delegated.clone(),
|
||||||
record: SpawnedWorkerRecord,
|
|
||||||
) -> io::Result<()> {
|
|
||||||
if let Some(parent_name) = parent_name {
|
|
||||||
reclaim_record(&parent_name, parent_scope.as_ref(), &record)?;
|
|
||||||
} else {
|
|
||||||
release_child_allocation(&record.worker_name)?;
|
|
||||||
}
|
}
|
||||||
if let Some(write_reclaim) = reclaim_writer {
|
|
||||||
write_reclaim(&record)?;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reclaim_record(
|
fn reclaim_record(
|
||||||
@@ -332,109 +249,59 @@ fn reclaim_record(
|
|||||||
parent_scope: Option<&SharedScope>,
|
parent_scope: Option<&SharedScope>,
|
||||||
record: &SpawnedWorkerRecord,
|
record: &SpawnedWorkerRecord,
|
||||||
) -> io::Result<()> {
|
) -> io::Result<()> {
|
||||||
|
if let Ok(path) = worker_allocation::default_allocation_path() {
|
||||||
|
if let Ok(mut guard) = worker_allocation::LockFileGuard::open(&path) {
|
||||||
|
match worker_allocation::reclaim_delegated_scope(
|
||||||
|
&mut guard,
|
||||||
|
parent_name,
|
||||||
|
&record.worker_name,
|
||||||
|
&record.scope_delegated,
|
||||||
|
) {
|
||||||
|
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
|
||||||
|
Err(error) => return Err(io::Error::other(error)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(parent_scope) = parent_scope {
|
||||||
let write_rules = record
|
let write_rules = record
|
||||||
.scope_delegated
|
.scope_delegated
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|rule| rule.permission == Permission::Write)
|
.filter(|rule| rule.permission == Permission::Write)
|
||||||
.cloned()
|
.cloned()
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
parent_scope
|
||||||
let lock_path = worker_allocation::default_allocation_path()
|
|
||||||
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
|
|
||||||
let mut guard = worker_allocation::LockFileGuard::open(&lock_path)
|
|
||||||
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
|
|
||||||
worker_allocation::reclaim_delegated_scope(
|
|
||||||
&mut guard,
|
|
||||||
parent_name,
|
|
||||||
&record.worker_name,
|
|
||||||
&record.scope_delegated,
|
|
||||||
)
|
|
||||||
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
|
|
||||||
|
|
||||||
if let Some(scope) = parent_scope {
|
|
||||||
scope
|
|
||||||
.update(|current| current.with_removed_deny_rules(write_rules))
|
.update(|current| current.with_removed_deny_rules(write_rules))
|
||||||
.map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?;
|
.map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn release_child_allocation(worker_name: &str) -> io::Result<()> {
|
fn record_from_worker_state(child: &WorkerSpawnedChild) -> io::Result<SpawnedWorkerRecord> {
|
||||||
let lock_path = worker_allocation::default_allocation_path()
|
let scope_delegated = child
|
||||||
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
|
|
||||||
let mut guard = worker_allocation::LockFileGuard::open(&lock_path)
|
|
||||||
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
|
|
||||||
match worker_allocation::release_worker(&mut guard, worker_name) {
|
|
||||||
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => Ok(()),
|
|
||||||
Err(err) => Err(io::Error::new(io::ErrorKind::Other, err)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_records_to_worker_state<St>(
|
|
||||||
store: &St,
|
|
||||||
worker_name: &str,
|
|
||||||
records: &[SpawnedWorkerRecord],
|
|
||||||
) -> Result<(), WorkerStoreError>
|
|
||||||
where
|
|
||||||
St: WorkerMetadataStore,
|
|
||||||
{
|
|
||||||
let children = records
|
|
||||||
.iter()
|
|
||||||
.map(record_to_worker_state)
|
|
||||||
.collect::<Result<Vec<_>, _>>()?;
|
|
||||||
store.set_spawned_children(worker_name, children)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn record_to_worker_state(
|
|
||||||
record: &SpawnedWorkerRecord,
|
|
||||||
) -> Result<WorkerSpawnedChild, serde_json::Error> {
|
|
||||||
Ok(WorkerSpawnedChild {
|
|
||||||
worker_name: record.worker_name.clone(),
|
|
||||||
socket_path: record.socket_path.clone(),
|
|
||||||
scope_delegated: record
|
|
||||||
.scope_delegated
|
|
||||||
.iter()
|
|
||||||
.map(|rule| WorkerSpawnedScopeRule {
|
|
||||||
target: rule.target.clone(),
|
|
||||||
permission: match rule.permission {
|
|
||||||
Permission::Read => "read".to_string(),
|
|
||||||
Permission::Write => "write".to_string(),
|
|
||||||
},
|
|
||||||
recursive: rule.recursive,
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
callback_address: record.callback_address.clone(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn record_from_worker_state(
|
|
||||||
child: &WorkerSpawnedChild,
|
|
||||||
) -> Result<SpawnedWorkerRecord, serde_json::Error> {
|
|
||||||
Ok(SpawnedWorkerRecord {
|
|
||||||
worker_name: child.worker_name.clone(),
|
|
||||||
socket_path: child.socket_path.clone(),
|
|
||||||
scope_delegated: child
|
|
||||||
.scope_delegated
|
.scope_delegated
|
||||||
.iter()
|
.iter()
|
||||||
.map(|rule| {
|
.map(|rule| {
|
||||||
Ok(ScopeRule {
|
let permission = match rule.permission.as_str() {
|
||||||
target: rule.target.clone(),
|
|
||||||
permission: match rule.permission.as_str() {
|
|
||||||
"read" => Permission::Read,
|
"read" => Permission::Read,
|
||||||
"write" => Permission::Write,
|
"write" => Permission::Write,
|
||||||
other => {
|
other => {
|
||||||
return Err(serde_json::Error::io(io::Error::new(
|
return Err(io::Error::new(
|
||||||
io::ErrorKind::InvalidData,
|
io::ErrorKind::InvalidData,
|
||||||
format!("invalid permission `{other}`"),
|
format!("unsupported spawned-worker permission `{other}`"),
|
||||||
)));
|
));
|
||||||
}
|
}
|
||||||
},
|
};
|
||||||
|
Ok(ScopeRule {
|
||||||
|
target: rule.target.clone(),
|
||||||
|
permission,
|
||||||
recursive: rule.recursive,
|
recursive: rule.recursive,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.collect::<Result<Vec<_>, _>>()?,
|
.collect::<io::Result<Vec<_>>>()?;
|
||||||
|
Ok(SpawnedWorkerRecord {
|
||||||
|
worker_name: child.worker_name.clone(),
|
||||||
|
socket_path: child.socket_path.clone(),
|
||||||
|
scope_delegated,
|
||||||
callback_address: child.callback_address.clone(),
|
callback_address: child.callback_address.clone(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -442,10 +309,3 @@ fn record_from_worker_state(
|
|||||||
fn store_error_to_io(error: WorkerStoreError) -> io::Error {
|
fn store_error_to_io(error: WorkerStoreError) -> io::Error {
|
||||||
io::Error::other(error)
|
io::Error::other(error)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn is_reachable(socket: &Path) -> bool {
|
|
||||||
tokio::time::timeout(RESTORE_REACHABILITY_TIMEOUT, UnixStream::connect(socket))
|
|
||||||
.await
|
|
||||||
.map(|result| result.is_ok())
|
|
||||||
.unwrap_or(false)
|
|
||||||
}
|
|
||||||
|
|||||||
+440
-347
File diff suppressed because it is too large
Load Diff
+463
-60
@@ -43,7 +43,10 @@ use crate::hook::{
|
|||||||
PreToolCall,
|
PreToolCall,
|
||||||
};
|
};
|
||||||
use crate::in_flight::InFlightEvents;
|
use crate::in_flight::InFlightEvents;
|
||||||
use crate::internal_worker::{InternalWorkerSpec, run_internal_worker};
|
use crate::internal_worker::{
|
||||||
|
InternalWorkerAuthority, InternalWorkerIdentity, InternalWorkerSpec, run_internal_worker,
|
||||||
|
run_internal_worker_with_cancel_sender,
|
||||||
|
};
|
||||||
|
|
||||||
const COMPACTION_EXTENSION_DOMAIN: &str = "yoi.compaction";
|
const COMPACTION_EXTENSION_DOMAIN: &str = "yoi.compaction";
|
||||||
const COMPACTION_BLOCK_ID: &str = "compact";
|
const COMPACTION_BLOCK_ID: &str = "compact";
|
||||||
@@ -568,9 +571,8 @@ where
|
|||||||
St: Store + Clone,
|
St: Store + Clone,
|
||||||
{
|
{
|
||||||
/// Append `entry` to the log: disk write → counter bump → in-memory
|
/// Append `entry` to the log: disk write → counter bump → in-memory
|
||||||
/// mirror push → broadcast. The kernel orders concurrent `O_APPEND`
|
/// mirror push → broadcast. The Store owns physical write ordering and
|
||||||
/// writes for `< PIPE_BUF` lines, so no user-space serialization is
|
/// partial-write recovery; publication happens only after it returns Ok.
|
||||||
/// needed across appenders.
|
|
||||||
pub fn append_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
pub fn append_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
||||||
let loc = self.state.location();
|
let loc = self.state.location();
|
||||||
self.store.append(loc.session_id, loc.segment_id, &entry)?;
|
self.store.append(loc.session_id, loc.segment_id, &entry)?;
|
||||||
@@ -602,13 +604,13 @@ where
|
|||||||
/// interceptor commit `SystemItem`s without being generic over the
|
/// interceptor commit `SystemItem`s without being generic over the
|
||||||
/// concrete `Store` type.
|
/// concrete `Store` type.
|
||||||
pub trait SystemItemCommitter: Send + Sync {
|
pub trait SystemItemCommitter: Send + Sync {
|
||||||
fn commit_log_entry(&self, entry: LogEntry);
|
fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError>;
|
||||||
|
|
||||||
fn commit_system_item(&self, item: SystemItem) {
|
fn commit_system_item(&self, item: SystemItem) -> Result<(), StoreError> {
|
||||||
self.commit_log_entry(LogEntry::SystemItem {
|
self.commit_log_entry(LogEntry::SystemItem {
|
||||||
ts: segment_log::now_millis(),
|
ts: segment_log::now_millis(),
|
||||||
item,
|
item,
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -616,10 +618,8 @@ impl<St> SystemItemCommitter for LogWriterHandle<St>
|
|||||||
where
|
where
|
||||||
St: Store + Clone + Send + Sync + 'static,
|
St: Store + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
fn commit_log_entry(&self, entry: LogEntry) {
|
fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
||||||
if let Err(err) = self.append_entry(entry) {
|
self.append_entry(entry)
|
||||||
warn!(error = %err, "session log entry commit failed; dropping");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -654,7 +654,7 @@ pub struct Worker<C: LlmClient, St: Store> {
|
|||||||
/// and compaction so updates propagate at the next permission check.
|
/// and compaction so updates propagate at the next permission check.
|
||||||
scope: SharedScope,
|
scope: SharedScope,
|
||||||
/// Filesystem authority this Worker may pass to spawned children. Direct tools
|
/// Filesystem authority this Worker may pass to spawned children. Direct tools
|
||||||
/// continue to use `scope`; SpawnWorker validates requested child scope here.
|
/// continue to use `scope`; SubWorkerSpawn validates requested child scope here.
|
||||||
delegation_scope: DelegationScope,
|
delegation_scope: DelegationScope,
|
||||||
hook_builder: HookRegistryBuilder,
|
hook_builder: HookRegistryBuilder,
|
||||||
interceptor_installed: bool,
|
interceptor_installed: bool,
|
||||||
@@ -914,7 +914,7 @@ impl<C: LlmClient + Clone + 'static, St: Store + Clone + 'static> Worker<C, St>
|
|||||||
let writer = self.log_writer_handle();
|
let writer = self.log_writer_handle();
|
||||||
self.engine_mut().on_history_append(move |item| {
|
self.engine_mut().on_history_append(move |item| {
|
||||||
if item.is_user_message() {
|
if item.is_user_message() {
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
if matches!(
|
if matches!(
|
||||||
item,
|
item,
|
||||||
@@ -923,12 +923,12 @@ impl<C: LlmClient + Clone + 'static, St: Store + Clone + 'static> Worker<C, St>
|
|||||||
..
|
..
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
let entry = session_store::classify_history_item(item, segment_log::now_millis());
|
let entry = session_store::classify_history_item(item, segment_log::now_millis());
|
||||||
if let Err(err) = writer.append_entry(entry) {
|
writer
|
||||||
warn!(error = %err, "history append commit failed; dropping");
|
.append_entry(entry)
|
||||||
}
|
.map_err(|error| error.to_string())
|
||||||
});
|
});
|
||||||
if self.manifest.session.record_event_trace {
|
if self.manifest.session.record_event_trace {
|
||||||
let writer = self.log_writer_handle();
|
let writer = self.log_writer_handle();
|
||||||
@@ -1162,6 +1162,10 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
self.workspace_context.client()
|
self.workspace_context.client()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn workspace_context_handle(&self) -> WorkerWorkspaceContext {
|
||||||
|
self.workspace_context.clone()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn workspace_client_handle(&self) -> Arc<dyn WorkspaceClient> {
|
pub fn workspace_client_handle(&self) -> Arc<dyn WorkspaceClient> {
|
||||||
self.workspace_context.client_handle()
|
self.workspace_context.client_handle()
|
||||||
}
|
}
|
||||||
@@ -1206,7 +1210,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
},
|
},
|
||||||
})?;
|
})?;
|
||||||
self.engine_mut()
|
self.engine_mut()
|
||||||
.append_history(std::iter::once(llm_engine::Item::system_message(body)));
|
.append_history(std::iter::once(llm_engine::Item::system_message(body)))?;
|
||||||
Ok(activation)
|
Ok(activation)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1251,9 +1255,8 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Append `entry` to the session log AND publish it through the
|
/// Append `entry` to the session log AND publish it through the
|
||||||
/// broadcast sink. No user-space serialization is needed across
|
/// broadcast sink. The Store is the commit boundary: a failed write is
|
||||||
/// concurrent appenders — the kernel orders `O_APPEND` writes for
|
/// never counted or published.
|
||||||
/// lines smaller than `PIPE_BUF`.
|
|
||||||
pub(crate) fn commit_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
pub(crate) fn commit_entry(&self, entry: LogEntry) -> Result<(), StoreError> {
|
||||||
let loc = self.segment_state.location();
|
let loc = self.segment_state.location();
|
||||||
self.store.append(loc.session_id, loc.segment_id, &entry)?;
|
self.store.append(loc.session_id, loc.segment_id, &entry)?;
|
||||||
@@ -1597,8 +1600,8 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
/// `Item::system_message` just before the next LLM request, via
|
/// `Item::system_message` just before the next LLM request, via
|
||||||
/// `WorkerInterceptor::pending_history_appends`. See [`NotifyBuffer`]
|
/// `WorkerInterceptor::pending_history_appends`. See [`NotifyBuffer`]
|
||||||
/// for overflow behaviour and the lane-of-record rationale.
|
/// for overflow behaviour and the lane-of-record rationale.
|
||||||
pub fn push_notify(&self, message: String) {
|
pub fn push_notify(&self, message: String, auto_run: bool) {
|
||||||
self.pending_notifies.push_notify(message);
|
self.pending_notifies.push_notify(message, auto_run);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Push an agent-visible typed `WorkerEvent` entry onto the pending buffer.
|
/// Push an agent-visible typed `WorkerEvent` entry onto the pending buffer.
|
||||||
@@ -2083,7 +2086,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
&tool_result_summary,
|
&tool_result_summary,
|
||||||
);
|
);
|
||||||
if !closures.is_empty() {
|
if !closures.is_empty() {
|
||||||
self.engine_mut().append_history(closures);
|
self.engine_mut().append_history(closures)?;
|
||||||
}
|
}
|
||||||
self.commit_entry(LogEntry::SystemItem {
|
self.commit_entry(LogEntry::SystemItem {
|
||||||
ts: segment_log::now_millis(),
|
ts: segment_log::now_millis(),
|
||||||
@@ -2094,7 +2097,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
self.engine_mut()
|
self.engine_mut()
|
||||||
.append_history(std::iter::once(llm_engine::Item::system_message(
|
.append_history(std::iter::once(llm_engine::Item::system_message(
|
||||||
system_note,
|
system_note,
|
||||||
)));
|
)))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2164,6 +2167,12 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
),
|
),
|
||||||
"run_for_notification expects a non-UserSend InvokeKind; got {kind:?}"
|
"run_for_notification expects a non-UserSend InvokeKind; got {kind:?}"
|
||||||
);
|
);
|
||||||
|
// This is a fresh Invoke, not an explicit resume of the interrupted
|
||||||
|
// turn. Close any dangling tool calls before an auto-run notification
|
||||||
|
// can enter `Engine::resume` and execute them again after a crash.
|
||||||
|
if self.engine.as_ref().unwrap().last_run_interrupted() {
|
||||||
|
self.apply_interrupt_prep()?;
|
||||||
|
}
|
||||||
self.prepare_for_run().await?;
|
self.prepare_for_run().await?;
|
||||||
|
|
||||||
// IDLE → active marker for the buffered notification / worker-event
|
// IDLE → active marker for the buffered notification / worker-event
|
||||||
@@ -3240,6 +3249,16 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
&mut self,
|
&mut self,
|
||||||
memory_cfg: &manifest::MemoryConfig,
|
memory_cfg: &manifest::MemoryConfig,
|
||||||
threshold: u64,
|
threshold: u64,
|
||||||
|
) -> Result<ExtractDecision, WorkerError> {
|
||||||
|
self.run_extract_once_with_cancel_observer(memory_cfg, threshold, None)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_extract_once_with_cancel_observer(
|
||||||
|
&mut self,
|
||||||
|
memory_cfg: &manifest::MemoryConfig,
|
||||||
|
threshold: u64,
|
||||||
|
cancel_observer: Option<Box<dyn FnOnce(tokio::sync::mpsc::Sender<()>) + Send + 'static>>,
|
||||||
) -> Result<ExtractDecision, WorkerError> {
|
) -> Result<ExtractDecision, WorkerError> {
|
||||||
use memory::extract;
|
use memory::extract;
|
||||||
|
|
||||||
@@ -3433,51 +3452,71 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
let session_explore_state =
|
let session_explore_state =
|
||||||
SessionExploreState::new(session_view, self.workspace_client_handle(), source);
|
SessionExploreState::new(session_view, self.workspace_client_handle(), source);
|
||||||
let input_text = render_extract_input(session_explore_state.view());
|
let input_text = render_extract_input(session_explore_state.view());
|
||||||
let mut internal_tools = Vec::new();
|
let features = FeatureRegistryBuilder::new()
|
||||||
let mut internal_hook_builder = HookRegistryBuilder::new();
|
.with_module(SessionExploreFeature::new(session_explore_state.clone()));
|
||||||
let feature_report = FeatureRegistryBuilder::new()
|
let mut internal_manifest = self.manifest.clone();
|
||||||
.with_module(SessionExploreFeature::new(session_explore_state.clone()))
|
internal_manifest.model = model.clone();
|
||||||
.install_into_pending(&mut internal_tools, &mut internal_hook_builder);
|
let internal_spec = InternalWorkerSpec {
|
||||||
let installed_tool_names = feature_report.installed_tool_names();
|
identity: InternalWorkerIdentity {
|
||||||
let expected_extract_tools = [
|
kind: "memory-extract",
|
||||||
|
run_id: audit.run_id,
|
||||||
|
},
|
||||||
|
manifest: internal_manifest,
|
||||||
|
client,
|
||||||
|
system_prompt: extract_system_prompt,
|
||||||
|
input: input_text,
|
||||||
|
cache_key: Some(self.segment_id().to_string()),
|
||||||
|
max_turns: extract_worker_max_turns,
|
||||||
|
features,
|
||||||
|
required_tools: &[
|
||||||
"search_evidence",
|
"search_evidence",
|
||||||
"read_evidence",
|
"read_evidence",
|
||||||
"stage_candidate",
|
"stage_candidate",
|
||||||
"finish_extraction",
|
"finish_extraction",
|
||||||
];
|
],
|
||||||
if !expected_extract_tools.iter().all(|name| {
|
authority: InternalWorkerAuthority {
|
||||||
installed_tool_names
|
workspace: self.workspace_context.clone(),
|
||||||
.iter()
|
filesystem: WorkerFilesystemAuthority::None,
|
||||||
.any(|installed| installed == name)
|
scope: Scope::empty(),
|
||||||
}) {
|
},
|
||||||
|
};
|
||||||
|
let internal_result = match cancel_observer {
|
||||||
|
Some(observer) => run_internal_worker_with_cancel_sender(internal_spec, observer).await,
|
||||||
|
None => run_internal_worker(internal_spec).await,
|
||||||
|
};
|
||||||
|
let usage = match internal_result {
|
||||||
|
Ok(result) => {
|
||||||
|
tracing::debug!(
|
||||||
|
internal_worker_kind = result.identity.kind,
|
||||||
|
internal_worker_run_id = %result.identity.run_id,
|
||||||
|
history_entries = result.history_entries,
|
||||||
|
lifecycle = ?result.lifecycle,
|
||||||
|
"internal Worker execution completed"
|
||||||
|
);
|
||||||
|
let usage = result.usage.as_ref().map(usage_audit_from_event);
|
||||||
|
if let Some(error) = extract_internal_worker_lifecycle_error(&result.lifecycle) {
|
||||||
audit
|
audit
|
||||||
.emit(
|
.emit(
|
||||||
self.workspace_client(),
|
self.workspace_client(),
|
||||||
event_tx,
|
event_tx,
|
||||||
memory::audit::WorkerLifecycleStatus::Failed,
|
memory::audit::WorkerLifecycleStatus::Cancelled,
|
||||||
"session_explore_feature_install_failed",
|
"worker_cancelled: internal Worker run rolled back before AI output",
|
||||||
None,
|
usage,
|
||||||
Some(extract_audit_base),
|
Some(extract_audit_base),
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
return Err(WorkerError::FeatureInstall(
|
return Err(error);
|
||||||
"session-explore feature install failed".to_string(),
|
}
|
||||||
));
|
usage
|
||||||
}
|
}
|
||||||
let internal_result = run_internal_worker(InternalWorkerSpec {
|
|
||||||
slug: "memory-extract",
|
|
||||||
system_prompt: extract_system_prompt,
|
|
||||||
input: input_text,
|
|
||||||
client,
|
|
||||||
cache_key: Some(self.segment_id().to_string()),
|
|
||||||
max_turns: extract_worker_max_turns,
|
|
||||||
tools: internal_tools,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
let usage = match internal_result {
|
|
||||||
Ok(result) => result.usage.as_ref().map(usage_audit_from_event),
|
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
|
tracing::debug!(
|
||||||
|
internal_worker_kind = err.identity.kind,
|
||||||
|
internal_worker_run_id = %err.identity.run_id,
|
||||||
|
history_entries = err.history_entries,
|
||||||
|
"internal Worker execution failed"
|
||||||
|
);
|
||||||
let usage = err.usage.as_ref().map(usage_audit_from_event);
|
let usage = err.usage.as_ref().map(usage_audit_from_event);
|
||||||
audit
|
audit
|
||||||
.emit(
|
.emit(
|
||||||
@@ -3490,7 +3529,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
return Err(WorkerError::Engine(err.source));
|
return Err(err.source);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -3623,8 +3662,15 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn lifecycle_status_for_worker_error(err: &EngineError) -> memory::audit::WorkerLifecycleStatus {
|
fn extract_internal_worker_lifecycle_error(lifecycle: &WorkerRunResult) -> Option<WorkerError> {
|
||||||
if matches!(err, EngineError::Cancelled) {
|
match lifecycle {
|
||||||
|
WorkerRunResult::RolledBack => Some(WorkerError::Engine(EngineError::Cancelled)),
|
||||||
|
WorkerRunResult::Finished | WorkerRunResult::Paused | WorkerRunResult::LimitReached => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lifecycle_status_for_worker_error(err: &WorkerError) -> memory::audit::WorkerLifecycleStatus {
|
||||||
|
if matches!(err, WorkerError::Engine(EngineError::Cancelled)) {
|
||||||
memory::audit::WorkerLifecycleStatus::Cancelled
|
memory::audit::WorkerLifecycleStatus::Cancelled
|
||||||
} else {
|
} else {
|
||||||
memory::audit::WorkerLifecycleStatus::Failed
|
memory::audit::WorkerLifecycleStatus::Failed
|
||||||
@@ -3794,7 +3840,7 @@ where
|
|||||||
/// The Worker's working directory is captured once here from the
|
/// The Worker's working directory is captured once here from the
|
||||||
/// process's `std::env::current_dir()` — callers that want a
|
/// process's `std::env::current_dir()` — callers that want a
|
||||||
/// different cwd must `cd` before constructing the Worker (e.g. the
|
/// different cwd must `cd` before constructing the Worker (e.g. the
|
||||||
/// `SpawnWorker` tool sets `Command::current_dir` on the child). The
|
/// `SubWorkerSpawn` tool sets `Command::current_dir` on the child). The
|
||||||
/// captured cwd is canonicalised and validated against
|
/// captured cwd is canonicalised and validated against
|
||||||
/// `manifest.scope`.
|
/// `manifest.scope`.
|
||||||
///
|
///
|
||||||
@@ -3907,6 +3953,79 @@ where
|
|||||||
Ok(worker)
|
Ok(worker)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build an in-process Internal Worker without machine-wide allocation or durable Worker metadata.
|
||||||
|
pub(crate) async fn from_internal_manifest_with_context(
|
||||||
|
manifest: WorkerManifest,
|
||||||
|
store: St,
|
||||||
|
loader: PromptLoader,
|
||||||
|
workspace_context: WorkerWorkspaceContext,
|
||||||
|
filesystem_authority: WorkerFilesystemAuthority,
|
||||||
|
client_override: Option<Box<dyn LlmClient>>,
|
||||||
|
) -> Result<Self, WorkerError> {
|
||||||
|
let mut common = prepare_worker_common_with_context(
|
||||||
|
&manifest,
|
||||||
|
&loader,
|
||||||
|
true,
|
||||||
|
workspace_context,
|
||||||
|
filesystem_authority,
|
||||||
|
manifest.scope.clone(),
|
||||||
|
)?;
|
||||||
|
if let Some(client) = client_override {
|
||||||
|
common.client = client;
|
||||||
|
}
|
||||||
|
let session_id = session_store::new_session_id();
|
||||||
|
let segment_id = session_store::new_segment_id();
|
||||||
|
let mut engine = Engine::new(common.client);
|
||||||
|
apply_worker_manifest(&mut engine, &manifest.engine);
|
||||||
|
engine.set_cache_key(Some(segment_id.to_string()));
|
||||||
|
let scope = SharedScope::new(common.scope);
|
||||||
|
let workdir_session = workdir_session_from_authority(&common.filesystem_authority, &scope);
|
||||||
|
let mut worker = Self {
|
||||||
|
manifest,
|
||||||
|
engine: Some(engine),
|
||||||
|
store,
|
||||||
|
worker_metadata_writer: None,
|
||||||
|
segment_state: SegmentState::new(session_id, segment_id, 0),
|
||||||
|
filesystem_authority: common.filesystem_authority,
|
||||||
|
workdir_session,
|
||||||
|
workspace_context: common.workspace_context,
|
||||||
|
scope,
|
||||||
|
delegation_scope: common.delegation_scope,
|
||||||
|
hook_builder: HookRegistryBuilder::new(),
|
||||||
|
interceptor_installed: false,
|
||||||
|
compact_state: None,
|
||||||
|
usage_tracker: Arc::new(UsageTracker::new()),
|
||||||
|
metrics_tracker: Arc::new(crate::compact::metrics_tracker::MetricsTracker::new()),
|
||||||
|
usage_history: Arc::new(Mutex::new(Vec::new())),
|
||||||
|
tracker: None,
|
||||||
|
task_feature: TaskFeature::new(),
|
||||||
|
system_prompt_template: common.system_prompt_template,
|
||||||
|
feature_instructions: common.feature_instructions,
|
||||||
|
alerter: None,
|
||||||
|
event_tx: None,
|
||||||
|
in_flight: None,
|
||||||
|
ai_activity_counter: Arc::new(AtomicUsize::new(0)),
|
||||||
|
pending_notifies: NotifyBuffer::new(),
|
||||||
|
pending_attachments: Arc::new(Mutex::new(Vec::<SystemItem>::new())),
|
||||||
|
scope_allocation: None,
|
||||||
|
callback_socket: None,
|
||||||
|
runtime_ticket_role: None,
|
||||||
|
prompts: common.prompts,
|
||||||
|
inject_resident_summary: true,
|
||||||
|
extract_in_flight: Arc::new(AtomicBool::new(false)),
|
||||||
|
consolidation_in_flight: Arc::new(AtomicBool::new(false)),
|
||||||
|
extract_pointer: Arc::new(Mutex::new(None)),
|
||||||
|
memory_task: None,
|
||||||
|
user_segments: Vec::new(),
|
||||||
|
sink: SegmentLogSink::new(),
|
||||||
|
history_persistence_wired: false,
|
||||||
|
log_writer: None,
|
||||||
|
};
|
||||||
|
worker.apply_permissions_from_manifest();
|
||||||
|
worker.apply_prune_from_manifest();
|
||||||
|
Ok(worker)
|
||||||
|
}
|
||||||
|
|
||||||
/// Build a Worker spawned by another Worker (sibling process).
|
/// Build a Worker spawned by another Worker (sibling process).
|
||||||
///
|
///
|
||||||
/// Behaves like [`Worker::from_manifest`] but claims the scope
|
/// Behaves like [`Worker::from_manifest`] but claims the scope
|
||||||
@@ -4364,6 +4483,7 @@ where
|
|||||||
self.push_notify(
|
self.push_notify(
|
||||||
"Restored Worker state contained missing or unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
"Restored Worker state contained missing or unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
||||||
.to_string(),
|
.to_string(),
|
||||||
|
false,
|
||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -5644,6 +5764,104 @@ mod build_summary_prompt_tests {
|
|||||||
assert!(prompt.contains("[1 Assistant] done"));
|
assert!(prompt.contains("[1 Assistant] done"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct CancelBeforeAiExtractClient {
|
||||||
|
cancel_tx: Arc<Mutex<Option<tokio::sync::mpsc::Sender<()>>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl LlmClient for CancelBeforeAiExtractClient {
|
||||||
|
async fn stream(
|
||||||
|
&self,
|
||||||
|
_request: llm_engine::llm_client::Request,
|
||||||
|
) -> Result<
|
||||||
|
std::pin::Pin<
|
||||||
|
Box<
|
||||||
|
dyn futures::Stream<
|
||||||
|
Item = Result<
|
||||||
|
llm_engine::llm_client::event::Event,
|
||||||
|
llm_engine::llm_client::ClientError,
|
||||||
|
>,
|
||||||
|
> + Send,
|
||||||
|
>,
|
||||||
|
>,
|
||||||
|
llm_engine::llm_client::ClientError,
|
||||||
|
> {
|
||||||
|
let tx = self
|
||||||
|
.cancel_tx
|
||||||
|
.lock()
|
||||||
|
.expect("cancel sender lock")
|
||||||
|
.clone()
|
||||||
|
.expect("extract caller must install the Internal Worker cancel sender");
|
||||||
|
tx.send(()).await.expect("cancel Internal Worker");
|
||||||
|
Ok(Box::pin(futures::stream::pending()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct RecordingAuditWorkspaceClient {
|
||||||
|
requests: Mutex<Vec<WorkspaceRequest>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RecordingAuditWorkspaceClient {
|
||||||
|
fn lifecycle_audits(&self) -> Vec<memory::audit::WorkerLifecycleAudit> {
|
||||||
|
self.requests
|
||||||
|
.lock()
|
||||||
|
.expect("recorded workspace requests lock")
|
||||||
|
.iter()
|
||||||
|
.filter_map(|request| {
|
||||||
|
let operation: memory::backend::MemoryBackendOperation = serde_json::from_str(
|
||||||
|
request
|
||||||
|
.body
|
||||||
|
.as_deref()
|
||||||
|
.expect("memory backend operation body"),
|
||||||
|
)
|
||||||
|
.expect("memory backend operation");
|
||||||
|
match operation {
|
||||||
|
memory::backend::MemoryBackendOperation::AppendAudit(operation) => {
|
||||||
|
match operation.event.payload {
|
||||||
|
memory::audit::AuditPayload::WorkerLifecycle(audit) => Some(audit),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkspaceClient for RecordingAuditWorkspaceClient {
|
||||||
|
fn workspace_id(&self) -> Option<&str> {
|
||||||
|
Some("workspace-test")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn kind(&self) -> &str {
|
||||||
|
"recording-audit"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_available(&self) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn execute(
|
||||||
|
&self,
|
||||||
|
request: WorkspaceRequest,
|
||||||
|
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||||
|
self.requests
|
||||||
|
.lock()
|
||||||
|
.expect("recorded workspace requests lock")
|
||||||
|
.push(request);
|
||||||
|
Err(WorkspaceClientError::Unavailable(
|
||||||
|
"audit response is irrelevant to this regression test".to_string(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct NoopClient;
|
struct NoopClient;
|
||||||
|
|
||||||
@@ -5913,6 +6131,57 @@ mod build_summary_prompt_tests {
|
|||||||
assert_eq!(interrupt_system_count, 1);
|
assert_eq!(interrupt_system_count, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn notification_run_closes_interrupted_tool_call_before_engine_resume() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let manifest = minimal_manifest();
|
||||||
|
let store = session_store::FsStore::new(dir.path().join("sessions")).unwrap();
|
||||||
|
let cwd = dir.path().join("workspace");
|
||||||
|
std::fs::create_dir_all(&cwd).unwrap();
|
||||||
|
let scope = Scope::writable(&cwd).unwrap();
|
||||||
|
let authority = WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone());
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
Engine::new(NoopClient),
|
||||||
|
store,
|
||||||
|
WorkerWorkspaceContext::local_filesystem(None),
|
||||||
|
authority,
|
||||||
|
scope,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
worker.ensure_segment_head().unwrap();
|
||||||
|
worker.wire_history_persistence();
|
||||||
|
let dangling_call = Item::tool_call("call-1", "SideEffect", "{}");
|
||||||
|
worker
|
||||||
|
.commit_entry(LogEntry::AssistantItem {
|
||||||
|
ts: segment_log::now_millis(),
|
||||||
|
item: dangling_call.clone().into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
worker.engine_mut().set_history(vec![dangling_call]);
|
||||||
|
worker.engine_mut().set_last_run_interrupted(true);
|
||||||
|
|
||||||
|
worker
|
||||||
|
.run_for_notification(protocol::InvokeKind::Notify)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let history = worker.engine().history();
|
||||||
|
assert!(matches!(
|
||||||
|
history.get(1),
|
||||||
|
Some(Item::ToolResult { call_id, .. }) if call_id == "call-1"
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
history.get(2),
|
||||||
|
Some(Item::Message {
|
||||||
|
role: Role::System,
|
||||||
|
..
|
||||||
|
})
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
struct ResidentInjectionGates {
|
struct ResidentInjectionGates {
|
||||||
summary: bool,
|
summary: bool,
|
||||||
@@ -6265,6 +6534,140 @@ mod build_summary_prompt_tests {
|
|||||||
server.join().unwrap();
|
server.join().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_internal_extract_does_not_commit_pointer_or_completed_audit() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let cwd = dir.path().join("workspace");
|
||||||
|
std::fs::create_dir_all(&cwd).unwrap();
|
||||||
|
let store = session_store::FsStore::new(dir.path().join("sessions")).unwrap();
|
||||||
|
let cancel_tx = Arc::new(Mutex::new(None));
|
||||||
|
let client = CancelBeforeAiExtractClient {
|
||||||
|
cancel_tx: cancel_tx.clone(),
|
||||||
|
};
|
||||||
|
let audit_client = Arc::new(RecordingAuditWorkspaceClient::default());
|
||||||
|
let mut manifest = minimal_manifest();
|
||||||
|
manifest.memory = Some(manifest::MemoryConfig {
|
||||||
|
extract_threshold: Some(1),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let memory_config = manifest.memory.clone().unwrap();
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
Engine::new(client),
|
||||||
|
store,
|
||||||
|
WorkerWorkspaceContext::with_client(
|
||||||
|
Some(WorkspaceId::new("workspace-test").unwrap()),
|
||||||
|
audit_client.clone(),
|
||||||
|
),
|
||||||
|
WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone()),
|
||||||
|
Scope::writable(&cwd).unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
worker.ensure_segment_head().unwrap();
|
||||||
|
worker.wire_history_persistence();
|
||||||
|
let evidence = Item::user_message(
|
||||||
|
"The cancellation regression must leave this evidence available for retry.",
|
||||||
|
);
|
||||||
|
worker.engine_mut().set_history(vec![evidence.clone()]);
|
||||||
|
worker
|
||||||
|
.commit_entry(LogEntry::UserInput {
|
||||||
|
ts: segment_log::now_millis(),
|
||||||
|
segments: vec![text_segment(
|
||||||
|
"The cancellation regression must leave this evidence available for retry.",
|
||||||
|
)],
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
worker
|
||||||
|
.usage_history
|
||||||
|
.lock()
|
||||||
|
.expect("usage history lock")
|
||||||
|
.push(UsageRecord {
|
||||||
|
history_len: 1,
|
||||||
|
input_total_tokens: 100,
|
||||||
|
cache_read_tokens: 0,
|
||||||
|
cache_write_tokens: 0,
|
||||||
|
output_tokens: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
let entries_before = worker
|
||||||
|
.store
|
||||||
|
.read_all(worker.session_id(), worker.segment_id())
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
worker
|
||||||
|
.extract_pointer
|
||||||
|
.lock()
|
||||||
|
.expect("extract pointer lock")
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
|
||||||
|
let cancel_tx_for_extract = cancel_tx.clone();
|
||||||
|
let error = match worker
|
||||||
|
.run_extract_once_with_cancel_observer(
|
||||||
|
&memory_config,
|
||||||
|
1,
|
||||||
|
Some(Box::new(move |cancel_sender| {
|
||||||
|
*cancel_tx_for_extract
|
||||||
|
.lock()
|
||||||
|
.expect("cancel sender slot lock") = Some(cancel_sender);
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Err(error) => error,
|
||||||
|
Ok(_) => panic!("pre-AI cancellation must not complete extraction"),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(matches!(error, WorkerError::Engine(EngineError::Cancelled)));
|
||||||
|
assert!(
|
||||||
|
worker
|
||||||
|
.extract_pointer
|
||||||
|
.lock()
|
||||||
|
.expect("extract pointer lock")
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(worker.engine().history(), &[evidence]);
|
||||||
|
|
||||||
|
let entries_after = worker
|
||||||
|
.store
|
||||||
|
.read_all(worker.session_id(), worker.segment_id())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(entries_after.len(), entries_before.len());
|
||||||
|
assert!(!entries_after.iter().any(|entry| matches!(
|
||||||
|
entry,
|
||||||
|
LogEntry::Extension { domain, .. } if domain == memory::extract::EXTRACT_DOMAIN
|
||||||
|
)));
|
||||||
|
|
||||||
|
let audits = audit_client.lifecycle_audits();
|
||||||
|
assert_eq!(audits.len(), 2);
|
||||||
|
assert_eq!(audits[0].run_id, audits[1].run_id);
|
||||||
|
assert_eq!(audits[0].worker, memory::audit::AuditWorker::MemoryExtract);
|
||||||
|
assert_eq!(
|
||||||
|
audits.iter().map(|audit| audit.status).collect::<Vec<_>>(),
|
||||||
|
vec![
|
||||||
|
memory::audit::WorkerLifecycleStatus::Started,
|
||||||
|
memory::audit::WorkerLifecycleStatus::Cancelled,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!audits
|
||||||
|
.iter()
|
||||||
|
.any(|audit| { audit.status == memory::audit::WorkerLifecycleStatus::Completed })
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn successful_internal_extract_lifecycles_enter_the_commit_path() {
|
||||||
|
for lifecycle in [
|
||||||
|
WorkerRunResult::Finished,
|
||||||
|
WorkerRunResult::Paused,
|
||||||
|
WorkerRunResult::LimitReached,
|
||||||
|
] {
|
||||||
|
assert!(extract_internal_worker_lifecycle_error(&lifecycle).is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn minimal_manifest() -> WorkerManifest {
|
fn minimal_manifest() -> WorkerManifest {
|
||||||
let toml_str = r#"
|
let toml_str = r#"
|
||||||
[worker]
|
[worker]
|
||||||
|
|||||||
@@ -342,7 +342,7 @@ async fn feature_flags_default_to_core_tool_surface_only() {
|
|||||||
assert_eq!(names, vec!["Bash", "Edit", "Glob", "Grep", "Read", "Write"]);
|
assert_eq!(names, vec!["Bash", "Edit", "Glob", "Grep", "Read", "Write"]);
|
||||||
assert!(!names.iter().any(|name| name == "TaskCreate"));
|
assert!(!names.iter().any(|name| name == "TaskCreate"));
|
||||||
assert!(!names.iter().any(|name| name == "WebSearch"));
|
assert!(!names.iter().any(|name| name == "WebSearch"));
|
||||||
assert!(!names.iter().any(|name| name == "SpawnWorker"));
|
assert!(!names.iter().any(|name| name == "SubWorkerSpawn"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -386,7 +386,7 @@ permission = "write"
|
|||||||
assert!(names.iter().any(|name| name == "TaskUpdate"));
|
assert!(names.iter().any(|name| name == "TaskUpdate"));
|
||||||
assert!(names.iter().any(|name| name == "WebSearch"));
|
assert!(names.iter().any(|name| name == "WebSearch"));
|
||||||
assert!(names.iter().any(|name| name == "WebFetch"));
|
assert!(names.iter().any(|name| name == "WebFetch"));
|
||||||
assert!(!names.iter().any(|name| name == "SpawnWorker"));
|
assert!(!names.iter().any(|name| name == "SubWorkerSpawn"));
|
||||||
assert!(!names.iter().any(|name| name == "MemoryRead"));
|
assert!(!names.iter().any(|name| name == "MemoryRead"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -394,34 +394,34 @@ permission = "write"
|
|||||||
async fn project_role_tool_surfaces_keep_task_disabled_and_workers_role_scoped() {
|
async fn project_role_tool_surfaces_keep_task_disabled_and_workers_role_scoped() {
|
||||||
struct Case {
|
struct Case {
|
||||||
role: &'static str,
|
role: &'static str,
|
||||||
workers_enabled: bool,
|
sub_worker_enabled: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
let cases = [
|
let cases = [
|
||||||
Case {
|
Case {
|
||||||
role: "orchestrator",
|
role: "orchestrator",
|
||||||
workers_enabled: true,
|
sub_worker_enabled: true,
|
||||||
},
|
},
|
||||||
Case {
|
Case {
|
||||||
role: "coder",
|
role: "coder",
|
||||||
workers_enabled: false,
|
sub_worker_enabled: false,
|
||||||
},
|
},
|
||||||
Case {
|
Case {
|
||||||
role: "intake",
|
role: "intake",
|
||||||
workers_enabled: false,
|
sub_worker_enabled: false,
|
||||||
},
|
},
|
||||||
Case {
|
Case {
|
||||||
role: "reviewer",
|
role: "reviewer",
|
||||||
workers_enabled: false,
|
sub_worker_enabled: false,
|
||||||
},
|
},
|
||||||
Case {
|
Case {
|
||||||
role: "companion",
|
role: "companion",
|
||||||
workers_enabled: false,
|
sub_worker_enabled: false,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
for case in cases {
|
for case in cases {
|
||||||
let delegation = if case.workers_enabled {
|
let delegation = if case.sub_worker_enabled {
|
||||||
r#"
|
r#"
|
||||||
[[delegation_scope.allow]]
|
[[delegation_scope.allow]]
|
||||||
target = "/tmp"
|
target = "/tmp"
|
||||||
@@ -446,8 +446,8 @@ max_tokens = 100
|
|||||||
[feature.task]
|
[feature.task]
|
||||||
enabled = false
|
enabled = false
|
||||||
|
|
||||||
[feature.workers]
|
[feature.sub_worker]
|
||||||
enabled = {workers_enabled}
|
enabled = {sub_worker_enabled}
|
||||||
|
|
||||||
[[scope.allow]]
|
[[scope.allow]]
|
||||||
target = "./"
|
target = "./"
|
||||||
@@ -455,7 +455,7 @@ permission = "write"
|
|||||||
{delegation}
|
{delegation}
|
||||||
"#,
|
"#,
|
||||||
role = case.role,
|
role = case.role,
|
||||||
workers_enabled = case.workers_enabled,
|
sub_worker_enabled = case.sub_worker_enabled,
|
||||||
delegation = delegation,
|
delegation = delegation,
|
||||||
);
|
);
|
||||||
let client = MockClient::new(simple_text_events());
|
let client = MockClient::new(simple_text_events());
|
||||||
@@ -474,16 +474,16 @@ permission = "write"
|
|||||||
case.role
|
case.role
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
names.iter().any(|name| name == "SpawnWorker"),
|
names.iter().any(|name| name == "SubWorkerSpawn"),
|
||||||
case.workers_enabled,
|
case.sub_worker_enabled,
|
||||||
"{} role Worker tool exposure mismatch: {names:?}",
|
"{} role SubWorker tool exposure mismatch: {names:?}",
|
||||||
case.role
|
case.role
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn workers_feature_requires_delegation_scope() {
|
async fn sub_worker_feature_exposure_does_not_require_delegation_scope() {
|
||||||
let manifest = r#"
|
let manifest = r#"
|
||||||
[worker]
|
[worker]
|
||||||
name = "worker-management-feature-test"
|
name = "worker-management-feature-test"
|
||||||
@@ -496,7 +496,7 @@ model_id = "test-model"
|
|||||||
[engine]
|
[engine]
|
||||||
max_tokens = 100
|
max_tokens = 100
|
||||||
|
|
||||||
[feature.workers]
|
[feature.sub_worker]
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|
||||||
[[scope.allow]]
|
[[scope.allow]]
|
||||||
@@ -507,11 +507,9 @@ permission = "write"
|
|||||||
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
let result = WorkerController::spawn(worker, tmp.path()).await;
|
let result = WorkerController::spawn(worker, tmp.path()).await;
|
||||||
assert!(result.is_err());
|
|
||||||
let message = result.err().unwrap().to_string();
|
|
||||||
assert!(
|
assert!(
|
||||||
message.contains("[feature.workers].enabled = true requires non-empty"),
|
result.is_ok(),
|
||||||
"unexpected error: {message}"
|
"feature exposure must not imply delegation authority"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use manifest::{Permission, Scope, ScopeConfig, ScopeRule, SharedScope};
|
||||||
|
use session_store::{
|
||||||
|
CombinedStore, FsStore, FsWorkerStore, WorkerMetadata, WorkerMetadataStore, WorkerSpawnedChild,
|
||||||
|
WorkerSpawnedScopeRule,
|
||||||
|
};
|
||||||
|
use tempfile::TempDir;
|
||||||
|
use worker::runtime::dir::RuntimeDir;
|
||||||
|
use worker::spawn::registry::SpawnedWorkerRegistry;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_reclaims_and_clears_legacy_process_children() {
|
||||||
|
let runtime = TempDir::new().unwrap();
|
||||||
|
let sessions = TempDir::new().unwrap();
|
||||||
|
let scope_root = TempDir::new().unwrap();
|
||||||
|
let store = CombinedStore::new(
|
||||||
|
FsStore::new(sessions.path()).unwrap(),
|
||||||
|
FsWorkerStore::new(sessions.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
let mut metadata = WorkerMetadata::new("parent", None);
|
||||||
|
metadata.spawned_children.push(WorkerSpawnedChild {
|
||||||
|
worker_name: "legacy-child".into(),
|
||||||
|
socket_path: runtime.path().join("legacy.sock"),
|
||||||
|
callback_address: runtime.path().join("parent.sock"),
|
||||||
|
scope_delegated: vec![WorkerSpawnedScopeRule {
|
||||||
|
target: scope_root.path().to_path_buf(),
|
||||||
|
permission: "write".into(),
|
||||||
|
recursive: true,
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
store.write(&metadata).unwrap();
|
||||||
|
|
||||||
|
let write_rule = ScopeRule {
|
||||||
|
target: scope_root.path().to_path_buf(),
|
||||||
|
permission: Permission::Write,
|
||||||
|
recursive: true,
|
||||||
|
};
|
||||||
|
let parent_scope = SharedScope::new(
|
||||||
|
Scope::from_config(&ScopeConfig {
|
||||||
|
allow: vec![write_rule.clone()],
|
||||||
|
deny: vec![write_rule],
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
assert!(!parent_scope.snapshot().is_writable(scope_root.path()));
|
||||||
|
let runtime_dir = Arc::new(RuntimeDir::create(runtime.path(), "parent").await.unwrap());
|
||||||
|
|
||||||
|
let loaded = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim(
|
||||||
|
runtime_dir.clone(),
|
||||||
|
store.clone(),
|
||||||
|
"parent".into(),
|
||||||
|
Some(parent_scope.clone()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(loaded.reclaimed_unreachable);
|
||||||
|
assert!(parent_scope.snapshot().is_writable(scope_root.path()));
|
||||||
|
let metadata = store.read_by_name("parent").unwrap().unwrap();
|
||||||
|
assert!(metadata.spawned_children.is_empty());
|
||||||
|
assert_eq!(metadata.reclaimed_children.len(), 1);
|
||||||
|
assert_eq!(metadata.reclaimed_children[0].worker_name, "legacy-child");
|
||||||
|
let runtime_projection =
|
||||||
|
std::fs::read_to_string(runtime_dir.path().join("spawned_workers.json")).unwrap();
|
||||||
|
assert_eq!(runtime_projection.trim(), "[]");
|
||||||
|
}
|
||||||
@@ -1,743 +0,0 @@
|
|||||||
//! Integration tests for the `SpawnWorker` tool.
|
|
||||||
//!
|
|
||||||
//! These tests exercise the tool's worker-allocation delegation, subprocess
|
|
||||||
//! launch, socket handoff, and `spawned_workers.json` write through an injected
|
|
||||||
//! typed runtime command. The mock command exits immediately while a
|
|
||||||
//! test-owned Unix listener pre-binds the predicted socket path, so the tool
|
|
||||||
//! sees the "child" as live.
|
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::sync::{LazyLock, Mutex};
|
|
||||||
|
|
||||||
use client::WorkerRuntimeCommand;
|
|
||||||
use llm_engine::tool::{ToolError, ToolOutput};
|
|
||||||
use manifest::{
|
|
||||||
AuthRef, ModelManifest, Permission, SchemeKind, Scope, ScopeConfig, ScopeRule, SharedScope,
|
|
||||||
WorkerManifest, WorkerManifestConfig, WorkerMetaConfig,
|
|
||||||
};
|
|
||||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
|
||||||
use protocol::{Event, Method};
|
|
||||||
use serde_json::json;
|
|
||||||
use std::sync::Arc;
|
|
||||||
use tempfile::TempDir;
|
|
||||||
use tokio::net::UnixListener;
|
|
||||||
use worker::runtime::dir::{RuntimeDir, SpawnedWorkerRecord};
|
|
||||||
use worker::runtime::worker_allocation::{self, LockFileGuard};
|
|
||||||
use worker::spawn::registry::SpawnedWorkerRegistry;
|
|
||||||
use worker::spawn::tool::spawn_worker_tool_with_runtime_command;
|
|
||||||
|
|
||||||
/// Serialises tests that mutate `YOI_RUNTIME_DIR` across the
|
|
||||||
/// thread-pooled test harness.
|
|
||||||
static ENV_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
|
|
||||||
|
|
||||||
struct EnvGuard {
|
|
||||||
_lock: std::sync::MutexGuard<'static, ()>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl EnvGuard {
|
|
||||||
fn acquire() -> Self {
|
|
||||||
Self {
|
|
||||||
_lock: ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Set up a tempdir, point `YOI_RUNTIME_DIR` at it (so
|
|
||||||
/// `workers.json` and per-Worker runtime subdirs both land in the
|
|
||||||
/// sandbox), and install a live top-level "spawner" allocation so the
|
|
||||||
/// tool has something to delegate from. Returns the tempdir (keeps it
|
|
||||||
/// alive for the test's lifetime), runtime base, spawner socket, and
|
|
||||||
/// the spawner's runtime dir.
|
|
||||||
async fn setup_spawner(
|
|
||||||
spawner_name: &str,
|
|
||||||
allow_root: &Path,
|
|
||||||
) -> (TempDir, PathBuf, PathBuf, Arc<RuntimeDir>) {
|
|
||||||
let tmp = TempDir::new().unwrap();
|
|
||||||
let runtime_base = tmp.path().to_path_buf();
|
|
||||||
unsafe {
|
|
||||||
// Outranking env vars must be cleared so `paths::runtime_dir`
|
|
||||||
// resolves to our sandbox instead of the developer's real one.
|
|
||||||
std::env::remove_var("YOI_HOME");
|
|
||||||
std::env::remove_var("XDG_RUNTIME_DIR");
|
|
||||||
std::env::set_var("YOI_RUNTIME_DIR", &runtime_base);
|
|
||||||
}
|
|
||||||
|
|
||||||
let spawner_rd = RuntimeDir::create(&runtime_base, spawner_name)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let spawner_socket = spawner_rd.socket_path();
|
|
||||||
|
|
||||||
let _guard = worker_allocation::install_top_level(
|
|
||||||
spawner_name.into(),
|
|
||||||
std::process::id(),
|
|
||||||
spawner_socket.clone(),
|
|
||||||
vec![ScopeRule {
|
|
||||||
target: allow_root.to_path_buf(),
|
|
||||||
permission: Permission::Write,
|
|
||||||
recursive: true,
|
|
||||||
}],
|
|
||||||
session_store::new_segment_id(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
// Leak the guard — the spawner allocation needs to outlive the
|
|
||||||
// tool call. Dropping it would auto-release the allocation, which
|
|
||||||
// defeats the point of the test.
|
|
||||||
std::mem::forget(_guard);
|
|
||||||
|
|
||||||
(tmp, runtime_base, spawner_socket, Arc::new(spawner_rd))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Bind a Unix listener at the path the tool will predict for the
|
|
||||||
/// spawned worker. The tool only needs the socket to accept a connection
|
|
||||||
/// and receive one `Method::Run` line; the returned `UnixListener` is
|
|
||||||
/// read from by the caller in a joined task.
|
|
||||||
async fn bind_mock_worker_socket(
|
|
||||||
runtime_base: &Path,
|
|
||||||
worker_name: &str,
|
|
||||||
) -> (PathBuf, UnixListener) {
|
|
||||||
let dir = runtime_base.join(worker_name);
|
|
||||||
tokio::fs::create_dir_all(&dir).await.unwrap();
|
|
||||||
let socket = dir.join("sock");
|
|
||||||
let listener = UnixListener::bind(&socket).unwrap();
|
|
||||||
(socket, listener)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Launch a tokio task that accepts connections until one carries a
|
|
||||||
/// `Method` line, then acknowledges it and returns it. `wait_for_socket`
|
|
||||||
/// inside the tool makes a probe connection that carries no data, so the
|
|
||||||
/// task must tolerate an empty connection and keep listening.
|
|
||||||
fn accept_one_method(listener: UnixListener) -> tokio::task::JoinHandle<Option<Method>> {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
loop {
|
|
||||||
let (stream, _) = listener.accept().await.ok()?;
|
|
||||||
let (reader, writer) = stream.into_split();
|
|
||||||
let mut r = JsonLineReader::new(reader);
|
|
||||||
let mut w = JsonLineWriter::new(writer);
|
|
||||||
if w.write(&Event::Snapshot {
|
|
||||||
entries: Vec::new(),
|
|
||||||
greeting: protocol::Greeting {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
cwd: "/tmp".into(),
|
|
||||||
provider: "test".into(),
|
|
||||||
model: "test".into(),
|
|
||||||
scope_summary: String::new(),
|
|
||||||
tools: Vec::new(),
|
|
||||||
context_window: 200_000,
|
|
||||||
context_tokens: 0,
|
|
||||||
},
|
|
||||||
status: protocol::WorkerStatus::Idle,
|
|
||||||
in_flight: Default::default(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.is_err()
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if let Ok(Some(method)) = r.next::<Method>().await {
|
|
||||||
w.write(&Event::UserMessage {
|
|
||||||
segments: vec![protocol::Segment::text("accepted")],
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.ok()?;
|
|
||||||
return Some(method);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn mock_runtime_command() -> WorkerRuntimeCommand {
|
|
||||||
WorkerRuntimeCommand::new(which_true(), Vec::new())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn cwd_recording_runtime_command(script_path: &Path, output_path: &Path) -> WorkerRuntimeCommand {
|
|
||||||
let output = output_path.display();
|
|
||||||
std::fs::write(
|
|
||||||
script_path,
|
|
||||||
format!(
|
|
||||||
"tmp=\"{output}.tmp\"\npwd > \"$tmp\"\nprintf '%s\\n' \"$@\" >> \"$tmp\"\nmv \"$tmp\" \"{output}\"\n"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
WorkerRuntimeCommand::new(which_sh(), vec![script_path.as_os_str().to_os_string()])
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_recorded_runtime_invocation(output_path: &Path) -> Vec<String> {
|
|
||||||
for _ in 0..50 {
|
|
||||||
if let Ok(content) = std::fs::read_to_string(output_path) {
|
|
||||||
return content.lines().map(str::to_owned).collect();
|
|
||||||
}
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
|
||||||
}
|
|
||||||
panic!(
|
|
||||||
"runtime command did not record invocation at {}",
|
|
||||||
output_path.display()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `/bin/true` only exists on FHS-compliant systems. Resolve it via PATH
|
|
||||||
/// so the tests work regardless of distro.
|
|
||||||
fn which_true() -> String {
|
|
||||||
for dir in std::env::var_os("PATH")
|
|
||||||
.map(|p| std::env::split_paths(&p).collect::<Vec<_>>())
|
|
||||||
.unwrap_or_default()
|
|
||||||
{
|
|
||||||
let candidate = dir.join("true");
|
|
||||||
if candidate.is_file() {
|
|
||||||
return candidate.to_string_lossy().into_owned();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
"/bin/true".into()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn which_sh() -> String {
|
|
||||||
for dir in std::env::var_os("PATH")
|
|
||||||
.map(|p| std::env::split_paths(&p).collect::<Vec<_>>())
|
|
||||||
.unwrap_or_default()
|
|
||||||
{
|
|
||||||
let candidate = dir.join("sh");
|
|
||||||
if candidate.is_file() {
|
|
||||||
return candidate.to_string_lossy().into_owned();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
"/bin/sh".into()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Tests don't exercise the model — they intercept the spawned
|
|
||||||
/// child via a mock socket — but `spawn_worker_tool` needs a value to
|
|
||||||
/// embed in the overlay TOML. Any well-formed `ModelManifest` works.
|
|
||||||
fn dummy_model() -> ModelManifest {
|
|
||||||
ModelManifest {
|
|
||||||
scheme: Some(SchemeKind::Anthropic),
|
|
||||||
base_url: None,
|
|
||||||
model_id: Some("claude-test".into()),
|
|
||||||
auth: Some(AuthRef::None),
|
|
||||||
capability: None,
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn dummy_manifest(allow_root: &Path) -> WorkerManifest {
|
|
||||||
dummy_manifest_with_delegation(allow_root, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn dummy_manifest_with_delegation(allow_root: &Path, allow_delegation: bool) -> WorkerManifest {
|
|
||||||
let direct_scope = ScopeConfig {
|
|
||||||
allow: vec![ScopeRule {
|
|
||||||
target: allow_root.to_path_buf(),
|
|
||||||
permission: Permission::Write,
|
|
||||||
recursive: true,
|
|
||||||
}],
|
|
||||||
deny: Vec::new(),
|
|
||||||
};
|
|
||||||
let delegation_scope = if allow_delegation {
|
|
||||||
direct_scope.clone()
|
|
||||||
} else {
|
|
||||||
ScopeConfig::default()
|
|
||||||
};
|
|
||||||
dummy_manifest_with_scopes(direct_scope, delegation_scope)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn dummy_manifest_with_scopes(
|
|
||||||
direct_scope: ScopeConfig,
|
|
||||||
delegation_scope: ScopeConfig,
|
|
||||||
) -> WorkerManifest {
|
|
||||||
WorkerManifestConfig {
|
|
||||||
worker: WorkerMetaConfig {
|
|
||||||
name: Some("root".into()),
|
|
||||||
prompt_pack: None,
|
|
||||||
},
|
|
||||||
model: dummy_model(),
|
|
||||||
scope: direct_scope,
|
|
||||||
delegation_scope,
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
.try_into()
|
|
||||||
.unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn builtin_prompts() -> Arc<worker::PromptCatalog> {
|
|
||||||
worker::PromptCatalog::builtins_only().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Spawner-side `SharedScope` mirroring the `allow_root` granted by
|
|
||||||
/// `setup_spawner`. The tool revokes Write rules from this scope on
|
|
||||||
/// successful spawn — tests can `load()` it to assert the
|
|
||||||
/// revocation took effect.
|
|
||||||
fn shared_scope_for(allow_root: &Path) -> SharedScope {
|
|
||||||
SharedScope::new(Scope::writable(allow_root).unwrap())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn clear_env() {
|
|
||||||
unsafe {
|
|
||||||
std::env::remove_var("YOI_RUNTIME_DIR");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn spawn_worker_launches_runtime_in_workspace_and_process_cwd() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
|
|
||||||
let allow_root = TempDir::new().unwrap();
|
|
||||||
let child_cwd = allow_root.path().join("child-cwd");
|
|
||||||
std::fs::create_dir(&child_cwd).unwrap();
|
|
||||||
let script = allow_root.path().join("record-pwd.sh");
|
|
||||||
let output_path = allow_root.path().join("pwd.txt");
|
|
||||||
let (_tmp, runtime_base, spawner_socket, spawner_rd) =
|
|
||||||
setup_spawner("root", allow_root.path()).await;
|
|
||||||
|
|
||||||
let (_predicted_socket, listener) = bind_mock_worker_socket(&runtime_base, "child-cwd").await;
|
|
||||||
let received = accept_one_method(listener);
|
|
||||||
|
|
||||||
let registry = SpawnedWorkerRegistry::new(spawner_rd);
|
|
||||||
let def = spawn_worker_tool_with_runtime_command(
|
|
||||||
"root".into(),
|
|
||||||
spawner_socket,
|
|
||||||
runtime_base,
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
registry,
|
|
||||||
None,
|
|
||||||
dummy_manifest(allow_root.path()),
|
|
||||||
shared_scope_for(allow_root.path()),
|
|
||||||
builtin_prompts(),
|
|
||||||
cwd_recording_runtime_command(&script, &output_path),
|
|
||||||
);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
|
|
||||||
let input = json!({
|
|
||||||
"name": "child-cwd",
|
|
||||||
"task": "hello",
|
|
||||||
"profile": "inherit",
|
|
||||||
"cwd": child_cwd.to_str().unwrap(),
|
|
||||||
"scope": [{
|
|
||||||
"target": allow_root.path().to_str().unwrap(),
|
|
||||||
"permission": "write"
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
assert!(matches!(received.await.unwrap(), Some(Method::Run { .. })));
|
|
||||||
let invocation = read_recorded_runtime_invocation(&output_path).await;
|
|
||||||
assert_eq!(invocation[0], child_cwd.to_str().unwrap());
|
|
||||||
assert!(
|
|
||||||
invocation
|
|
||||||
.windows(2)
|
|
||||||
.any(|pair| pair[0] == "--workspace" && pair[1] == allow_root.path().to_str().unwrap()),
|
|
||||||
"invocation should carry inherited workspace root: {invocation:?}"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
!invocation.iter().any(|arg| arg == "--tool-cwd"),
|
|
||||||
"cwd should be process current directory, not a runtime argument: {invocation:?}"
|
|
||||||
);
|
|
||||||
|
|
||||||
clear_env();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn spawn_worker_omitted_cwd_preserves_spawner_cwd() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
|
|
||||||
let allow_root = TempDir::new().unwrap();
|
|
||||||
let script = allow_root.path().join("record-pwd.sh");
|
|
||||||
let output_path = allow_root.path().join("pwd.txt");
|
|
||||||
let (_tmp, runtime_base, spawner_socket, spawner_rd) =
|
|
||||||
setup_spawner("root", allow_root.path()).await;
|
|
||||||
|
|
||||||
let (_predicted_socket, listener) =
|
|
||||||
bind_mock_worker_socket(&runtime_base, "child-default-cwd").await;
|
|
||||||
let received = accept_one_method(listener);
|
|
||||||
|
|
||||||
let registry = SpawnedWorkerRegistry::new(spawner_rd);
|
|
||||||
let def = spawn_worker_tool_with_runtime_command(
|
|
||||||
"root".into(),
|
|
||||||
spawner_socket,
|
|
||||||
runtime_base,
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
registry,
|
|
||||||
None,
|
|
||||||
dummy_manifest(allow_root.path()),
|
|
||||||
shared_scope_for(allow_root.path()),
|
|
||||||
builtin_prompts(),
|
|
||||||
cwd_recording_runtime_command(&script, &output_path),
|
|
||||||
);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
|
|
||||||
let input = json!({
|
|
||||||
"name": "child-default-cwd",
|
|
||||||
"task": "hello",
|
|
||||||
"profile": "inherit",
|
|
||||||
"scope": [{
|
|
||||||
"target": allow_root.path().to_str().unwrap(),
|
|
||||||
"permission": "write"
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
assert!(matches!(received.await.unwrap(), Some(Method::Run { .. })));
|
|
||||||
let invocation = read_recorded_runtime_invocation(&output_path).await;
|
|
||||||
assert_eq!(invocation[0], allow_root.path().to_str().unwrap());
|
|
||||||
assert!(
|
|
||||||
!invocation.iter().any(|arg| arg == "--tool-cwd"),
|
|
||||||
"omitted cwd should preserve spawner cwd as process cwd: {invocation:?}"
|
|
||||||
);
|
|
||||||
|
|
||||||
clear_env();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn spawn_worker_delegates_scope_and_sends_run() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
|
|
||||||
let allow_root = TempDir::new().unwrap();
|
|
||||||
let (_tmp, runtime_base, spawner_socket, spawner_rd) =
|
|
||||||
setup_spawner("root", allow_root.path()).await;
|
|
||||||
|
|
||||||
let (_predicted_socket, listener) = bind_mock_worker_socket(&runtime_base, "child").await;
|
|
||||||
let received = accept_one_method(listener);
|
|
||||||
|
|
||||||
let registry = SpawnedWorkerRegistry::new(spawner_rd.clone());
|
|
||||||
let spawner_scope = shared_scope_for(allow_root.path());
|
|
||||||
let def = spawn_worker_tool_with_runtime_command(
|
|
||||||
"root".into(),
|
|
||||||
spawner_socket.clone(),
|
|
||||||
runtime_base.clone(),
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
registry,
|
|
||||||
None,
|
|
||||||
dummy_manifest(allow_root.path()),
|
|
||||||
spawner_scope.clone(),
|
|
||||||
builtin_prompts(),
|
|
||||||
mock_runtime_command(),
|
|
||||||
);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
|
|
||||||
let input = json!({
|
|
||||||
"name": "child",
|
|
||||||
"task": "hello",
|
|
||||||
"profile": "inherit",
|
|
||||||
"scope": [{
|
|
||||||
"target": allow_root.path().to_str().unwrap(),
|
|
||||||
"permission": "write"
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
// Pre-spawn: the spawner can write to the delegated path.
|
|
||||||
assert!(
|
|
||||||
spawner_scope
|
|
||||||
.load()
|
|
||||||
.is_writable(&allow_root.path().join("a.txt"))
|
|
||||||
);
|
|
||||||
|
|
||||||
let output: ToolOutput = tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
assert!(
|
|
||||||
output.summary.contains("child"),
|
|
||||||
"summary: {}",
|
|
||||||
output.summary
|
|
||||||
);
|
|
||||||
|
|
||||||
// Verify the tool delivered Method::Run to the socket.
|
|
||||||
let method = received.await.unwrap().expect("expected one Method line");
|
|
||||||
match method {
|
|
||||||
Method::Run { input } => match input.as_slice() {
|
|
||||||
[protocol::Segment::Text { content }] => assert_eq!(content, "hello"),
|
|
||||||
other => panic!("expected single Text segment, got {other:?}"),
|
|
||||||
},
|
|
||||||
other => panic!("expected Run, got {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify worker_allocation has the child allocation under `root`.
|
|
||||||
let lock_path = worker_allocation::default_allocation_path().unwrap();
|
|
||||||
let guard = LockFileGuard::open(&lock_path).unwrap();
|
|
||||||
let child = guard
|
|
||||||
.data()
|
|
||||||
.find("child")
|
|
||||||
.expect("child allocation missing after spawn");
|
|
||||||
assert_eq!(child.delegated_from.as_deref(), Some("root"));
|
|
||||||
drop(guard);
|
|
||||||
|
|
||||||
// Verify spawned_workers.json was written.
|
|
||||||
let spawned_file = spawner_rd.path().join("spawned_workers.json");
|
|
||||||
let contents = std::fs::read_to_string(&spawned_file).unwrap();
|
|
||||||
let records: Vec<SpawnedWorkerRecord> = serde_json::from_str(&contents).unwrap();
|
|
||||||
assert_eq!(records.len(), 1);
|
|
||||||
assert_eq!(records[0].worker_name, "child");
|
|
||||||
assert_eq!(records[0].callback_address, spawner_socket);
|
|
||||||
|
|
||||||
// Post-spawn: the spawner's runtime scope has been demoted on the
|
|
||||||
// delegated path. Write is gone, Read remains.
|
|
||||||
let post = spawner_scope.load();
|
|
||||||
assert_eq!(
|
|
||||||
post.permission_at(&allow_root.path().join("a.txt")),
|
|
||||||
Some(Permission::Read),
|
|
||||||
"spawner should still be able to read delegated path"
|
|
||||||
);
|
|
||||||
|
|
||||||
clear_env();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn spawn_worker_requires_explicit_delegation_even_with_direct_scope() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
|
|
||||||
let allow_root = TempDir::new().unwrap();
|
|
||||||
let (_tmp, runtime_base, spawner_socket, spawner_rd) =
|
|
||||||
setup_spawner("root", allow_root.path()).await;
|
|
||||||
|
|
||||||
let manifest = dummy_manifest_with_delegation(allow_root.path(), false);
|
|
||||||
let direct = Scope::from_config(&manifest.scope).unwrap();
|
|
||||||
assert!(direct.is_writable(&allow_root.path().join("direct.txt")));
|
|
||||||
|
|
||||||
let registry = SpawnedWorkerRegistry::new(spawner_rd.clone());
|
|
||||||
let def = spawn_worker_tool_with_runtime_command(
|
|
||||||
"root".into(),
|
|
||||||
spawner_socket,
|
|
||||||
runtime_base,
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
registry,
|
|
||||||
None,
|
|
||||||
manifest,
|
|
||||||
shared_scope_for(allow_root.path()),
|
|
||||||
builtin_prompts(),
|
|
||||||
mock_runtime_command(),
|
|
||||||
);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
|
|
||||||
let input = json!({
|
|
||||||
"name": "child-no-delegation",
|
|
||||||
"task": "hello",
|
|
||||||
"profile": "inherit",
|
|
||||||
"scope": [{
|
|
||||||
"target": allow_root.path().to_str().unwrap(),
|
|
||||||
"permission": "write"
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
let err = tool.execute(&input, Default::default()).await.unwrap_err();
|
|
||||||
match err {
|
|
||||||
ToolError::InvalidArgument(message) => {
|
|
||||||
assert!(message.contains("no delegation scope grant"), "{message}");
|
|
||||||
assert!(message.contains("direct filesystem scope"), "{message}");
|
|
||||||
}
|
|
||||||
other => panic!("expected InvalidArgument, got {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
clear_env();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn spawn_worker_rejects_child_non_recursive_scope_under_parent_non_recursive_delegation() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
|
|
||||||
let allow_root = TempDir::new().unwrap();
|
|
||||||
let child = allow_root.path().join("child");
|
|
||||||
std::fs::create_dir(&child).unwrap();
|
|
||||||
let (_tmp, runtime_base, spawner_socket, spawner_rd) =
|
|
||||||
setup_spawner("root", allow_root.path()).await;
|
|
||||||
|
|
||||||
let direct_scope = ScopeConfig {
|
|
||||||
allow: vec![ScopeRule {
|
|
||||||
target: allow_root.path().to_path_buf(),
|
|
||||||
permission: Permission::Write,
|
|
||||||
recursive: true,
|
|
||||||
}],
|
|
||||||
deny: Vec::new(),
|
|
||||||
};
|
|
||||||
let delegation_scope = ScopeConfig {
|
|
||||||
allow: vec![ScopeRule {
|
|
||||||
target: allow_root.path().to_path_buf(),
|
|
||||||
permission: Permission::Write,
|
|
||||||
recursive: false,
|
|
||||||
}],
|
|
||||||
deny: Vec::new(),
|
|
||||||
};
|
|
||||||
let manifest = dummy_manifest_with_scopes(direct_scope, delegation_scope);
|
|
||||||
|
|
||||||
let registry = SpawnedWorkerRegistry::new(spawner_rd.clone());
|
|
||||||
let def = spawn_worker_tool_with_runtime_command(
|
|
||||||
"root".into(),
|
|
||||||
spawner_socket,
|
|
||||||
runtime_base,
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
registry,
|
|
||||||
None,
|
|
||||||
manifest,
|
|
||||||
shared_scope_for(allow_root.path()),
|
|
||||||
builtin_prompts(),
|
|
||||||
mock_runtime_command(),
|
|
||||||
);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
|
|
||||||
let input = json!({
|
|
||||||
"name": "child-nonrecursive-overgrant",
|
|
||||||
"task": "hello",
|
|
||||||
"profile": "inherit",
|
|
||||||
"scope": [{
|
|
||||||
"target": child.to_str().unwrap(),
|
|
||||||
"permission": "write",
|
|
||||||
"recursive": false
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
let err = tool.execute(&input, Default::default()).await.unwrap_err();
|
|
||||||
match err {
|
|
||||||
ToolError::InvalidArgument(message) => {
|
|
||||||
assert!(
|
|
||||||
message.contains("outside this Worker's delegation scope grant"),
|
|
||||||
"{message}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
other => panic!("expected InvalidArgument, got {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
clear_env();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn spawn_worker_rejects_scope_outside_spawner() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
|
|
||||||
let allow_root = TempDir::new().unwrap();
|
|
||||||
let outside = TempDir::new().unwrap();
|
|
||||||
let (_tmp, runtime_base, spawner_socket, spawner_rd) =
|
|
||||||
setup_spawner("root", allow_root.path()).await;
|
|
||||||
|
|
||||||
let registry = SpawnedWorkerRegistry::new(spawner_rd);
|
|
||||||
let spawner_scope = shared_scope_for(allow_root.path());
|
|
||||||
let def = spawn_worker_tool_with_runtime_command(
|
|
||||||
"root".into(),
|
|
||||||
spawner_socket,
|
|
||||||
runtime_base,
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
registry,
|
|
||||||
None,
|
|
||||||
dummy_manifest(allow_root.path()),
|
|
||||||
spawner_scope.clone(),
|
|
||||||
builtin_prompts(),
|
|
||||||
mock_runtime_command(),
|
|
||||||
);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
|
|
||||||
// Request write access to a path the spawner doesn't own.
|
|
||||||
let input = json!({
|
|
||||||
"name": "child",
|
|
||||||
"task": "nope",
|
|
||||||
"profile": "inherit",
|
|
||||||
"scope": [{
|
|
||||||
"target": outside.path().to_str().unwrap(),
|
|
||||||
"permission": "write"
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
let err = tool.execute(&input, Default::default()).await.unwrap_err();
|
|
||||||
match err {
|
|
||||||
ToolError::InvalidArgument(msg) => {
|
|
||||||
assert!(
|
|
||||||
msg.contains("outside this Worker's delegation scope grant"),
|
|
||||||
"expected delegation-scope wording: {msg}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
other => panic!("expected InvalidArgument, got {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
// The spawner's allocation is unchanged; no "child" appeared.
|
|
||||||
let lock_path = worker_allocation::default_allocation_path().unwrap();
|
|
||||||
let guard = LockFileGuard::open(&lock_path).unwrap();
|
|
||||||
assert!(guard.data().find("child").is_none());
|
|
||||||
|
|
||||||
// Failed spawn must not have demoted the spawner's scope either.
|
|
||||||
assert!(
|
|
||||||
spawner_scope
|
|
||||||
.load()
|
|
||||||
.is_writable(&allow_root.path().join("a.txt"))
|
|
||||||
);
|
|
||||||
|
|
||||||
clear_env();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn spawn_worker_rolls_back_reservation_when_socket_never_appears() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
|
|
||||||
let allow_root = TempDir::new().unwrap();
|
|
||||||
let (_tmp, runtime_base, spawner_socket, spawner_rd) =
|
|
||||||
setup_spawner("root", allow_root.path()).await;
|
|
||||||
|
|
||||||
// Deliberately do NOT bind a socket at the predicted path. The
|
|
||||||
// tool's wait_for_socket should time out, triggering rollback.
|
|
||||||
// `SOCKET_WAIT_TIMEOUT` is 10s in production; we override via a
|
|
||||||
// tighter env-based lock path and just accept the wait in test.
|
|
||||||
// To keep the test fast, use a shorter wait by constructing a
|
|
||||||
// short-lived separate instance.
|
|
||||||
//
|
|
||||||
// As the tool's timeout is internal, we accept the 10s wait here —
|
|
||||||
// marked with `// slow_test`. Keep the rest of the test suite fast
|
|
||||||
// by running this test alone when iterating.
|
|
||||||
|
|
||||||
let registry = SpawnedWorkerRegistry::new(spawner_rd);
|
|
||||||
let spawner_scope = shared_scope_for(allow_root.path());
|
|
||||||
let def = spawn_worker_tool_with_runtime_command(
|
|
||||||
"root".into(),
|
|
||||||
spawner_socket,
|
|
||||||
runtime_base,
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
allow_root.path().to_path_buf(),
|
|
||||||
registry,
|
|
||||||
None,
|
|
||||||
dummy_manifest(allow_root.path()),
|
|
||||||
spawner_scope.clone(),
|
|
||||||
builtin_prompts(),
|
|
||||||
mock_runtime_command(),
|
|
||||||
);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
|
|
||||||
let input = json!({
|
|
||||||
"name": "ghost",
|
|
||||||
"task": "will never be delivered",
|
|
||||||
"profile": "inherit",
|
|
||||||
"scope": [{
|
|
||||||
"target": allow_root.path().to_str().unwrap(),
|
|
||||||
"permission": "write"
|
|
||||||
}]
|
|
||||||
})
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
let err = tool.execute(&input, Default::default()).await.unwrap_err();
|
|
||||||
match err {
|
|
||||||
ToolError::ExecutionFailed(msg) => {
|
|
||||||
assert!(
|
|
||||||
msg.contains("socket did not appear"),
|
|
||||||
"expected socket timeout wording: {msg}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
other => panic!("expected ExecutionFailed, got {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rollback assertion: the reserved "ghost" allocation is gone.
|
|
||||||
let lock_path = worker_allocation::default_allocation_path().unwrap();
|
|
||||||
let guard = LockFileGuard::open(&lock_path).unwrap();
|
|
||||||
assert!(
|
|
||||||
guard.data().find("ghost").is_none(),
|
|
||||||
"allocation was not rolled back after socket wait timed out"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Spawner's runtime scope must also be untouched — revoke is
|
|
||||||
// performed only after exec_child succeeds.
|
|
||||||
assert!(
|
|
||||||
spawner_scope
|
|
||||||
.load()
|
|
||||||
.is_writable(&allow_root.path().join("a.txt"))
|
|
||||||
);
|
|
||||||
|
|
||||||
clear_env();
|
|
||||||
}
|
|
||||||
@@ -1,728 +0,0 @@
|
|||||||
//! Integration tests for the worker-comm tools (`SendToWorker`,
|
|
||||||
//! `ReadWorkerOutput`, `StopWorker`).
|
|
||||||
//!
|
|
||||||
//! The real child Worker binary is not started. Instead each test stands
|
|
||||||
//! up a mock `UnixListener` that speaks the socket protocol directly:
|
|
||||||
//! it emits the connect-time `Event::Snapshot`, accepts methods such as
|
|
||||||
//! `Method::Run` / `Method::Shutdown`, and responds with the relevant
|
|
||||||
//! events when needed. This keeps the tests fast and independent of the
|
|
||||||
//! LLM layer — the tools are exercised for their wire behaviour alone.
|
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::sync::{Arc, LazyLock, Mutex};
|
|
||||||
|
|
||||||
use llm_engine::llm_client::types::{ContentPart, Item, Role};
|
|
||||||
use llm_engine::tool::ToolOutput;
|
|
||||||
use manifest::{Permission, Scope, ScopeRule, SharedScope};
|
|
||||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
|
||||||
use protocol::{ErrorCode, Event, Greeting, Method};
|
|
||||||
use serde_json::json;
|
|
||||||
use session_store::FsStore;
|
|
||||||
use session_store::{CombinedStore, FsWorkerStore, WorkerMetadataStore};
|
|
||||||
use tempfile::TempDir;
|
|
||||||
use tokio::net::UnixListener;
|
|
||||||
use tokio::sync::mpsc;
|
|
||||||
use tokio::task::JoinHandle;
|
|
||||||
use worker::runtime::dir::{RuntimeDir, SpawnedWorkerRecord};
|
|
||||||
use worker::runtime::worker_allocation::{self, LockFileGuard};
|
|
||||||
use worker::spawn::comm_tools::{read_worker_output_tool, send_to_worker_tool, stop_worker_tool};
|
|
||||||
use worker::spawn::registry::SpawnedWorkerRegistry;
|
|
||||||
|
|
||||||
/// Serialises env-mutating tests. The test harness runs tasks across
|
|
||||||
/// threads, and `YOI_RUNTIME_DIR` is a process-wide resource.
|
|
||||||
static ENV_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
|
|
||||||
|
|
||||||
/// Take `ENV_LOCK` and clear any env vars that would outrank
|
|
||||||
/// `YOI_RUNTIME_DIR` in `paths::runtime_dir` resolution; restore
|
|
||||||
/// previous values on drop.
|
|
||||||
struct EnvGuard {
|
|
||||||
prev_home: Option<String>,
|
|
||||||
prev_xdg: Option<String>,
|
|
||||||
_lock: std::sync::MutexGuard<'static, ()>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl EnvGuard {
|
|
||||||
fn acquire() -> Self {
|
|
||||||
let lock = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
|
||||||
let prev_home = std::env::var("YOI_HOME").ok();
|
|
||||||
let prev_xdg = std::env::var("XDG_RUNTIME_DIR").ok();
|
|
||||||
unsafe {
|
|
||||||
std::env::remove_var("YOI_HOME");
|
|
||||||
std::env::remove_var("XDG_RUNTIME_DIR");
|
|
||||||
}
|
|
||||||
Self {
|
|
||||||
prev_home,
|
|
||||||
prev_xdg,
|
|
||||||
_lock: lock,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for EnvGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
unsafe {
|
|
||||||
match &self.prev_home {
|
|
||||||
Some(v) => std::env::set_var("YOI_HOME", v),
|
|
||||||
None => std::env::remove_var("YOI_HOME"),
|
|
||||||
}
|
|
||||||
match &self.prev_xdg {
|
|
||||||
Some(v) => std::env::set_var("XDG_RUNTIME_DIR", v),
|
|
||||||
None => std::env::remove_var("XDG_RUNTIME_DIR"),
|
|
||||||
}
|
|
||||||
std::env::remove_var("YOI_RUNTIME_DIR");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Create a spawner-owned `RuntimeDir` + `SpawnedWorkerRegistry` scoped to
|
|
||||||
/// a fresh tempdir. The returned `TempDir` must be kept alive by the
|
|
||||||
/// caller for the duration of the test.
|
|
||||||
async fn setup_registry() -> (TempDir, Arc<SpawnedWorkerRegistry>, Arc<RuntimeDir>) {
|
|
||||||
let tmp = TempDir::new().unwrap();
|
|
||||||
let rd = RuntimeDir::create(tmp.path(), "spawner").await.unwrap();
|
|
||||||
let rd = Arc::new(rd);
|
|
||||||
let registry = SpawnedWorkerRegistry::new(rd.clone());
|
|
||||||
(tmp, registry, rd)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Register a fake spawned-child record pointing at a given socket
|
|
||||||
/// path, with a trivial write-scope for `scope_path`. Does not touch
|
|
||||||
/// workers.json.
|
|
||||||
async fn register_child(
|
|
||||||
registry: &SpawnedWorkerRegistry,
|
|
||||||
name: &str,
|
|
||||||
socket: &Path,
|
|
||||||
scope_path: &Path,
|
|
||||||
) {
|
|
||||||
let record = SpawnedWorkerRecord {
|
|
||||||
worker_name: name.into(),
|
|
||||||
socket_path: socket.to_path_buf(),
|
|
||||||
scope_delegated: vec![ScopeRule {
|
|
||||||
target: scope_path.to_path_buf(),
|
|
||||||
permission: Permission::Write,
|
|
||||||
recursive: true,
|
|
||||||
}],
|
|
||||||
callback_address: "/dev/null".into(),
|
|
||||||
};
|
|
||||||
registry.add(record).await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Bind a Unix listener at a socket path inside the given directory.
|
|
||||||
async fn bind_mock_socket(dir: &Path, name: &str) -> (PathBuf, UnixListener) {
|
|
||||||
let socket = dir.join(format!("{name}.sock"));
|
|
||||||
let listener = UnixListener::bind(&socket).unwrap();
|
|
||||||
(socket, listener)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Minimal connect-time snapshot used by mock socket servers.
|
|
||||||
fn empty_snapshot() -> Event {
|
|
||||||
Event::Snapshot {
|
|
||||||
entries: Vec::new(),
|
|
||||||
greeting: Greeting {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
cwd: "/tmp".into(),
|
|
||||||
provider: "anthropic".into(),
|
|
||||||
model: "x".into(),
|
|
||||||
scope_summary: String::new(),
|
|
||||||
tools: Vec::new(),
|
|
||||||
context_window: 200_000,
|
|
||||||
context_tokens: 0,
|
|
||||||
},
|
|
||||||
status: protocol::WorkerStatus::Idle,
|
|
||||||
in_flight: Default::default(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Accept one connection, send the protocol's connect-time snapshot,
|
|
||||||
/// and read exactly one `Method` line from it.
|
|
||||||
/// The reader half is kept open; caller awaits the returned handle.
|
|
||||||
fn accept_one_method(listener: UnixListener) -> JoinHandle<Option<Method>> {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let (stream, _) = listener.accept().await.ok()?;
|
|
||||||
let (r, w) = stream.into_split();
|
|
||||||
let mut reader = JsonLineReader::new(r);
|
|
||||||
let mut writer = JsonLineWriter::new(w);
|
|
||||||
writer.write(&empty_snapshot()).await.ok()?;
|
|
||||||
reader.next::<Method>().await.ok().flatten()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Accept one connection, send the protocol's connect-time snapshot,
|
|
||||||
/// read one `Method`, then write `response` back. Used by `SendToWorker`
|
|
||||||
/// tests to mock the real controller's `TurnStart` acknowledgement (or
|
|
||||||
/// its `AlreadyRunning` rejection).
|
|
||||||
fn accept_method_and_respond(
|
|
||||||
listener: UnixListener,
|
|
||||||
response: Event,
|
|
||||||
) -> JoinHandle<Option<Method>> {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let (stream, _) = listener.accept().await.ok()?;
|
|
||||||
let (r, w) = stream.into_split();
|
|
||||||
let mut reader = JsonLineReader::new(r);
|
|
||||||
let mut writer = JsonLineWriter::new(w);
|
|
||||||
writer.write(&empty_snapshot()).await.ok()?;
|
|
||||||
let method = reader.next::<Method>().await.ok().flatten();
|
|
||||||
if method.is_some() {
|
|
||||||
let _ = writer.write(&response).await;
|
|
||||||
}
|
|
||||||
method
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Pretend to be a spawned Worker whose connect-time snapshot carries a
|
|
||||||
/// fixed set of assistant items. Sends `Event::Snapshot` immediately on
|
|
||||||
/// every accept — the real Worker does the same, so `ReadWorkerOutput`'s
|
|
||||||
/// `fetch_history` just consumes the first non-Alert event.
|
|
||||||
fn serve_history(listener: UnixListener, items: Vec<Item>) -> JoinHandle<()> {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
loop {
|
|
||||||
let Ok((stream, _)) = listener.accept().await else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
let (_r, w) = stream.into_split();
|
|
||||||
let mut writer = JsonLineWriter::new(w);
|
|
||||||
let entries: Vec<serde_json::Value> = items
|
|
||||||
.iter()
|
|
||||||
.map(|item| {
|
|
||||||
let entry = session_store::LogEntry::AssistantItem {
|
|
||||||
ts: 0,
|
|
||||||
item: session_store::LoggedItem::from(item),
|
|
||||||
};
|
|
||||||
serde_json::to_value(&entry).unwrap()
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
let event = Event::Snapshot {
|
|
||||||
entries,
|
|
||||||
greeting: Greeting {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
cwd: "/tmp".into(),
|
|
||||||
provider: "anthropic".into(),
|
|
||||||
model: "x".into(),
|
|
||||||
scope_summary: String::new(),
|
|
||||||
tools: Vec::new(),
|
|
||||||
context_window: 200_000,
|
|
||||||
context_tokens: 0,
|
|
||||||
},
|
|
||||||
status: protocol::WorkerStatus::Idle,
|
|
||||||
in_flight: Default::default(),
|
|
||||||
};
|
|
||||||
let _ = writer.write(&event).await;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn serve_worker_methods(listener: UnixListener) -> mpsc::Receiver<Method> {
|
|
||||||
let (tx, rx) = mpsc::channel(8);
|
|
||||||
tokio::spawn(async move {
|
|
||||||
loop {
|
|
||||||
let Ok((stream, _)) = listener.accept().await else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
let (r, w) = stream.into_split();
|
|
||||||
let mut reader = JsonLineReader::new(r);
|
|
||||||
let mut writer = JsonLineWriter::new(w);
|
|
||||||
if writer.write(&empty_snapshot()).await.is_err() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let Some(method) = reader.next::<Method>().await.ok().flatten() else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let is_shutdown = matches!(method, Method::Shutdown);
|
|
||||||
if matches!(method, Method::Run { .. }) {
|
|
||||||
let _ = writer.write(&Event::TurnStart { turn: 1 }).await;
|
|
||||||
}
|
|
||||||
if tx.send(method).await.is_err() || is_shutdown {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
rx
|
|
||||||
}
|
|
||||||
|
|
||||||
fn assistant(text: &str) -> Item {
|
|
||||||
Item::Message {
|
|
||||||
id: None,
|
|
||||||
role: Role::Assistant,
|
|
||||||
content: vec![ContentPart::Text { text: text.into() }],
|
|
||||||
status: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// SendToWorker
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn send_to_worker_delivers_run_method() {
|
|
||||||
let (tmp, registry, _rd) = setup_registry().await;
|
|
||||||
let (socket, listener) = bind_mock_socket(tmp.path(), "child").await;
|
|
||||||
// Mock the controller's accept path: after reading the method,
|
|
||||||
// ack with `TurnStart` so `SendToWorker`'s confirmation loop succeeds.
|
|
||||||
let received = accept_method_and_respond(listener, Event::TurnStart { turn: 1 });
|
|
||||||
register_child(®istry, "child", &socket, tmp.path()).await;
|
|
||||||
|
|
||||||
let def = send_to_worker_tool(registry);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
let input = json!({ "name": "child", "message": "hello there" }).to_string();
|
|
||||||
let output: ToolOutput = tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
assert!(
|
|
||||||
output.summary.contains("child"),
|
|
||||||
"summary: {}",
|
|
||||||
output.summary
|
|
||||||
);
|
|
||||||
|
|
||||||
let method = received.await.unwrap().expect("expected a method");
|
|
||||||
match method {
|
|
||||||
Method::Run { input } => match input.as_slice() {
|
|
||||||
[protocol::Segment::Text { content }] => assert_eq!(content, "hello there"),
|
|
||||||
other => panic!("expected single Text segment, got {other:?}"),
|
|
||||||
},
|
|
||||||
other => panic!("expected Run, got {other:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn send_to_worker_errors_on_unknown_worker() {
|
|
||||||
let (_tmp, registry, _rd) = setup_registry().await;
|
|
||||||
let def = send_to_worker_tool(registry);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
let input = json!({ "name": "nope", "message": "hi" }).to_string();
|
|
||||||
let err = tool.execute(&input, Default::default()).await.unwrap_err();
|
|
||||||
assert!(err.to_string().contains("no spawned worker"), "{err}");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn send_to_worker_errors_when_worker_already_running() {
|
|
||||||
let (tmp, registry, _rd) = setup_registry().await;
|
|
||||||
let (socket, listener) = bind_mock_socket(tmp.path(), "child").await;
|
|
||||||
// Respond with the same `Error { AlreadyRunning }` that the real
|
|
||||||
// controller emits when `Method::Run` arrives during RUNNING.
|
|
||||||
let received = accept_method_and_respond(
|
|
||||||
listener,
|
|
||||||
Event::Error {
|
|
||||||
code: ErrorCode::AlreadyRunning,
|
|
||||||
message: "Worker is already executing a turn".into(),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
register_child(®istry, "child", &socket, tmp.path()).await;
|
|
||||||
|
|
||||||
let def = send_to_worker_tool(registry);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
let input = json!({ "name": "child", "message": "hi" }).to_string();
|
|
||||||
let err = tool.execute(&input, Default::default()).await.unwrap_err();
|
|
||||||
assert!(
|
|
||||||
err.to_string().contains("already running"),
|
|
||||||
"expected AlreadyRunning wording: {err}"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Ensure the listener was in fact hit with a Method::Run before the
|
|
||||||
// rejection path fired — otherwise we'd be asserting on an error
|
|
||||||
// that came from a connect failure.
|
|
||||||
let method = received.await.unwrap().expect("expected a method");
|
|
||||||
assert!(matches!(method, Method::Run { .. }));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// ReadWorkerOutput
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn read_worker_output_returns_new_assistant_text_then_empty_on_second_call() {
|
|
||||||
let (tmp, registry, _rd) = setup_registry().await;
|
|
||||||
let (socket, listener) = bind_mock_socket(tmp.path(), "child").await;
|
|
||||||
register_child(®istry, "child", &socket, tmp.path()).await;
|
|
||||||
|
|
||||||
let items = vec![
|
|
||||||
Item::user_message("hello"),
|
|
||||||
assistant("hi back"),
|
|
||||||
assistant("still working"),
|
|
||||||
];
|
|
||||||
let _server = serve_history(listener, items);
|
|
||||||
|
|
||||||
let def = read_worker_output_tool(registry);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
let input = json!({ "name": "child" }).to_string();
|
|
||||||
|
|
||||||
let first: ToolOutput = tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
let body = first.content.expect("first read should have content");
|
|
||||||
assert!(body.contains("hi back"), "body: {body}");
|
|
||||||
assert!(body.contains("still working"), "body: {body}");
|
|
||||||
|
|
||||||
// Cursor now points past all items — second call returns no new text.
|
|
||||||
let second: ToolOutput = tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
assert!(
|
|
||||||
second.content.is_none(),
|
|
||||||
"unexpected content: {:?}",
|
|
||||||
second.content
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
second.summary.contains("no new assistant text"),
|
|
||||||
"summary: {}",
|
|
||||||
second.summary
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn read_worker_output_reports_stopped_on_dead_socket() {
|
|
||||||
let (tmp, registry, _rd) = setup_registry().await;
|
|
||||||
// Register a record pointing at a socket that nobody is listening
|
|
||||||
// on. Connect must fail → tool reports "stopped".
|
|
||||||
let dead_socket = tmp.path().join("dead.sock");
|
|
||||||
register_child(®istry, "child", &dead_socket, tmp.path()).await;
|
|
||||||
|
|
||||||
let def = read_worker_output_tool(registry);
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
let input = json!({ "name": "child" }).to_string();
|
|
||||||
let output: ToolOutput = tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
assert!(output.summary.contains("stopped"), "{}", output.summary);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// StopWorker
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn stop_worker_sends_shutdown_and_releases_scope() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
let tmp = TempDir::new().unwrap();
|
|
||||||
let store_tmp = TempDir::new().unwrap();
|
|
||||||
let store = CombinedStore::new(
|
|
||||||
FsStore::new(store_tmp.path()).unwrap(),
|
|
||||||
FsWorkerStore::new(store_tmp.path().join("pods")).unwrap(),
|
|
||||||
);
|
|
||||||
let rd = Arc::new(RuntimeDir::create(tmp.path(), "spawner").await.unwrap());
|
|
||||||
let parent_scope = SharedScope::new(
|
|
||||||
Scope::writable(tmp.path())
|
|
||||||
.unwrap()
|
|
||||||
.with_added_deny_rules([ScopeRule {
|
|
||||||
target: tmp.path().to_path_buf(),
|
|
||||||
permission: Permission::Write,
|
|
||||||
recursive: true,
|
|
||||||
}])
|
|
||||||
.unwrap(),
|
|
||||||
);
|
|
||||||
unsafe {
|
|
||||||
std::env::set_var("YOI_RUNTIME_DIR", tmp.path());
|
|
||||||
}
|
|
||||||
let lock_path = tmp.path().join("workers.json");
|
|
||||||
|
|
||||||
// Seed workers.json with a restored top-level `spawner` allocation whose
|
|
||||||
// scope_deny contains the delegated child path plus the live child
|
|
||||||
// allocation — mimics a parent resumed after SpawnWorker.
|
|
||||||
{
|
|
||||||
let mut g = LockFileGuard::open(&lock_path).unwrap();
|
|
||||||
let rule = ScopeRule {
|
|
||||||
target: tmp.path().to_path_buf(),
|
|
||||||
permission: Permission::Write,
|
|
||||||
recursive: true,
|
|
||||||
};
|
|
||||||
worker_allocation::register_worker_with_deny(
|
|
||||||
&mut g,
|
|
||||||
"spawner".into(),
|
|
||||||
std::process::id(),
|
|
||||||
"/tmp/spawner.sock".into(),
|
|
||||||
vec![rule.clone()],
|
|
||||||
vec![rule.clone()],
|
|
||||||
session_store::new_segment_id(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
worker_allocation::register_worker(
|
|
||||||
&mut g,
|
|
||||||
"child".into(),
|
|
||||||
std::process::id(),
|
|
||||||
"/tmp/child.sock".into(),
|
|
||||||
vec![rule],
|
|
||||||
session_store::new_segment_id(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
let loaded = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim(
|
|
||||||
rd.clone(),
|
|
||||||
store.clone(),
|
|
||||||
"spawner".into(),
|
|
||||||
Some(parent_scope.clone()),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let registry = loaded.registry;
|
|
||||||
|
|
||||||
let (socket, listener) = bind_mock_socket(tmp.path(), "child").await;
|
|
||||||
let received = accept_one_method(listener);
|
|
||||||
register_child(®istry, "child", &socket, tmp.path()).await;
|
|
||||||
|
|
||||||
let def = stop_worker_tool(registry.clone());
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
let input = json!({ "name": "child" }).to_string();
|
|
||||||
let output: ToolOutput = tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
assert!(output.summary.contains("stopped"), "{}", output.summary);
|
|
||||||
|
|
||||||
// The child got a Shutdown.
|
|
||||||
let method = received.await.unwrap().expect("expected shutdown");
|
|
||||||
assert!(matches!(method, Method::Shutdown));
|
|
||||||
|
|
||||||
// Allocation for `child` is gone; `spawner` remains and its restored
|
|
||||||
// dynamic deny layer has been reclaimed.
|
|
||||||
{
|
|
||||||
let g = LockFileGuard::open(&lock_path).unwrap();
|
|
||||||
assert!(g.data().find("child").is_none(), "child still allocated");
|
|
||||||
let spawner = g.data().find("spawner").expect("spawner missing");
|
|
||||||
assert!(spawner.scope_deny.is_empty(), "deny not reclaimed");
|
|
||||||
}
|
|
||||||
assert_eq!(
|
|
||||||
parent_scope
|
|
||||||
.snapshot()
|
|
||||||
.permission_at(&tmp.path().join("file.txt")),
|
|
||||||
Some(Permission::Write)
|
|
||||||
);
|
|
||||||
|
|
||||||
// spawned_workers.json now lists zero children.
|
|
||||||
let spawned = rd.path().join("spawned_workers.json");
|
|
||||||
let contents = std::fs::read_to_string(&spawned).unwrap();
|
|
||||||
let records: Vec<SpawnedWorkerRecord> = serde_json::from_str(&contents).unwrap();
|
|
||||||
assert!(records.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn stop_worker_succeeds_even_when_child_unreachable() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
let (tmp, registry, _rd) = setup_registry().await;
|
|
||||||
unsafe {
|
|
||||||
std::env::set_var("YOI_RUNTIME_DIR", tmp.path());
|
|
||||||
}
|
|
||||||
|
|
||||||
// No live listener — socket never bound. Registered record points
|
|
||||||
// at a dead path. StopWorker should still clean up local bookkeeping.
|
|
||||||
let dead_socket = tmp.path().join("dead.sock");
|
|
||||||
register_child(®istry, "child", &dead_socket, tmp.path()).await;
|
|
||||||
|
|
||||||
let def = stop_worker_tool(registry.clone());
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
let input = json!({ "name": "child" }).to_string();
|
|
||||||
let output: ToolOutput = tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
assert!(output.summary.contains("stopped"), "{}", output.summary);
|
|
||||||
|
|
||||||
// Registry no longer knows about the child.
|
|
||||||
assert!(registry.get("child").await.is_none());
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Persistence / restore
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn restored_registry_uses_worker_state_without_runtime_file() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
let runtime_tmp = TempDir::new().unwrap();
|
|
||||||
let store_tmp = TempDir::new().unwrap();
|
|
||||||
let store = CombinedStore::new(
|
|
||||||
FsStore::new(store_tmp.path()).unwrap(),
|
|
||||||
FsWorkerStore::new(store_tmp.path().join("pods")).unwrap(),
|
|
||||||
);
|
|
||||||
unsafe {
|
|
||||||
std::env::set_var("YOI_RUNTIME_DIR", runtime_tmp.path());
|
|
||||||
}
|
|
||||||
|
|
||||||
let rd = Arc::new(
|
|
||||||
RuntimeDir::create(runtime_tmp.path(), "spawner")
|
|
||||||
.await
|
|
||||||
.unwrap(),
|
|
||||||
);
|
|
||||||
let registry = SpawnedWorkerRegistry::load_from_worker_state(
|
|
||||||
rd.clone(),
|
|
||||||
store.clone(),
|
|
||||||
"spawner".to_string(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let (socket, listener) = bind_mock_socket(runtime_tmp.path(), "child").await;
|
|
||||||
let mut received = serve_worker_methods(listener);
|
|
||||||
register_child(®istry, "child", &socket, runtime_tmp.path()).await;
|
|
||||||
|
|
||||||
std::fs::remove_file(rd.path().join("spawned_workers.json")).unwrap();
|
|
||||||
|
|
||||||
let restored = SpawnedWorkerRegistry::load_from_worker_state(
|
|
||||||
rd.clone(),
|
|
||||||
store.clone(),
|
|
||||||
"spawner".to_string(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let def = send_to_worker_tool(restored.clone());
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
let input = json!({ "name": "child", "message": "after restart" }).to_string();
|
|
||||||
tool.execute(&input, Default::default()).await.unwrap();
|
|
||||||
match received.recv().await.expect("expected Run") {
|
|
||||||
Method::Run { input } => match input.as_slice() {
|
|
||||||
[protocol::Segment::Text { content }] => assert_eq!(content, "after restart"),
|
|
||||||
other => panic!("expected single Text segment, got {other:?}"),
|
|
||||||
},
|
|
||||||
other => panic!("expected Run, got {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
let def = stop_worker_tool(restored.clone());
|
|
||||||
let (_meta, tool) = def();
|
|
||||||
tool.execute(&json!({ "name": "child" }).to_string(), Default::default())
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(matches!(
|
|
||||||
received.recv().await.expect("expected Shutdown"),
|
|
||||||
Method::Shutdown
|
|
||||||
));
|
|
||||||
assert!(restored.get("child").await.is_none());
|
|
||||||
|
|
||||||
let metadata = store
|
|
||||||
.read_by_name("spawner")
|
|
||||||
.unwrap()
|
|
||||||
.expect("spawner metadata should remain");
|
|
||||||
assert!(metadata.spawned_children.is_empty());
|
|
||||||
assert_eq!(metadata.reclaimed_children.len(), 1);
|
|
||||||
assert_eq!(metadata.reclaimed_children[0].worker_name, "child");
|
|
||||||
let runtime_contents = std::fs::read_to_string(rd.path().join("spawned_workers.json")).unwrap();
|
|
||||||
let runtime_records: Vec<SpawnedWorkerRecord> =
|
|
||||||
serde_json::from_str(&runtime_contents).unwrap();
|
|
||||||
assert!(runtime_records.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn load_from_worker_state_prunes_runtime_children_and_reclaims_durable_delegation() {
|
|
||||||
let runtime_tmp = TempDir::new().unwrap();
|
|
||||||
let store_tmp = TempDir::new().unwrap();
|
|
||||||
let store = CombinedStore::new(
|
|
||||||
FsStore::new(store_tmp.path()).unwrap(),
|
|
||||||
FsWorkerStore::new(store_tmp.path().join("pods")).unwrap(),
|
|
||||||
);
|
|
||||||
let rd = Arc::new(
|
|
||||||
RuntimeDir::create(runtime_tmp.path(), "spawner")
|
|
||||||
.await
|
|
||||||
.unwrap(),
|
|
||||||
);
|
|
||||||
let registry = SpawnedWorkerRegistry::load_from_worker_state(
|
|
||||||
rd.clone(),
|
|
||||||
store.clone(),
|
|
||||||
"spawner".to_string(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let (live_socket, listener) = bind_mock_socket(runtime_tmp.path(), "alive").await;
|
|
||||||
let _server = serve_worker_methods(listener);
|
|
||||||
register_child(®istry, "alive", &live_socket, runtime_tmp.path()).await;
|
|
||||||
register_child(
|
|
||||||
®istry,
|
|
||||||
"missing",
|
|
||||||
&runtime_tmp.path().join("missing.sock"),
|
|
||||||
runtime_tmp.path(),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let restored = SpawnedWorkerRegistry::load_from_worker_state(
|
|
||||||
rd.clone(),
|
|
||||||
store.clone(),
|
|
||||||
"spawner".to_string(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert!(restored.get("alive").await.is_some());
|
|
||||||
assert!(restored.get("missing").await.is_none());
|
|
||||||
let metadata = store
|
|
||||||
.read_by_name("spawner")
|
|
||||||
.unwrap()
|
|
||||||
.expect("spawner metadata should be written");
|
|
||||||
assert_eq!(metadata.spawned_children.len(), 1);
|
|
||||||
assert_eq!(metadata.spawned_children[0].worker_name, "alive");
|
|
||||||
assert_eq!(metadata.reclaimed_children.len(), 1);
|
|
||||||
assert_eq!(metadata.reclaimed_children[0].worker_name, "missing");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn load_from_worker_state_reclaims_missing_child_scope_and_records_history() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
let runtime_tmp = TempDir::new().unwrap();
|
|
||||||
let store_tmp = TempDir::new().unwrap();
|
|
||||||
let store = CombinedStore::new(
|
|
||||||
FsStore::new(store_tmp.path()).unwrap(),
|
|
||||||
FsWorkerStore::new(store_tmp.path().join("pods")).unwrap(),
|
|
||||||
);
|
|
||||||
unsafe {
|
|
||||||
std::env::set_var("YOI_RUNTIME_DIR", runtime_tmp.path());
|
|
||||||
}
|
|
||||||
let rd = Arc::new(
|
|
||||||
RuntimeDir::create(runtime_tmp.path(), "spawner")
|
|
||||||
.await
|
|
||||||
.unwrap(),
|
|
||||||
);
|
|
||||||
let missing_rule = ScopeRule {
|
|
||||||
target: runtime_tmp.path().to_path_buf(),
|
|
||||||
permission: Permission::Write,
|
|
||||||
recursive: true,
|
|
||||||
};
|
|
||||||
|
|
||||||
{
|
|
||||||
let mut g = LockFileGuard::open(&runtime_tmp.path().join("workers.json")).unwrap();
|
|
||||||
worker_allocation::register_worker_with_deny(
|
|
||||||
&mut g,
|
|
||||||
"spawner".into(),
|
|
||||||
std::process::id(),
|
|
||||||
"/tmp/spawner.sock".into(),
|
|
||||||
vec![missing_rule.clone()],
|
|
||||||
vec![missing_rule.clone()],
|
|
||||||
session_store::new_segment_id(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
let parent_scope = SharedScope::new(
|
|
||||||
Scope::writable(runtime_tmp.path())
|
|
||||||
.unwrap()
|
|
||||||
.with_added_deny_rules([missing_rule.clone()])
|
|
||||||
.unwrap(),
|
|
||||||
);
|
|
||||||
let seed =
|
|
||||||
SpawnedWorkerRegistry::load_from_worker_state(rd.clone(), store.clone(), "spawner".into())
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
seed.add(SpawnedWorkerRecord {
|
|
||||||
worker_name: "missing".into(),
|
|
||||||
socket_path: runtime_tmp.path().join("missing.sock"),
|
|
||||||
scope_delegated: vec![missing_rule.clone()],
|
|
||||||
callback_address: "/dev/null".into(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let loaded = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim(
|
|
||||||
rd.clone(),
|
|
||||||
store.clone(),
|
|
||||||
"spawner".into(),
|
|
||||||
Some(parent_scope.clone()),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(loaded.reclaimed_unreachable);
|
|
||||||
assert!(loaded.registry.get("missing").await.is_none());
|
|
||||||
assert_eq!(
|
|
||||||
parent_scope
|
|
||||||
.snapshot()
|
|
||||||
.permission_at(&runtime_tmp.path().join("file.txt")),
|
|
||||||
Some(Permission::Write)
|
|
||||||
);
|
|
||||||
|
|
||||||
let g = LockFileGuard::open(&runtime_tmp.path().join("workers.json")).unwrap();
|
|
||||||
assert!(g.data().find("missing").is_none());
|
|
||||||
assert!(g.data().find("spawner").unwrap().scope_deny.is_empty());
|
|
||||||
let metadata = store
|
|
||||||
.read_by_name("spawner")
|
|
||||||
.unwrap()
|
|
||||||
.expect("spawner metadata should remain");
|
|
||||||
assert!(metadata.spawned_children.is_empty());
|
|
||||||
assert_eq!(metadata.reclaimed_children.len(), 1);
|
|
||||||
assert_eq!(metadata.reclaimed_children[0].worker_name, "missing");
|
|
||||||
let runtime_contents = std::fs::read_to_string(rd.path().join("spawned_workers.json")).unwrap();
|
|
||||||
let runtime_records: Vec<SpawnedWorkerRecord> =
|
|
||||||
serde_json::from_str(&runtime_contents).unwrap();
|
|
||||||
assert!(runtime_records.is_empty());
|
|
||||||
}
|
|
||||||
@@ -1,423 +1,45 @@
|
|||||||
//! Integration tests for the `WorkerEvent` send / receive primitive.
|
//! Legacy process callback events are diagnostics only after Internal SubWorker migration.
|
||||||
//!
|
|
||||||
//! These tests drive `worker_events::fire_and_forget` and
|
|
||||||
//! `worker_events::apply_event_side_effects` directly — the full
|
|
||||||
//! Controller wiring is exercised by the existing controller /
|
|
||||||
//! spawn-worker tests, which rely on the same primitives.
|
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::sync::Arc;
|
||||||
use std::sync::{Arc, LazyLock, Mutex};
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
use protocol::{Permission, ScopeRule, WorkerEvent};
|
||||||
use protocol::{Event, Greeting, Method, Permission, ScopeRule, WorkerEvent, WorkerStatus};
|
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
use tokio::net::UnixListener;
|
use worker::ipc::event::{apply_event_side_effects, render_event};
|
||||||
use worker::ipc::event::{apply_event_side_effects, fire_and_forget, render_event};
|
use worker::runtime::dir::RuntimeDir;
|
||||||
use worker::runtime::dir::{RuntimeDir, SpawnedWorkerRecord};
|
|
||||||
use worker::runtime::worker_allocation::{self, LockFileGuard};
|
|
||||||
use worker::spawn::registry::SpawnedWorkerRegistry;
|
use worker::spawn::registry::SpawnedWorkerRegistry;
|
||||||
|
|
||||||
/// Serialises tests that mutate `YOI_RUNTIME_DIR`.
|
|
||||||
static ENV_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
|
|
||||||
|
|
||||||
/// Take `ENV_LOCK` and clear any env vars that would outrank
|
|
||||||
/// `YOI_RUNTIME_DIR`; restore previous values on drop.
|
|
||||||
struct EnvGuard {
|
|
||||||
prev_home: Option<String>,
|
|
||||||
prev_xdg: Option<String>,
|
|
||||||
_lock: std::sync::MutexGuard<'static, ()>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl EnvGuard {
|
|
||||||
fn acquire() -> Self {
|
|
||||||
let lock = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
|
||||||
let prev_home = std::env::var("YOI_HOME").ok();
|
|
||||||
let prev_xdg = std::env::var("XDG_RUNTIME_DIR").ok();
|
|
||||||
unsafe {
|
|
||||||
std::env::remove_var("YOI_HOME");
|
|
||||||
std::env::remove_var("XDG_RUNTIME_DIR");
|
|
||||||
}
|
|
||||||
Self {
|
|
||||||
prev_home,
|
|
||||||
prev_xdg,
|
|
||||||
_lock: lock,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for EnvGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
unsafe {
|
|
||||||
match &self.prev_home {
|
|
||||||
Some(v) => std::env::set_var("YOI_HOME", v),
|
|
||||||
None => std::env::remove_var("YOI_HOME"),
|
|
||||||
}
|
|
||||||
match &self.prev_xdg {
|
|
||||||
Some(v) => std::env::set_var("XDG_RUNTIME_DIR", v),
|
|
||||||
None => std::env::remove_var("XDG_RUNTIME_DIR"),
|
|
||||||
}
|
|
||||||
std::env::remove_var("YOI_RUNTIME_DIR");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Point `YOI_RUNTIME_DIR` at `dir`. The worker-allocation then lives at
|
|
||||||
/// `<dir>/workers.json` and Worker runtime sub-dirs at `<dir>/{worker_name}/`.
|
|
||||||
fn set_runtime_dir(dir: &std::path::Path) {
|
|
||||||
unsafe {
|
|
||||||
std::env::set_var("YOI_RUNTIME_DIR", dir);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn clear_runtime_dir() {
|
|
||||||
unsafe {
|
|
||||||
std::env::remove_var("YOI_RUNTIME_DIR");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Minimal connect-time snapshot used by mock parent sockets.
|
|
||||||
fn empty_snapshot() -> Event {
|
|
||||||
Event::Snapshot {
|
|
||||||
entries: Vec::new(),
|
|
||||||
greeting: Greeting {
|
|
||||||
worker_name: "parent".into(),
|
|
||||||
cwd: "/tmp".into(),
|
|
||||||
provider: "test".into(),
|
|
||||||
model: "test".into(),
|
|
||||||
scope_summary: String::new(),
|
|
||||||
tools: Vec::new(),
|
|
||||||
context_window: 200_000,
|
|
||||||
context_tokens: 0,
|
|
||||||
},
|
|
||||||
status: WorkerStatus::Idle,
|
|
||||||
in_flight: Default::default(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Accept a single connection, send the protocol's connect-time snapshot,
|
|
||||||
/// read one `Method`, and return it.
|
|
||||||
fn accept_one_method(listener: UnixListener) -> tokio::task::JoinHandle<Option<Method>> {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let (stream, _) = listener.accept().await.ok()?;
|
|
||||||
let (reader, writer) = stream.into_split();
|
|
||||||
let mut w = JsonLineWriter::new(writer);
|
|
||||||
w.write(&empty_snapshot()).await.ok()?;
|
|
||||||
let mut r = JsonLineReader::new(reader);
|
|
||||||
r.next::<Method>().await.ok().flatten()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn render_event_all_variants_mention_worker_name() {
|
fn render_event_keeps_bounded_legacy_diagnostics() {
|
||||||
let t1 = render_event(&WorkerEvent::TurnEnded {
|
let rendered = render_event(&WorkerEvent::Errored {
|
||||||
worker_name: "alpha".into(),
|
worker_name: "legacy-child".into(),
|
||||||
});
|
|
||||||
assert!(t1.contains("alpha"), "{t1}");
|
|
||||||
|
|
||||||
let t2 = render_event(&WorkerEvent::Errored {
|
|
||||||
worker_name: "bravo".into(),
|
|
||||||
message: "boom".into(),
|
message: "boom".into(),
|
||||||
});
|
});
|
||||||
assert!(t2.contains("bravo") && t2.contains("boom"), "{t2}");
|
assert!(rendered.contains("legacy-child"));
|
||||||
|
assert!(rendered.contains("boom"));
|
||||||
let t3 = render_event(&WorkerEvent::ShutDown {
|
|
||||||
worker_name: "charlie".into(),
|
|
||||||
});
|
|
||||||
assert!(t3.contains("charlie"), "{t3}");
|
|
||||||
|
|
||||||
let t4 = render_event(&WorkerEvent::ScopeSubDelegated {
|
|
||||||
parent_worker: "delta".into(),
|
|
||||||
sub_worker: "echo".into(),
|
|
||||||
sub_socket: "/tmp/sock".into(),
|
|
||||||
scope: vec![],
|
|
||||||
});
|
|
||||||
assert!(t4.contains("delta") && t4.contains("echo"), "{t4}");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn fire_and_forget_delivers_worker_event_to_listener() {
|
async fn legacy_callback_cannot_register_process_subworker_authority() {
|
||||||
let dir = TempDir::new().unwrap();
|
|
||||||
let socket_path = dir.path().join("parent.sock");
|
|
||||||
let listener = UnixListener::bind(&socket_path).unwrap();
|
|
||||||
let received = accept_one_method(listener);
|
|
||||||
|
|
||||||
fire_and_forget(
|
|
||||||
Some(socket_path.clone()),
|
|
||||||
WorkerEvent::TurnEnded {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
let method = tokio::time::timeout(Duration::from_secs(2), received)
|
|
||||||
.await
|
|
||||||
.expect("send timed out")
|
|
||||||
.unwrap()
|
|
||||||
.expect("no method received");
|
|
||||||
match method {
|
|
||||||
Method::WorkerEvent(WorkerEvent::TurnEnded { worker_name }) => {
|
|
||||||
assert_eq!(worker_name, "child")
|
|
||||||
}
|
|
||||||
other => panic!("expected TurnEnded, got {other:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn fire_and_forget_with_none_socket_is_noop() {
|
|
||||||
// Nothing binds and nothing listens; the call must not panic and
|
|
||||||
// must not leak a task that never completes.
|
|
||||||
fire_and_forget(
|
|
||||||
None,
|
|
||||||
WorkerEvent::ShutDown {
|
|
||||||
worker_name: "x".into(),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
// Yield once so any accidentally-spawned task would surface.
|
|
||||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Build a registry backed by a fresh runtime dir.
|
|
||||||
async fn fresh_registry(
|
|
||||||
runtime_base: &std::path::Path,
|
|
||||||
worker_name: &str,
|
|
||||||
) -> Arc<SpawnedWorkerRegistry> {
|
|
||||||
let rd = RuntimeDir::create(runtime_base, worker_name).await.unwrap();
|
|
||||||
SpawnedWorkerRegistry::new(Arc::new(rd))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn apply_shutdown_removes_from_registry_and_tolerates_missing() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
let scope_dir = TempDir::new().unwrap();
|
|
||||||
set_runtime_dir(scope_dir.path());
|
|
||||||
|
|
||||||
let runtime_base = TempDir::new().unwrap();
|
let runtime_base = TempDir::new().unwrap();
|
||||||
let registry = fresh_registry(runtime_base.path(), "parent").await;
|
let runtime_dir = Arc::new(
|
||||||
|
RuntimeDir::create(runtime_base.path(), "parent")
|
||||||
// Seed a child record; then ShutDown for it should remove it.
|
|
||||||
registry
|
|
||||||
.add(SpawnedWorkerRecord {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
socket_path: "/tmp/child.sock".into(),
|
|
||||||
scope_delegated: vec![],
|
|
||||||
callback_address: "/tmp/parent.sock".into(),
|
|
||||||
})
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap(),
|
||||||
|
);
|
||||||
let event = WorkerEvent::ShutDown {
|
let registry = SpawnedWorkerRegistry::new(runtime_dir.clone());
|
||||||
worker_name: "child".into(),
|
let scope_root = TempDir::new().unwrap();
|
||||||
};
|
|
||||||
apply_event_side_effects(&event, ®istry, "parent", &None).await;
|
|
||||||
assert!(registry.get("child").await.is_none());
|
|
||||||
|
|
||||||
// Second ShutDown for the same (now-missing) child must be a no-op,
|
|
||||||
// not an error — this is the idempotency guarantee for out-of-order
|
|
||||||
// delivery.
|
|
||||||
apply_event_side_effects(&event, ®istry, "parent", &None).await;
|
|
||||||
assert!(registry.get("child").await.is_none());
|
|
||||||
|
|
||||||
clear_runtime_dir();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn apply_scope_sub_delegated_adds_grandchild_then_duplicate_is_noop() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
let scope_dir = TempDir::new().unwrap();
|
|
||||||
set_runtime_dir(scope_dir.path());
|
|
||||||
|
|
||||||
let runtime_base = TempDir::new().unwrap();
|
|
||||||
let registry = fresh_registry(runtime_base.path(), "grandparent").await;
|
|
||||||
|
|
||||||
// Seed the intermediate child so callback_address lookup succeeds.
|
|
||||||
registry
|
|
||||||
.add(SpawnedWorkerRecord {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
socket_path: "/tmp/child.sock".into(),
|
|
||||||
scope_delegated: vec![],
|
|
||||||
callback_address: "/tmp/grandparent.sock".into(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let event = WorkerEvent::ScopeSubDelegated {
|
let event = WorkerEvent::ScopeSubDelegated {
|
||||||
parent_worker: "child".into(),
|
parent_worker: "legacy-parent".into(),
|
||||||
sub_worker: "grandchild".into(),
|
sub_worker: "legacy-child".into(),
|
||||||
sub_socket: "/tmp/grandchild.sock".into(),
|
sub_socket: "/tmp/legacy-child.sock".into(),
|
||||||
scope: vec![ScopeRule {
|
scope: vec![ScopeRule {
|
||||||
target: scope_dir.path().to_path_buf(),
|
target: scope_root.path().to_path_buf(),
|
||||||
permission: Permission::Write,
|
permission: Permission::Write,
|
||||||
recursive: true,
|
recursive: true,
|
||||||
}],
|
}],
|
||||||
};
|
};
|
||||||
|
|
||||||
apply_event_side_effects(&event, ®istry, "grandparent", &None).await;
|
apply_event_side_effects(&event, ®istry, "parent", &None).await;
|
||||||
let gc = registry
|
|
||||||
.get("grandchild")
|
|
||||||
.await
|
|
||||||
.expect("grandchild missing after ScopeSubDelegated");
|
|
||||||
assert_eq!(gc.socket_path, PathBuf::from("/tmp/grandchild.sock"));
|
|
||||||
assert_eq!(gc.callback_address, PathBuf::from("/tmp/child.sock"));
|
|
||||||
|
|
||||||
// Duplicate delivery must not error and must not overwrite.
|
assert!(!runtime_dir.path().join("spawned_workers.json").exists());
|
||||||
apply_event_side_effects(&event, ®istry, "grandparent", &None).await;
|
|
||||||
let gc2 = registry.get("grandchild").await.unwrap();
|
|
||||||
assert_eq!(gc2.socket_path, PathBuf::from("/tmp/grandchild.sock"));
|
|
||||||
|
|
||||||
clear_runtime_dir();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn apply_scope_sub_delegated_reemits_to_own_parent() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
let scope_dir = TempDir::new().unwrap();
|
|
||||||
set_runtime_dir(scope_dir.path());
|
|
||||||
|
|
||||||
let runtime_base = TempDir::new().unwrap();
|
|
||||||
let registry = fresh_registry(runtime_base.path(), "B").await;
|
|
||||||
|
|
||||||
// Bind a listener at "A's" socket so we can watch the re-emission
|
|
||||||
// climb one level up the tree.
|
|
||||||
let sock_dir = TempDir::new().unwrap();
|
|
||||||
let a_socket = sock_dir.path().join("A.sock");
|
|
||||||
let listener = UnixListener::bind(&a_socket).unwrap();
|
|
||||||
let received = accept_one_method(listener);
|
|
||||||
|
|
||||||
// Seed the child record that the event claims spawned the grandchild.
|
|
||||||
registry
|
|
||||||
.add(SpawnedWorkerRecord {
|
|
||||||
worker_name: "C".into(),
|
|
||||||
socket_path: "/tmp/C.sock".into(),
|
|
||||||
scope_delegated: vec![],
|
|
||||||
callback_address: "/tmp/B.sock".into(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let event = WorkerEvent::ScopeSubDelegated {
|
|
||||||
parent_worker: "C".into(),
|
|
||||||
sub_worker: "D".into(),
|
|
||||||
sub_socket: "/tmp/D.sock".into(),
|
|
||||||
scope: vec![],
|
|
||||||
};
|
|
||||||
|
|
||||||
// Self is B, and B's parent socket is A's listener.
|
|
||||||
apply_event_side_effects(&event, ®istry, "B", &Some(a_socket.clone())).await;
|
|
||||||
|
|
||||||
// A must see the re-emission with parent_worker set to "B" (the
|
|
||||||
// sender from A's perspective), not "C" (the original sender's
|
|
||||||
// local view).
|
|
||||||
let method = tokio::time::timeout(Duration::from_secs(2), received)
|
|
||||||
.await
|
|
||||||
.expect("re-emission timed out")
|
|
||||||
.unwrap()
|
|
||||||
.expect("no method received on A's socket");
|
|
||||||
match method {
|
|
||||||
Method::WorkerEvent(WorkerEvent::ScopeSubDelegated {
|
|
||||||
parent_worker,
|
|
||||||
sub_worker,
|
|
||||||
..
|
|
||||||
}) => {
|
|
||||||
assert_eq!(parent_worker, "B");
|
|
||||||
assert_eq!(sub_worker, "D");
|
|
||||||
}
|
|
||||||
other => panic!("expected re-emitted ScopeSubDelegated, got {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
clear_runtime_dir();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn apply_turn_ended_and_errored_are_system_noops() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
let scope_dir = TempDir::new().unwrap();
|
|
||||||
set_runtime_dir(scope_dir.path());
|
|
||||||
|
|
||||||
let runtime_base = TempDir::new().unwrap();
|
|
||||||
let registry = fresh_registry(runtime_base.path(), "parent").await;
|
|
||||||
|
|
||||||
// Seed a child to verify it survives the no-op path.
|
|
||||||
registry
|
|
||||||
.add(SpawnedWorkerRecord {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
socket_path: "/tmp/child.sock".into(),
|
|
||||||
scope_delegated: vec![],
|
|
||||||
callback_address: "/tmp/parent.sock".into(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
apply_event_side_effects(
|
|
||||||
&WorkerEvent::TurnEnded {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
},
|
|
||||||
®istry,
|
|
||||||
"parent",
|
|
||||||
&None,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
apply_event_side_effects(
|
|
||||||
&WorkerEvent::Errored {
|
|
||||||
worker_name: "child".into(),
|
|
||||||
message: "x".into(),
|
|
||||||
},
|
|
||||||
®istry,
|
|
||||||
"parent",
|
|
||||||
&None,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
assert!(registry.get("child").await.is_some());
|
|
||||||
clear_runtime_dir();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn shutdown_releases_scope_allocation_when_present() {
|
|
||||||
let _env = EnvGuard::acquire();
|
|
||||||
let scope_dir = TempDir::new().unwrap();
|
|
||||||
let lock_path = scope_dir.path().join("workers.json");
|
|
||||||
set_runtime_dir(scope_dir.path());
|
|
||||||
|
|
||||||
// Install a top-level allocation for "kid" so ShutDown has
|
|
||||||
// something to release.
|
|
||||||
let guard = worker_allocation::install_top_level(
|
|
||||||
"kid".into(),
|
|
||||||
std::process::id(),
|
|
||||||
"/tmp/kid.sock".into(),
|
|
||||||
vec![],
|
|
||||||
session_store::new_segment_id(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
std::mem::forget(guard);
|
|
||||||
|
|
||||||
let runtime_base = TempDir::new().unwrap();
|
|
||||||
let registry = fresh_registry(runtime_base.path(), "parent").await;
|
|
||||||
registry
|
|
||||||
.add(SpawnedWorkerRecord {
|
|
||||||
worker_name: "kid".into(),
|
|
||||||
socket_path: "/tmp/kid.sock".into(),
|
|
||||||
scope_delegated: vec![],
|
|
||||||
callback_address: "/tmp/parent.sock".into(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
apply_event_side_effects(
|
|
||||||
&WorkerEvent::ShutDown {
|
|
||||||
worker_name: "kid".into(),
|
|
||||||
},
|
|
||||||
®istry,
|
|
||||||
"parent",
|
|
||||||
&None,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
// Allocation is gone from the worker-allocation.
|
|
||||||
let g = LockFileGuard::open(&lock_path).unwrap();
|
|
||||||
assert!(
|
|
||||||
g.data().find("kid").is_none(),
|
|
||||||
"ShutDown should have released the scope allocation"
|
|
||||||
);
|
|
||||||
|
|
||||||
clear_runtime_dir();
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
use crate::Error;
|
use crate::Error;
|
||||||
use crate::resource_broker::BackendResourceBroker;
|
use crate::resource_broker::{BackendResourceBroker, BackendResourceTarget};
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use reqwest::blocking::{Client as BlockingHttpClient, RequestBuilder};
|
use reqwest::blocking::{Client as BlockingHttpClient, RequestBuilder};
|
||||||
use reqwest::header::{AUTHORIZATION, CONTENT_TYPE};
|
use reqwest::header::{AUTHORIZATION, CONTENT_TYPE};
|
||||||
@@ -16,9 +16,10 @@ use std::{
|
|||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
use workdir::{
|
use workdir::{
|
||||||
Workdir, WorkdirError,
|
Workdir, WorkdirError, WorkdirSessionHandle,
|
||||||
http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization},
|
http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirHttpAuthorization},
|
||||||
};
|
};
|
||||||
|
use worker_runtime::RuntimeWorkspaceScope;
|
||||||
use worker_runtime::auth::{CapabilityTokenSigner, capability_claims};
|
use worker_runtime::auth::{CapabilityTokenSigner, capability_claims};
|
||||||
use worker_runtime::catalog::{
|
use worker_runtime::catalog::{
|
||||||
ConfigBundleRef, CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveHttpRef,
|
ConfigBundleRef, CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveHttpRef,
|
||||||
@@ -44,7 +45,9 @@ use worker_runtime::http_server::{
|
|||||||
RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse,
|
RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse,
|
||||||
RuntimeHttpWorkingDirectoriesResponse, RuntimeHttpWorkingDirectoryResponse,
|
RuntimeHttpWorkingDirectoriesResponse, RuntimeHttpWorkingDirectoryResponse,
|
||||||
};
|
};
|
||||||
use worker_runtime::identity::{WorkerId as EmbeddedWorkerId, WorkerRef as EmbeddedWorkerRef};
|
use worker_runtime::identity::{
|
||||||
|
RuntimeWorkerRef, WorkerId as EmbeddedWorkerId, WorkerRef as EmbeddedWorkerRef,
|
||||||
|
};
|
||||||
use worker_runtime::interaction::{
|
use worker_runtime::interaction::{
|
||||||
WorkerInput as EmbeddedWorkerInput, WorkerInputKind as EmbeddedWorkerInputKind,
|
WorkerInput as EmbeddedWorkerInput, WorkerInputKind as EmbeddedWorkerInputKind,
|
||||||
};
|
};
|
||||||
@@ -234,8 +237,8 @@ pub struct WorkerCapabilitySummary {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct WorkerSummary {
|
pub struct WorkerSummary {
|
||||||
pub runtime_id: String,
|
#[serde(flatten)]
|
||||||
pub worker_id: String,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub host_id: String,
|
pub host_id: String,
|
||||||
/// Human-readable display name. This is not identity and may be duplicated.
|
/// Human-readable display name. This is not identity and may be duplicated.
|
||||||
pub display_name: String,
|
pub display_name: String,
|
||||||
@@ -486,8 +489,8 @@ pub struct WorkerLifecycleRequest {
|
|||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct WorkerLifecycleResult {
|
pub struct WorkerLifecycleResult {
|
||||||
pub state: WorkerOperationState,
|
pub state: WorkerOperationState,
|
||||||
pub runtime_id: String,
|
#[serde(flatten)]
|
||||||
pub worker_id: String,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub diagnostics: Vec<RuntimeDiagnostic>,
|
pub diagnostics: Vec<RuntimeDiagnostic>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -495,7 +498,7 @@ pub struct WorkerLifecycleResult {
|
|||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum WorkerInputKind {
|
pub enum WorkerInputKind {
|
||||||
User,
|
User,
|
||||||
System,
|
Notify,
|
||||||
Compact,
|
Compact,
|
||||||
ListRewindTargets,
|
ListRewindTargets,
|
||||||
RegisterPeer,
|
RegisterPeer,
|
||||||
@@ -504,8 +507,8 @@ pub enum WorkerInputKind {
|
|||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct WorkerDeleteResult {
|
pub struct WorkerDeleteResult {
|
||||||
pub state: WorkerOperationState,
|
pub state: WorkerOperationState,
|
||||||
pub runtime_id: String,
|
#[serde(flatten)]
|
||||||
pub worker_id: String,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub deleted: bool,
|
pub deleted: bool,
|
||||||
pub diagnostics: Vec<RuntimeDiagnostic>,
|
pub diagnostics: Vec<RuntimeDiagnostic>,
|
||||||
}
|
}
|
||||||
@@ -528,8 +531,8 @@ pub struct WorkerCompletionsRequest {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct WorkerCompletionsResult {
|
pub struct WorkerCompletionsResult {
|
||||||
pub runtime_id: String,
|
#[serde(flatten)]
|
||||||
pub worker_id: String,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub kind: protocol::CompletionKind,
|
pub kind: protocol::CompletionKind,
|
||||||
pub prefix: String,
|
pub prefix: String,
|
||||||
pub entries: Vec<protocol::CompletionEntry>,
|
pub entries: Vec<protocol::CompletionEntry>,
|
||||||
@@ -539,8 +542,8 @@ pub struct WorkerCompletionsResult {
|
|||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct WorkerInputResult {
|
pub struct WorkerInputResult {
|
||||||
pub state: WorkerOperationState,
|
pub state: WorkerOperationState,
|
||||||
pub runtime_id: String,
|
#[serde(flatten)]
|
||||||
pub worker_id: String,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub diagnostics: Vec<RuntimeDiagnostic>,
|
pub diagnostics: Vec<RuntimeDiagnostic>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -560,8 +563,7 @@ pub enum RuntimeRegistryError {
|
|||||||
UnknownRuntime(String),
|
UnknownRuntime(String),
|
||||||
UnknownHost(String),
|
UnknownHost(String),
|
||||||
UnknownWorker {
|
UnknownWorker {
|
||||||
runtime_id: String,
|
worker: RuntimeWorkerRef,
|
||||||
worker_id: String,
|
|
||||||
},
|
},
|
||||||
RuntimeOperationFailed {
|
RuntimeOperationFailed {
|
||||||
runtime_id: String,
|
runtime_id: String,
|
||||||
@@ -578,10 +580,10 @@ impl RuntimeRegistryError {
|
|||||||
}
|
}
|
||||||
Self::UnknownRuntime(runtime_id) => format!("unknown runtime `{runtime_id}`"),
|
Self::UnknownRuntime(runtime_id) => format!("unknown runtime `{runtime_id}`"),
|
||||||
Self::UnknownHost(host_id) => format!("unknown host `{host_id}`"),
|
Self::UnknownHost(host_id) => format!("unknown host `{host_id}`"),
|
||||||
Self::UnknownWorker {
|
Self::UnknownWorker { worker } => format!(
|
||||||
runtime_id,
|
"unknown worker `{}` in runtime `{}`",
|
||||||
worker_id,
|
worker.worker_id, worker.runtime_id
|
||||||
} => format!("unknown worker `{worker_id}` in runtime `{runtime_id}`"),
|
),
|
||||||
Self::RuntimeOperationFailed { message, .. } => message.clone(),
|
Self::RuntimeOperationFailed { message, .. } => message.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -594,13 +596,7 @@ impl RuntimeRegistryError {
|
|||||||
},
|
},
|
||||||
Self::UnknownRuntime(runtime_id) => Error::UnknownRuntime(runtime_id),
|
Self::UnknownRuntime(runtime_id) => Error::UnknownRuntime(runtime_id),
|
||||||
Self::UnknownHost(host_id) => Error::UnknownHost(host_id),
|
Self::UnknownHost(host_id) => Error::UnknownHost(host_id),
|
||||||
Self::UnknownWorker {
|
Self::UnknownWorker { worker } => Error::UnknownWorker { worker },
|
||||||
runtime_id,
|
|
||||||
worker_id,
|
|
||||||
} => Error::UnknownWorker {
|
|
||||||
runtime_id,
|
|
||||||
worker_id,
|
|
||||||
},
|
|
||||||
Self::RuntimeOperationFailed {
|
Self::RuntimeOperationFailed {
|
||||||
runtime_id,
|
runtime_id,
|
||||||
code,
|
code,
|
||||||
@@ -797,8 +793,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
|||||||
) -> WorkerLifecycleResult {
|
) -> WorkerLifecycleResult {
|
||||||
WorkerLifecycleResult {
|
WorkerLifecycleResult {
|
||||||
state: WorkerOperationState::Unsupported,
|
state: WorkerOperationState::Unsupported,
|
||||||
runtime_id: self.runtime_id().to_string(),
|
worker: RuntimeWorkerRef::new(self.runtime_id().to_string(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: vec![diagnostic(
|
diagnostics: vec![diagnostic(
|
||||||
"worker_stop_pending",
|
"worker_stop_pending",
|
||||||
DiagnosticSeverity::Info,
|
DiagnosticSeverity::Info,
|
||||||
@@ -816,8 +811,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
|||||||
) -> WorkerLifecycleResult {
|
) -> WorkerLifecycleResult {
|
||||||
WorkerLifecycleResult {
|
WorkerLifecycleResult {
|
||||||
state: WorkerOperationState::Unsupported,
|
state: WorkerOperationState::Unsupported,
|
||||||
runtime_id: self.runtime_id().to_string(),
|
worker: RuntimeWorkerRef::new(self.runtime_id().to_string(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: vec![diagnostic(
|
diagnostics: vec![diagnostic(
|
||||||
"worker_cancel_pending",
|
"worker_cancel_pending",
|
||||||
DiagnosticSeverity::Info,
|
DiagnosticSeverity::Info,
|
||||||
@@ -831,8 +825,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
|||||||
fn delete_worker(&self, worker_id: &str) -> WorkerDeleteResult {
|
fn delete_worker(&self, worker_id: &str) -> WorkerDeleteResult {
|
||||||
WorkerDeleteResult {
|
WorkerDeleteResult {
|
||||||
state: WorkerOperationState::Unsupported,
|
state: WorkerOperationState::Unsupported,
|
||||||
runtime_id: self.runtime_id().to_string(),
|
worker: RuntimeWorkerRef::new(self.runtime_id().to_string(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
deleted: false,
|
deleted: false,
|
||||||
diagnostics: vec![diagnostic(
|
diagnostics: vec![diagnostic(
|
||||||
"worker_delete_unsupported",
|
"worker_delete_unsupported",
|
||||||
@@ -852,8 +845,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
|||||||
fn send_input(&self, worker_id: &str, _request: WorkerInputRequest) -> WorkerInputResult {
|
fn send_input(&self, worker_id: &str, _request: WorkerInputRequest) -> WorkerInputResult {
|
||||||
WorkerInputResult {
|
WorkerInputResult {
|
||||||
state: WorkerOperationState::Unsupported,
|
state: WorkerOperationState::Unsupported,
|
||||||
runtime_id: self.runtime_id().to_string(),
|
worker: RuntimeWorkerRef::new(self.runtime_id().to_string(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: vec![diagnostic(
|
diagnostics: vec![diagnostic(
|
||||||
"worker_input_pending",
|
"worker_input_pending",
|
||||||
DiagnosticSeverity::Info,
|
DiagnosticSeverity::Info,
|
||||||
@@ -870,8 +862,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
|||||||
request: WorkerCompletionsRequest,
|
request: WorkerCompletionsRequest,
|
||||||
) -> WorkerCompletionsResult {
|
) -> WorkerCompletionsResult {
|
||||||
WorkerCompletionsResult {
|
WorkerCompletionsResult {
|
||||||
runtime_id: self.runtime_id().to_string(),
|
worker: RuntimeWorkerRef::new(self.runtime_id().to_string(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
kind: request.kind,
|
kind: request.kind,
|
||||||
prefix: request.prefix,
|
prefix: request.prefix,
|
||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
@@ -1098,11 +1089,9 @@ impl RuntimeRegistry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn worker(
|
pub fn worker(&self, worker: &RuntimeWorkerRef) -> Result<WorkerSummary, RuntimeRegistryError> {
|
||||||
&self,
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
runtime_id: &str,
|
let worker_id = worker.worker_id.as_str();
|
||||||
worker_id: &str,
|
|
||||||
) -> Result<WorkerSummary, RuntimeRegistryError> {
|
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
@@ -1115,9 +1104,10 @@ impl RuntimeRegistry {
|
|||||||
|
|
||||||
pub fn restore_worker(
|
pub fn restore_worker(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
) -> Result<WorkerRestoreResult, RuntimeRegistryError> {
|
) -> Result<WorkerRestoreResult, RuntimeRegistryError> {
|
||||||
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
@@ -1126,10 +1116,11 @@ impl RuntimeRegistry {
|
|||||||
|
|
||||||
pub fn replace_worker_workspace_api(
|
pub fn replace_worker_workspace_api(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
workspace_api: WorkspaceApiRef,
|
workspace_api: WorkspaceApiRef,
|
||||||
) -> Result<WorkerWorkspaceApiResult, RuntimeRegistryError> {
|
) -> Result<WorkerWorkspaceApiResult, RuntimeRegistryError> {
|
||||||
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
@@ -1176,6 +1167,28 @@ impl RuntimeRegistry {
|
|||||||
Ok(runtime.working_directory(working_directory_id))
|
Ok(runtime.working_directory(working_directory_id))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn open_workdir_session(
|
||||||
|
&self,
|
||||||
|
runtime_id: &str,
|
||||||
|
working_directory_id: &str,
|
||||||
|
owner_worker_id: Option<&str>,
|
||||||
|
) -> Result<WorkdirSessionHandle, RuntimeRegistryError> {
|
||||||
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
|
validate_backend_identifier("working_directory_id", working_directory_id)?;
|
||||||
|
if let Some(owner_worker_id) = owner_worker_id {
|
||||||
|
validate_backend_identifier("owner_worker_id", owner_worker_id)?;
|
||||||
|
}
|
||||||
|
let runtime = self.runtime(runtime_id)?;
|
||||||
|
runtime
|
||||||
|
.open_workdir_session(working_directory_id, owner_worker_id)
|
||||||
|
.await
|
||||||
|
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
|
||||||
|
runtime_id: runtime_id.to_string(),
|
||||||
|
code: "workdir_session_open_failed".to_string(),
|
||||||
|
message: error.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn cleanup_working_directory(
|
pub fn cleanup_working_directory(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
runtime_id: &str,
|
||||||
@@ -1218,10 +1231,11 @@ impl RuntimeRegistry {
|
|||||||
|
|
||||||
pub fn send_protocol_method(
|
pub fn send_protocol_method(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
method: protocol::Method,
|
method: protocol::Method,
|
||||||
) -> Result<Vec<protocol::Event>, RuntimeRegistryError> {
|
) -> Result<Vec<protocol::Event>, RuntimeRegistryError> {
|
||||||
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
@@ -1238,10 +1252,11 @@ impl RuntimeRegistry {
|
|||||||
|
|
||||||
pub fn send_input(
|
pub fn send_input(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
request: WorkerInputRequest,
|
request: WorkerInputRequest,
|
||||||
) -> Result<WorkerInputResult, RuntimeRegistryError> {
|
) -> Result<WorkerInputResult, RuntimeRegistryError> {
|
||||||
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
@@ -1258,10 +1273,11 @@ impl RuntimeRegistry {
|
|||||||
|
|
||||||
pub fn worker_completions(
|
pub fn worker_completions(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
request: WorkerCompletionsRequest,
|
request: WorkerCompletionsRequest,
|
||||||
) -> Result<WorkerCompletionsResult, RuntimeRegistryError> {
|
) -> Result<WorkerCompletionsResult, RuntimeRegistryError> {
|
||||||
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
@@ -1278,10 +1294,11 @@ impl RuntimeRegistry {
|
|||||||
|
|
||||||
pub fn stop_worker(
|
pub fn stop_worker(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
request: WorkerLifecycleRequest,
|
request: WorkerLifecycleRequest,
|
||||||
) -> Result<WorkerLifecycleResult, RuntimeRegistryError> {
|
) -> Result<WorkerLifecycleResult, RuntimeRegistryError> {
|
||||||
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
@@ -1298,10 +1315,11 @@ impl RuntimeRegistry {
|
|||||||
|
|
||||||
pub fn cancel_worker(
|
pub fn cancel_worker(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
request: WorkerLifecycleRequest,
|
request: WorkerLifecycleRequest,
|
||||||
) -> Result<WorkerLifecycleResult, RuntimeRegistryError> {
|
) -> Result<WorkerLifecycleResult, RuntimeRegistryError> {
|
||||||
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
@@ -1318,9 +1336,10 @@ impl RuntimeRegistry {
|
|||||||
|
|
||||||
pub fn delete_worker(
|
pub fn delete_worker(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
) -> Result<WorkerDeleteResult, RuntimeRegistryError> {
|
) -> Result<WorkerDeleteResult, RuntimeRegistryError> {
|
||||||
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
@@ -1337,17 +1356,17 @@ impl RuntimeRegistry {
|
|||||||
|
|
||||||
pub fn observation_source(
|
pub fn observation_source(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
) -> Result<crate::observation::RuntimeObservationSource, RuntimeRegistryError> {
|
) -> Result<crate::observation::RuntimeObservationSource, RuntimeRegistryError> {
|
||||||
|
let runtime_id = worker.runtime_id.as_str();
|
||||||
|
let worker_id = worker.worker_id.as_str();
|
||||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
validate_backend_identifier("worker_id", worker_id)?;
|
validate_backend_identifier("worker_id", worker_id)?;
|
||||||
let runtime = self.runtime(runtime_id)?;
|
let runtime = self.runtime(runtime_id)?;
|
||||||
runtime
|
runtime
|
||||||
.observation_source(worker_id)
|
.observation_source(worker_id)
|
||||||
.ok_or_else(|| RuntimeRegistryError::UnknownWorker {
|
.ok_or_else(|| RuntimeRegistryError::UnknownWorker {
|
||||||
runtime_id: runtime_id.to_string(),
|
worker: worker.clone(),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1460,8 +1479,7 @@ impl EmbeddedWorkerRuntime {
|
|||||||
true,
|
true,
|
||||||
);
|
);
|
||||||
WorkerSummary {
|
WorkerSummary {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||||
worker_id,
|
|
||||||
host_id: self.host_id.clone(),
|
host_id: self.host_id.clone(),
|
||||||
display_name: display.display_name.clone(),
|
display_name: display.display_name.clone(),
|
||||||
label: display.display_name,
|
label: display.display_name,
|
||||||
@@ -1499,8 +1517,7 @@ impl EmbeddedWorkerRuntime {
|
|||||||
true,
|
true,
|
||||||
);
|
);
|
||||||
WorkerSummary {
|
WorkerSummary {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||||
worker_id,
|
|
||||||
host_id: self.host_id.clone(),
|
host_id: self.host_id.clone(),
|
||||||
display_name: display.display_name.clone(),
|
display_name: display.display_name.clone(),
|
||||||
label: display.display_name,
|
label: display.display_name,
|
||||||
@@ -1830,6 +1847,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let workspace_id = workspace_api.workspace_id.clone();
|
||||||
let create_request = CreateWorkerRequest {
|
let create_request = CreateWorkerRequest {
|
||||||
idempotency_key,
|
idempotency_key,
|
||||||
idempotency_fingerprint,
|
idempotency_fingerprint,
|
||||||
@@ -1842,7 +1860,11 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
working_directory: request.resolved_working_directory.clone(),
|
working_directory: request.resolved_working_directory.clone(),
|
||||||
workspace_api: Some(workspace_api),
|
workspace_api: Some(workspace_api),
|
||||||
};
|
};
|
||||||
match self.runtime.create_worker(create_request) {
|
let workspace_scope = RuntimeWorkspaceScope::new(workspace_id, "embedded-backend");
|
||||||
|
match self
|
||||||
|
.runtime
|
||||||
|
.create_worker_scoped(&workspace_scope, create_request)
|
||||||
|
{
|
||||||
Ok(detail) => WorkerSpawnResult {
|
Ok(detail) => WorkerSpawnResult {
|
||||||
state: WorkerOperationState::Accepted,
|
state: WorkerOperationState::Accepted,
|
||||||
worker: Some(self.map_worker_detail(detail)),
|
worker: Some(self.map_worker_detail(detail)),
|
||||||
@@ -1946,8 +1968,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
match self.runtime.stop_worker(&worker_ref, request.reason) {
|
match self.runtime.stop_worker(&worker_ref, request.reason) {
|
||||||
Ok(_) => WorkerLifecycleResult {
|
Ok(_) => WorkerLifecycleResult {
|
||||||
state: WorkerOperationState::Accepted,
|
state: WorkerOperationState::Accepted,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
},
|
},
|
||||||
Err(error) => embedded_lifecycle_rejected(
|
Err(error) => embedded_lifecycle_rejected(
|
||||||
@@ -1990,8 +2011,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
match self.runtime.cancel_worker(&worker_ref, request.reason) {
|
match self.runtime.cancel_worker(&worker_ref, request.reason) {
|
||||||
Ok(_) => WorkerLifecycleResult {
|
Ok(_) => WorkerLifecycleResult {
|
||||||
state: WorkerOperationState::Accepted,
|
state: WorkerOperationState::Accepted,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
},
|
},
|
||||||
Err(error) => embedded_lifecycle_rejected(
|
Err(error) => embedded_lifecycle_rejected(
|
||||||
@@ -2006,8 +2026,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
||||||
return WorkerDeleteResult {
|
return WorkerDeleteResult {
|
||||||
state: WorkerOperationState::Rejected,
|
state: WorkerOperationState::Rejected,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
deleted: false,
|
deleted: false,
|
||||||
diagnostics: vec![diagnostic(
|
diagnostics: vec![diagnostic(
|
||||||
"embedded_worker_id_invalid",
|
"embedded_worker_id_invalid",
|
||||||
@@ -2019,15 +2038,16 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
match self.runtime.delete_worker(&worker_ref) {
|
match self.runtime.delete_worker(&worker_ref) {
|
||||||
Ok(result) => WorkerDeleteResult {
|
Ok(result) => WorkerDeleteResult {
|
||||||
state: WorkerOperationState::Accepted,
|
state: WorkerOperationState::Accepted,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(
|
||||||
worker_id: result.worker_id.to_string(),
|
self.runtime_id.clone(),
|
||||||
|
result.worker_id.to_string(),
|
||||||
|
),
|
||||||
deleted: result.deleted,
|
deleted: result.deleted,
|
||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
},
|
},
|
||||||
Err(error) => WorkerDeleteResult {
|
Err(error) => WorkerDeleteResult {
|
||||||
state: WorkerOperationState::Rejected,
|
state: WorkerOperationState::Rejected,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
deleted: false,
|
deleted: false,
|
||||||
diagnostics: vec![embedded_runtime_diagnostic(&error)],
|
diagnostics: vec![embedded_runtime_diagnostic(&error)],
|
||||||
},
|
},
|
||||||
@@ -2044,8 +2064,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
}
|
}
|
||||||
Some(crate::observation::RuntimeObservationSource::embedded(
|
Some(crate::observation::RuntimeObservationSource::embedded(
|
||||||
crate::observation::EmbeddedRuntimeObservationSource {
|
crate::observation::EmbeddedRuntimeObservationSource {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
runtime: self.runtime.clone(),
|
runtime: self.runtime.clone(),
|
||||||
worker_ref,
|
worker_ref,
|
||||||
},
|
},
|
||||||
@@ -2068,8 +2087,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
}
|
}
|
||||||
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
||||||
return Err(RuntimeRegistryError::UnknownWorker {
|
return Err(RuntimeRegistryError::UnknownWorker {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
self.runtime
|
self.runtime
|
||||||
@@ -2109,7 +2127,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
let input = EmbeddedWorkerInput {
|
let input = EmbeddedWorkerInput {
|
||||||
kind: match request.kind {
|
kind: match request.kind {
|
||||||
WorkerInputKind::User => EmbeddedWorkerInputKind::User,
|
WorkerInputKind::User => EmbeddedWorkerInputKind::User,
|
||||||
WorkerInputKind::System => EmbeddedWorkerInputKind::System,
|
WorkerInputKind::Notify => EmbeddedWorkerInputKind::Notify,
|
||||||
WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact,
|
WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact,
|
||||||
WorkerInputKind::ListRewindTargets => EmbeddedWorkerInputKind::ListRewindTargets,
|
WorkerInputKind::ListRewindTargets => EmbeddedWorkerInputKind::ListRewindTargets,
|
||||||
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
||||||
@@ -2120,8 +2138,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
match self.runtime.send_input(&worker_ref, input) {
|
match self.runtime.send_input(&worker_ref, input) {
|
||||||
Ok(_) => WorkerInputResult {
|
Ok(_) => WorkerInputResult {
|
||||||
state: WorkerOperationState::Accepted,
|
state: WorkerOperationState::Accepted,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
},
|
},
|
||||||
Err(error) => embedded_input_rejected(
|
Err(error) => embedded_input_rejected(
|
||||||
@@ -2139,8 +2156,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
) -> WorkerCompletionsResult {
|
) -> WorkerCompletionsResult {
|
||||||
if !self.execution_enabled {
|
if !self.execution_enabled {
|
||||||
return WorkerCompletionsResult {
|
return WorkerCompletionsResult {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
kind: request.kind,
|
kind: request.kind,
|
||||||
prefix: request.prefix,
|
prefix: request.prefix,
|
||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
@@ -2155,8 +2171,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
}
|
}
|
||||||
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
let Some(worker_ref) = self.worker_ref(worker_id) else {
|
||||||
return WorkerCompletionsResult {
|
return WorkerCompletionsResult {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
kind: request.kind,
|
kind: request.kind,
|
||||||
prefix: request.prefix,
|
prefix: request.prefix,
|
||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
@@ -2172,16 +2187,14 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
.worker_completions(&worker_ref, request.kind, &request.prefix)
|
.worker_completions(&worker_ref, request.kind, &request.prefix)
|
||||||
{
|
{
|
||||||
Ok(entries) => WorkerCompletionsResult {
|
Ok(entries) => WorkerCompletionsResult {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
kind: request.kind,
|
kind: request.kind,
|
||||||
prefix: request.prefix,
|
prefix: request.prefix,
|
||||||
entries,
|
entries,
|
||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
},
|
},
|
||||||
Err(error) => WorkerCompletionsResult {
|
Err(error) => WorkerCompletionsResult {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
kind: request.kind,
|
kind: request.kind,
|
||||||
prefix: request.prefix,
|
prefix: request.prefix,
|
||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
@@ -2544,8 +2557,7 @@ impl RemoteWorkerRuntime {
|
|||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
WorkerSummary {
|
WorkerSummary {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||||
worker_id,
|
|
||||||
host_id: self.host_id.clone(),
|
host_id: self.host_id.clone(),
|
||||||
display_name: display.display_name.clone(),
|
display_name: display.display_name.clone(),
|
||||||
label: display.display_name,
|
label: display.display_name,
|
||||||
@@ -2587,8 +2599,7 @@ impl RemoteWorkerRuntime {
|
|||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
WorkerSummary {
|
WorkerSummary {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()),
|
||||||
worker_id,
|
|
||||||
host_id: self.host_id.clone(),
|
host_id: self.host_id.clone(),
|
||||||
display_name: display.display_name.clone(),
|
display_name: display.display_name.clone(),
|
||||||
label: display.display_name,
|
label: display.display_name,
|
||||||
@@ -2627,8 +2638,7 @@ impl RemoteWorkerRuntime {
|
|||||||
) -> WorkerLifecycleResult {
|
) -> WorkerLifecycleResult {
|
||||||
WorkerLifecycleResult {
|
WorkerLifecycleResult {
|
||||||
state: WorkerOperationState::Accepted,
|
state: WorkerOperationState::Accepted,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: vec![diagnostic(
|
diagnostics: vec![diagnostic(
|
||||||
"remote_runtime_lifecycle_accepted",
|
"remote_runtime_lifecycle_accepted",
|
||||||
DiagnosticSeverity::Info,
|
DiagnosticSeverity::Info,
|
||||||
@@ -3045,15 +3055,16 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
{
|
{
|
||||||
Ok(response) => WorkerDeleteResult {
|
Ok(response) => WorkerDeleteResult {
|
||||||
state: WorkerOperationState::Accepted,
|
state: WorkerOperationState::Accepted,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(
|
||||||
worker_id: response.worker.worker_id.to_string(),
|
self.runtime_id.clone(),
|
||||||
|
response.worker.worker_id.to_string(),
|
||||||
|
),
|
||||||
deleted: response.worker.deleted,
|
deleted: response.worker.deleted,
|
||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
},
|
},
|
||||||
Err(diagnostic) => WorkerDeleteResult {
|
Err(diagnostic) => WorkerDeleteResult {
|
||||||
state: WorkerOperationState::Rejected,
|
state: WorkerOperationState::Rejected,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
deleted: false,
|
deleted: false,
|
||||||
diagnostics: vec![diagnostic],
|
diagnostics: vec![diagnostic],
|
||||||
},
|
},
|
||||||
@@ -3066,8 +3077,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
) -> Option<crate::observation::RuntimeObservationSource> {
|
) -> Option<crate::observation::RuntimeObservationSource> {
|
||||||
Some(crate::observation::RuntimeObservationSource::remote_ws(
|
Some(crate::observation::RuntimeObservationSource::remote_ws(
|
||||||
crate::observation::RuntimeObservationSourceConfig {
|
crate::observation::RuntimeObservationSourceConfig {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
endpoint: self.ws_endpoint(worker_id),
|
endpoint: self.ws_endpoint(worker_id),
|
||||||
bearer_token: self
|
bearer_token: self
|
||||||
.runtime_capability_token(&format!("/v1/workers/{worker_id}/protocol"))
|
.runtime_capability_token(&format!("/v1/workers/{worker_id}/protocol"))
|
||||||
@@ -3080,7 +3090,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
let input = EmbeddedWorkerInput {
|
let input = EmbeddedWorkerInput {
|
||||||
kind: match request.kind {
|
kind: match request.kind {
|
||||||
WorkerInputKind::User => EmbeddedWorkerInputKind::User,
|
WorkerInputKind::User => EmbeddedWorkerInputKind::User,
|
||||||
WorkerInputKind::System => EmbeddedWorkerInputKind::System,
|
WorkerInputKind::Notify => EmbeddedWorkerInputKind::Notify,
|
||||||
WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact,
|
WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact,
|
||||||
WorkerInputKind::ListRewindTargets => EmbeddedWorkerInputKind::ListRewindTargets,
|
WorkerInputKind::ListRewindTargets => EmbeddedWorkerInputKind::ListRewindTargets,
|
||||||
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
|
||||||
@@ -3094,8 +3104,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
) {
|
) {
|
||||||
Ok(_) => WorkerInputResult {
|
Ok(_) => WorkerInputResult {
|
||||||
state: WorkerOperationState::Accepted,
|
state: WorkerOperationState::Accepted,
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
},
|
},
|
||||||
Err(diagnostic) => remote_input_rejected(&self.runtime_id, worker_id, diagnostic),
|
Err(diagnostic) => remote_input_rejected(&self.runtime_id, worker_id, diagnostic),
|
||||||
@@ -3116,16 +3125,14 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
&request,
|
&request,
|
||||||
) {
|
) {
|
||||||
Ok(response) => WorkerCompletionsResult {
|
Ok(response) => WorkerCompletionsResult {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
kind: response.kind,
|
kind: response.kind,
|
||||||
prefix: response.prefix,
|
prefix: response.prefix,
|
||||||
entries: response.entries,
|
entries: response.entries,
|
||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
},
|
},
|
||||||
Err(diagnostic) => WorkerCompletionsResult {
|
Err(diagnostic) => WorkerCompletionsResult {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
kind: request.kind,
|
kind: request.kind,
|
||||||
prefix: request.prefix,
|
prefix: request.prefix,
|
||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
@@ -3219,10 +3226,12 @@ fn profile_source_archive_http_source(
|
|||||||
backend_base_url: &str,
|
backend_base_url: &str,
|
||||||
) -> Result<ProfileSourceArchiveSource, String> {
|
) -> Result<ProfileSourceArchiveSource, String> {
|
||||||
let archive = profile_source_archive_for_request(request, profile)?;
|
let archive = profile_source_archive_for_request(request, profile)?;
|
||||||
|
let target = runtime_id
|
||||||
|
.map(BackendResourceTarget::Runtime)
|
||||||
|
.unwrap_or(BackendResourceTarget::Workspace);
|
||||||
let _handle = resource_broker.issue_profile_source_archive_handle(
|
let _handle = resource_broker.issue_profile_source_archive_handle(
|
||||||
workspace_id.to_string(),
|
workspace_id.to_string(),
|
||||||
runtime_id,
|
target,
|
||||||
None,
|
|
||||||
archive.clone(),
|
archive.clone(),
|
||||||
);
|
);
|
||||||
let etag = format!("\"profile-source:{}\"", archive.reference.digest);
|
let etag = format!("\"profile-source:{}\"", archive.reference.digest);
|
||||||
@@ -3266,10 +3275,12 @@ fn builtin_profile_config_bundle(
|
|||||||
let (profile_source_archive, profile_source_archive_handle) = match archive_transport {
|
let (profile_source_archive, profile_source_archive_handle) = match archive_transport {
|
||||||
ProfileSourceArchiveTransport::Inline => (Some(archive), None),
|
ProfileSourceArchiveTransport::Inline => (Some(archive), None),
|
||||||
ProfileSourceArchiveTransport::BackendResourceHandle => {
|
ProfileSourceArchiveTransport::BackendResourceHandle => {
|
||||||
|
let target = runtime_id
|
||||||
|
.map(BackendResourceTarget::Runtime)
|
||||||
|
.unwrap_or(BackendResourceTarget::Workspace);
|
||||||
let handle = resource_broker.issue_profile_source_archive_handle(
|
let handle = resource_broker.issue_profile_source_archive_handle(
|
||||||
workspace_id.to_string(),
|
workspace_id.to_string(),
|
||||||
runtime_id,
|
target,
|
||||||
None,
|
|
||||||
archive,
|
archive,
|
||||||
);
|
);
|
||||||
(None, Some(handle))
|
(None, Some(handle))
|
||||||
@@ -3433,16 +3444,15 @@ fn worker_display_metadata(
|
|||||||
tags,
|
tags,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if profile_label == Some(WORKSPACE_ORCHESTRATOR_PROFILE) {
|
if profile_label == Some(WORKSPACE_ORCHESTRATOR_PROFILE)
|
||||||
|
&& requested_display_name == Some(WORKSPACE_ORCHESTRATOR_SINGLETON_KEY)
|
||||||
|
{
|
||||||
let mut tags = vec!["orchestrator".to_string(), "singleton".to_string()];
|
let mut tags = vec!["orchestrator".to_string(), "singleton".to_string()];
|
||||||
if internal {
|
if internal {
|
||||||
tags.insert(0, "internal".to_string());
|
tags.insert(0, "internal".to_string());
|
||||||
}
|
}
|
||||||
return WorkerDisplayMetadata {
|
return WorkerDisplayMetadata {
|
||||||
display_name: requested_display_name
|
display_name: "Workspace Orchestrator".to_string(),
|
||||||
.filter(|value| !value.trim().is_empty())
|
|
||||||
.map(safe_display_hint)
|
|
||||||
.unwrap_or_else(|| "Workspace Orchestrator".to_string()),
|
|
||||||
singleton_key: Some(WORKSPACE_ORCHESTRATOR_SINGLETON_KEY.to_string()),
|
singleton_key: Some(WORKSPACE_ORCHESTRATOR_SINGLETON_KEY.to_string()),
|
||||||
tags,
|
tags,
|
||||||
};
|
};
|
||||||
@@ -3488,8 +3498,7 @@ fn embedded_input_rejected(
|
|||||||
) -> WorkerInputResult {
|
) -> WorkerInputResult {
|
||||||
WorkerInputResult {
|
WorkerInputResult {
|
||||||
state: WorkerOperationState::Rejected,
|
state: WorkerOperationState::Rejected,
|
||||||
runtime_id: runtime_id.to_string(),
|
worker: RuntimeWorkerRef::new(runtime_id.to_string(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: vec![diagnostic],
|
diagnostics: vec![diagnostic],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3501,8 +3510,7 @@ fn remote_input_rejected(
|
|||||||
) -> WorkerInputResult {
|
) -> WorkerInputResult {
|
||||||
WorkerInputResult {
|
WorkerInputResult {
|
||||||
state: WorkerOperationState::Rejected,
|
state: WorkerOperationState::Rejected,
|
||||||
runtime_id: runtime_id.to_string(),
|
worker: RuntimeWorkerRef::new(runtime_id.to_string(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: vec![diagnostic],
|
diagnostics: vec![diagnostic],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3514,8 +3522,7 @@ fn embedded_lifecycle_rejected(
|
|||||||
) -> WorkerLifecycleResult {
|
) -> WorkerLifecycleResult {
|
||||||
WorkerLifecycleResult {
|
WorkerLifecycleResult {
|
||||||
state: WorkerOperationState::Rejected,
|
state: WorkerOperationState::Rejected,
|
||||||
runtime_id: runtime_id.to_string(),
|
worker: RuntimeWorkerRef::new(runtime_id.to_string(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: vec![diagnostic],
|
diagnostics: vec![diagnostic],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3527,8 +3534,7 @@ fn remote_lifecycle_rejected(
|
|||||||
) -> WorkerLifecycleResult {
|
) -> WorkerLifecycleResult {
|
||||||
WorkerLifecycleResult {
|
WorkerLifecycleResult {
|
||||||
state: WorkerOperationState::Rejected,
|
state: WorkerOperationState::Rejected,
|
||||||
runtime_id: runtime_id.to_string(),
|
worker: RuntimeWorkerRef::new(runtime_id.to_string(), worker_id.to_string()),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
diagnostics: vec![diagnostic],
|
diagnostics: vec![diagnostic],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3752,8 +3758,7 @@ fn operation_failed_or_unknown_worker(
|
|||||||
message: diagnostic.message,
|
message: diagnostic.message,
|
||||||
})
|
})
|
||||||
.unwrap_or_else(|| RuntimeRegistryError::UnknownWorker {
|
.unwrap_or_else(|| RuntimeRegistryError::UnknownWorker {
|
||||||
runtime_id: runtime_id.to_string(),
|
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3860,8 +3865,7 @@ fn worker_spawn_intent_label(intent: &WorkerSpawnIntent) -> &'static str {
|
|||||||
pub fn placeholder_worker(host_id: impl Into<String>) -> WorkerSummary {
|
pub fn placeholder_worker(host_id: impl Into<String>) -> WorkerSummary {
|
||||||
let host_id = host_id.into();
|
let host_id = host_id.into();
|
||||||
WorkerSummary {
|
WorkerSummary {
|
||||||
runtime_id: "placeholder".to_string(),
|
worker: RuntimeWorkerRef::new("placeholder", "worker-placeholder"),
|
||||||
worker_id: "worker-placeholder".to_string(),
|
|
||||||
host_id,
|
host_id,
|
||||||
display_name: "Worker runtime actions are not implemented".to_string(),
|
display_name: "Worker runtime actions are not implemented".to_string(),
|
||||||
label: "Worker runtime actions are not implemented".to_string(),
|
label: "Worker runtime actions are not implemented".to_string(),
|
||||||
@@ -3924,6 +3928,83 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_summary_keeps_flat_wire_identity_while_using_structured_internal_identity() {
|
||||||
|
let summary = placeholder_worker("placeholder");
|
||||||
|
assert_eq!(
|
||||||
|
summary.worker,
|
||||||
|
RuntimeWorkerRef::new("placeholder", "worker-placeholder")
|
||||||
|
);
|
||||||
|
let value = serde_json::to_value(summary).unwrap();
|
||||||
|
assert_eq!(value["runtime_id"], "placeholder");
|
||||||
|
assert_eq!(value["worker_id"], "worker-placeholder");
|
||||||
|
assert!(value.get("worker").is_none());
|
||||||
|
|
||||||
|
let lifecycle = WorkerLifecycleResult {
|
||||||
|
state: WorkerOperationState::Accepted,
|
||||||
|
worker: RuntimeWorkerRef::new("arcadia", "30"),
|
||||||
|
diagnostics: Vec::new(),
|
||||||
|
};
|
||||||
|
let value = serde_json::to_value(lifecycle).unwrap();
|
||||||
|
assert_eq!(value["runtime_id"], "arcadia");
|
||||||
|
assert_eq!(value["worker_id"], "30");
|
||||||
|
assert!(value.get("worker").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn embedded_orchestrator_profile_enables_workdir_and_worker_authority() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let broker = BackendResourceBroker::default();
|
||||||
|
let runtime_id = "runtime-test";
|
||||||
|
let selector = ProfileSelector::Builtin("builtin:orchestrator".to_string());
|
||||||
|
let bundle = builtin_profile_config_bundle(
|
||||||
|
&selector,
|
||||||
|
"workspace-test",
|
||||||
|
Some(runtime_id),
|
||||||
|
&broker,
|
||||||
|
ProfileSourceArchiveTransport::BackendResourceHandle,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let handle = bundle.profile_source_archive_handle.as_ref().unwrap();
|
||||||
|
let response = broker
|
||||||
|
.fetch_profile_source_archive(worker_runtime::resource::BackendResourceFetchRequest {
|
||||||
|
handle: handle.clone(),
|
||||||
|
runtime_id: runtime_id.to_string(),
|
||||||
|
worker_id: None,
|
||||||
|
audit_correlation_id: handle.audit_correlation_id.clone(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let archive =
|
||||||
|
worker_runtime::resource::profile_source_archive_from_response(handle, response)
|
||||||
|
.unwrap()
|
||||||
|
.verify()
|
||||||
|
.unwrap();
|
||||||
|
let manifest = archive
|
||||||
|
.resolve_profile("builtin:orchestrator", root.path(), "embedded-orchestrator")
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(manifest.feature.manage_workdir.enabled);
|
||||||
|
assert!(!manifest.feature.sub_worker.enabled);
|
||||||
|
assert!(manifest.feature.worker.enabled);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn embedded_companion_profile_enables_worker_management() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let selector = ProfileSelector::Builtin("builtin:companion".to_string());
|
||||||
|
let archive = builtin_profile_source_archive(&selector)
|
||||||
|
.unwrap()
|
||||||
|
.verify()
|
||||||
|
.unwrap();
|
||||||
|
let manifest = archive
|
||||||
|
.resolve_profile("builtin:companion", root.path(), "companion-test-worker")
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(manifest.feature.worker.enabled);
|
||||||
|
assert!(manifest.feature.sub_worker.enabled);
|
||||||
|
assert!(!manifest.feature.manage_workdir.enabled);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn embedded_builtin_decodal_profiles_resolve_through_archive() {
|
fn embedded_builtin_decodal_profiles_resolve_through_archive() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let root = tempfile::tempdir().unwrap();
|
||||||
@@ -3969,7 +4050,10 @@ mod tests {
|
|||||||
.resolve_profile(&selector_key, root.path(), "embedded-test-worker")
|
.resolve_profile(&selector_key, root.path(), "embedded-test-worker")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(manifest.worker.name, "embedded-test-worker");
|
assert_eq!(manifest.worker.name, "embedded-test-worker");
|
||||||
assert_eq!(manifest.model.ref_.as_deref(), Some("codex-oauth/gpt-5.5"));
|
assert_eq!(
|
||||||
|
manifest.model.ref_.as_deref(),
|
||||||
|
Some("codex-oauth/gpt-5.6-sol")
|
||||||
|
);
|
||||||
if selector_key == "builtin:memory-consolidation" {
|
if selector_key == "builtin:memory-consolidation" {
|
||||||
assert!(manifest.feature.memory.enabled);
|
assert!(manifest.feature.memory.enabled);
|
||||||
assert!(manifest.feature.memory.staging);
|
assert!(manifest.feature.memory.staging);
|
||||||
@@ -4225,8 +4309,7 @@ mod tests {
|
|||||||
runtime_id: runtime_id.to_string(),
|
runtime_id: runtime_id.to_string(),
|
||||||
host_id: host_id.to_string(),
|
host_id: host_id.to_string(),
|
||||||
workers: vec![WorkerSummary {
|
workers: vec![WorkerSummary {
|
||||||
runtime_id: runtime_id.to_string(),
|
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
host_id: host_id.to_string(),
|
host_id: host_id.to_string(),
|
||||||
display_name: label.to_string(),
|
display_name: label.to_string(),
|
||||||
label: label.to_string(),
|
label: label.to_string(),
|
||||||
@@ -4318,7 +4401,7 @@ mod tests {
|
|||||||
worker: self
|
worker: self
|
||||||
.workers
|
.workers
|
||||||
.iter()
|
.iter()
|
||||||
.find(|worker| worker.worker_id == worker_id)
|
.find(|worker| worker.worker.worker_id == worker_id)
|
||||||
.cloned(),
|
.cloned(),
|
||||||
diagnostics: Vec::new(),
|
diagnostics: Vec::new(),
|
||||||
}
|
}
|
||||||
@@ -4342,13 +4425,17 @@ mod tests {
|
|||||||
)),
|
)),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
let from_runtime_b = registry.worker("runtime-b", "shared-worker").unwrap();
|
let from_runtime_b = registry
|
||||||
assert_eq!(from_runtime_b.runtime_id, "runtime-b");
|
.worker(&RuntimeWorkerRef::new("runtime-b", "shared-worker"))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(from_runtime_b.worker.runtime_id, "runtime-b");
|
||||||
assert_eq!(from_runtime_b.host_id, "host-b");
|
assert_eq!(from_runtime_b.host_id, "host-b");
|
||||||
assert_eq!(from_runtime_b.label, "worker from runtime b");
|
assert_eq!(from_runtime_b.label, "worker from runtime b");
|
||||||
|
|
||||||
let from_runtime_a = registry.worker("runtime-a", "shared-worker").unwrap();
|
let from_runtime_a = registry
|
||||||
assert_eq!(from_runtime_a.runtime_id, "runtime-a");
|
.worker(&RuntimeWorkerRef::new("runtime-a", "shared-worker"))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(from_runtime_a.worker.runtime_id, "runtime-a");
|
||||||
assert_eq!(from_runtime_a.host_id, "host-a");
|
assert_eq!(from_runtime_a.host_id, "host-a");
|
||||||
assert_eq!(from_runtime_a.label, "worker from runtime a");
|
assert_eq!(from_runtime_a.label, "worker from runtime a");
|
||||||
}
|
}
|
||||||
@@ -4372,7 +4459,7 @@ mod tests {
|
|||||||
|
|
||||||
let listed = registry.list_workers_for_runtime("runtime-b", 10).unwrap();
|
let listed = registry.list_workers_for_runtime("runtime-b", 10).unwrap();
|
||||||
assert_eq!(listed.items.len(), 1);
|
assert_eq!(listed.items.len(), 1);
|
||||||
assert_eq!(listed.items[0].runtime_id, "runtime-b");
|
assert_eq!(listed.items[0].worker.runtime_id, "runtime-b");
|
||||||
assert_eq!(listed.items[0].host_id, "host-b");
|
assert_eq!(listed.items[0].host_id, "host-b");
|
||||||
assert_eq!(listed.items[0].label, "worker from runtime b");
|
assert_eq!(listed.items[0].label, "worker from runtime b");
|
||||||
}
|
}
|
||||||
@@ -4391,7 +4478,9 @@ mod tests {
|
|||||||
Some("builtin:companion")
|
Some("builtin:companion")
|
||||||
);
|
);
|
||||||
|
|
||||||
let worker = registry.worker("runtime-a", "worker-a").unwrap();
|
let worker = registry
|
||||||
|
.worker(&RuntimeWorkerRef::new("runtime-a", "worker-a"))
|
||||||
|
.unwrap();
|
||||||
assert_eq!(worker.profile.as_deref(), Some("builtin:companion"));
|
assert_eq!(worker.profile.as_deref(), Some("builtin:companion"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4404,7 +4493,9 @@ mod tests {
|
|||||||
"worker from runtime a",
|
"worker from runtime a",
|
||||||
))]);
|
))]);
|
||||||
|
|
||||||
let unknown_runtime = registry.worker("runtime-missing", "worker-a").unwrap_err();
|
let unknown_runtime = registry
|
||||||
|
.worker(&RuntimeWorkerRef::new("runtime-missing", "worker-a"))
|
||||||
|
.unwrap_err();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
unknown_runtime,
|
unknown_runtime,
|
||||||
RuntimeRegistryError::UnknownRuntime("runtime-missing".to_string())
|
RuntimeRegistryError::UnknownRuntime("runtime-missing".to_string())
|
||||||
@@ -4414,18 +4505,19 @@ mod tests {
|
|||||||
Error::UnknownRuntime(runtime_id) if runtime_id == "runtime-missing"
|
Error::UnknownRuntime(runtime_id) if runtime_id == "runtime-missing"
|
||||||
));
|
));
|
||||||
|
|
||||||
let unknown_worker = registry.worker("runtime-a", "999").unwrap_err();
|
let unknown_worker = registry
|
||||||
|
.worker(&RuntimeWorkerRef::new("runtime-a", "999"))
|
||||||
|
.unwrap_err();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
unknown_worker,
|
unknown_worker,
|
||||||
RuntimeRegistryError::UnknownWorker {
|
RuntimeRegistryError::UnknownWorker {
|
||||||
runtime_id: "runtime-a".to_string(),
|
worker: RuntimeWorkerRef::new("runtime-a", "999"),
|
||||||
worker_id: "999".to_string(),
|
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
unknown_worker.into_error(),
|
unknown_worker.into_error(),
|
||||||
Error::UnknownWorker { runtime_id, worker_id }
|
Error::UnknownWorker { worker }
|
||||||
if runtime_id == "runtime-a" && worker_id == "999"
|
if worker == RuntimeWorkerRef::new("runtime-a", "999")
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4497,7 +4589,7 @@ mod tests {
|
|||||||
.expect("test backend should connect");
|
.expect("test backend should connect");
|
||||||
let mut request = embedded_spawn_request();
|
let mut request = embedded_spawn_request();
|
||||||
request.initial_input = Some(EmbeddedWorkerInput {
|
request.initial_input = Some(EmbeddedWorkerInput {
|
||||||
kind: EmbeddedWorkerInputKind::System,
|
kind: EmbeddedWorkerInputKind::Notify,
|
||||||
content: "system/role instruction belongs in profile".to_string(),
|
content: "system/role instruction belongs in profile".to_string(),
|
||||||
segments: None,
|
segments: None,
|
||||||
});
|
});
|
||||||
@@ -4527,7 +4619,7 @@ mod tests {
|
|||||||
assert!(worker.capabilities.can_stop);
|
assert!(worker.capabilities.can_stop);
|
||||||
|
|
||||||
let input = runtime.send_input(
|
let input = runtime.send_input(
|
||||||
&worker.worker_id,
|
&worker.worker.worker_id,
|
||||||
WorkerInputRequest {
|
WorkerInputRequest {
|
||||||
kind: WorkerInputKind::User,
|
kind: WorkerInputKind::User,
|
||||||
content: "hello".to_string(),
|
content: "hello".to_string(),
|
||||||
@@ -4539,7 +4631,7 @@ mod tests {
|
|||||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
|
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||||
loop {
|
loop {
|
||||||
let detail = runtime
|
let detail = runtime
|
||||||
.worker(&worker.worker_id)
|
.worker(&worker.worker.worker_id)
|
||||||
.worker
|
.worker
|
||||||
.expect("worker detail");
|
.expect("worker detail");
|
||||||
if detail.state == "idle" {
|
if detail.state == "idle" {
|
||||||
@@ -4607,15 +4699,14 @@ mod tests {
|
|||||||
.any(|evidence| evidence.kind == "embedded_runtime_backend_internal_projection")
|
.any(|evidence| evidence.kind == "embedded_runtime_backend_internal_projection")
|
||||||
);
|
);
|
||||||
let worker = spawned.worker.expect("created embedded worker");
|
let worker = spawned.worker.expect("created embedded worker");
|
||||||
assert_eq!(worker.runtime_id, EMBEDDED_RUNTIME_ID);
|
assert_eq!(worker.worker.runtime_id, EMBEDDED_RUNTIME_ID);
|
||||||
assert_eq!(worker.workspace.visibility, "backend_internal");
|
assert_eq!(worker.workspace.visibility, "backend_internal");
|
||||||
assert_eq!(worker.workspace.identity, "runtime_registry_worker");
|
assert_eq!(worker.workspace.identity, "runtime_registry_worker");
|
||||||
assert_eq!(worker.implementation.kind, "embedded_worker_runtime");
|
assert_eq!(worker.implementation.kind, "embedded_worker_runtime");
|
||||||
assert_eq!(worker.profile.as_deref(), Some("builtin:coder"));
|
assert_eq!(worker.profile.as_deref(), Some("builtin:coder"));
|
||||||
let input = registry
|
let input = registry
|
||||||
.send_input(
|
.send_input(
|
||||||
EMBEDDED_RUNTIME_ID,
|
&worker.worker,
|
||||||
&worker.worker_id,
|
|
||||||
WorkerInputRequest {
|
WorkerInputRequest {
|
||||||
kind: WorkerInputKind::User,
|
kind: WorkerInputKind::User,
|
||||||
content: "hello embedded runtime".to_string(),
|
content: "hello embedded runtime".to_string(),
|
||||||
@@ -4624,12 +4715,10 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(input.state, WorkerOperationState::Accepted);
|
assert_eq!(input.state, WorkerOperationState::Accepted);
|
||||||
assert_eq!(input.runtime_id, EMBEDDED_RUNTIME_ID);
|
assert_eq!(input.worker.runtime_id, EMBEDDED_RUNTIME_ID);
|
||||||
assert_eq!(input.worker_id, worker.worker_id);
|
assert_eq!(input.worker.worker_id, worker.worker.worker_id);
|
||||||
|
|
||||||
let detail = registry
|
let detail = registry.worker(&worker.worker).unwrap();
|
||||||
.worker(EMBEDDED_RUNTIME_ID, &worker.worker_id)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let json = serde_json::to_string(&(embedded_summary, worker, input, detail)).unwrap();
|
let json = serde_json::to_string(&(embedded_summary, worker, input, detail)).unwrap();
|
||||||
for forbidden in [
|
for forbidden in [
|
||||||
@@ -4807,7 +4896,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let observation = registry
|
let observation = registry
|
||||||
.observation_source("remote:primary", "1")
|
.observation_source(&RuntimeWorkerRef::new("remote:primary", "1"))
|
||||||
.expect("remote runtime exposes backend-owned WS observation source");
|
.expect("remote runtime exposes backend-owned WS observation source");
|
||||||
let crate::observation::RuntimeObservationSource::RemoteWs(observation) = observation
|
let crate::observation::RuntimeObservationSource::RemoteWs(observation) = observation
|
||||||
else {
|
else {
|
||||||
@@ -4819,8 +4908,8 @@ mod tests {
|
|||||||
|
|
||||||
let workers = registry.list_workers(10);
|
let workers = registry.list_workers(10);
|
||||||
assert_eq!(workers.items.len(), 1);
|
assert_eq!(workers.items.len(), 1);
|
||||||
assert_eq!(workers.items[0].runtime_id, "remote:primary");
|
assert_eq!(workers.items[0].worker.runtime_id, "remote:primary");
|
||||||
assert_eq!(workers.items[0].worker_id, "1");
|
assert_eq!(workers.items[0].worker.worker_id, "1");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
workers.items[0].implementation.kind,
|
workers.items[0].implementation.kind,
|
||||||
"remote_worker_runtime"
|
"remote_worker_runtime"
|
||||||
@@ -4833,8 +4922,7 @@ mod tests {
|
|||||||
|
|
||||||
let input = registry
|
let input = registry
|
||||||
.send_input(
|
.send_input(
|
||||||
"remote:primary",
|
&RuntimeWorkerRef::new("remote:primary", "1"),
|
||||||
"1",
|
|
||||||
WorkerInputRequest {
|
WorkerInputRequest {
|
||||||
kind: WorkerInputKind::User,
|
kind: WorkerInputKind::User,
|
||||||
content: "hello remote".to_string(),
|
content: "hello remote".to_string(),
|
||||||
@@ -4911,7 +4999,9 @@ mod tests {
|
|||||||
assert_eq!(workers.items[2].state, "paused");
|
assert_eq!(workers.items[2].state, "paused");
|
||||||
assert_eq!(workers.items[3].state, "idle");
|
assert_eq!(workers.items[3].state, "idle");
|
||||||
|
|
||||||
let stopped_detail = registry.worker("remote:primary", "1").unwrap();
|
let stopped_detail = registry
|
||||||
|
.worker(&RuntimeWorkerRef::new("remote:primary", "1"))
|
||||||
|
.unwrap();
|
||||||
assert!(!stopped_detail.capabilities.can_stop);
|
assert!(!stopped_detail.capabilities.can_stop);
|
||||||
assert_eq!(stopped_detail.state, "stopped");
|
assert_eq!(stopped_detail.state, "stopped");
|
||||||
|
|
||||||
@@ -5090,7 +5180,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let error = registry
|
let error = registry
|
||||||
.worker("remote:primary", "999")
|
.worker(&RuntimeWorkerRef::new("remote:primary", "999"))
|
||||||
.expect_err("auth failure is a backend operation error");
|
.expect_err("auth failure is a backend operation error");
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
error,
|
error,
|
||||||
|
|||||||
@@ -43,6 +43,8 @@ pub use repositories::{
|
|||||||
pub use server::{AuthConfig, ServerConfig, WorkspaceApi, build_router, serve};
|
pub use server::{AuthConfig, ServerConfig, WorkspaceApi, build_router, serve};
|
||||||
pub use store::{ControlPlaneStore, SqliteWorkspaceStore, WorkspaceRecord};
|
pub use store::{ControlPlaneStore, SqliteWorkspaceStore, WorkspaceRecord};
|
||||||
|
|
||||||
|
use worker_runtime::identity::RuntimeWorkerRef;
|
||||||
|
|
||||||
pub type Result<T> = std::result::Result<T, Error>;
|
pub type Result<T> = std::result::Result<T, Error>;
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
@@ -65,13 +67,12 @@ pub enum Error {
|
|||||||
UnknownHost(String),
|
UnknownHost(String),
|
||||||
#[error("unknown runtime `{0}`")]
|
#[error("unknown runtime `{0}`")]
|
||||||
UnknownRuntime(String),
|
UnknownRuntime(String),
|
||||||
#[error("unknown worker `{worker_id}` in runtime `{runtime_id}`")]
|
#[error("unknown worker `{}` in runtime `{}`", worker.worker_id, worker.runtime_id)]
|
||||||
UnknownWorker {
|
UnknownWorker { worker: RuntimeWorkerRef },
|
||||||
runtime_id: String,
|
|
||||||
worker_id: String,
|
|
||||||
},
|
|
||||||
#[error("invalid runtime {kind} `{value}`")]
|
#[error("invalid runtime {kind} `{value}`")]
|
||||||
InvalidRuntimeIdentifier { kind: String, value: String },
|
InvalidRuntimeIdentifier { kind: String, value: String },
|
||||||
|
#[error("worker name is reserved for a dedicated Workspace service: {0}")]
|
||||||
|
ReservedWorkerName(String),
|
||||||
#[error("runtime `{runtime_id}` operation failed ({code}): {message}")]
|
#[error("runtime `{runtime_id}` operation failed ({code}): {message}")]
|
||||||
RuntimeOperationFailed {
|
RuntimeOperationFailed {
|
||||||
runtime_id: String,
|
runtime_id: String,
|
||||||
@@ -89,6 +90,10 @@ pub enum Error {
|
|||||||
WorkspaceIdMismatch,
|
WorkspaceIdMismatch,
|
||||||
#[error("Ticket assignment conflict: {0}")]
|
#[error("Ticket assignment conflict: {0}")]
|
||||||
TicketAssignmentConflict(String),
|
TicketAssignmentConflict(String),
|
||||||
|
#[error("Workdir attachment conflict: {0}")]
|
||||||
|
WorkdirAttachmentConflict(String),
|
||||||
|
#[error("Registry inconsistency: {0}")]
|
||||||
|
RegistryInconsistency(String),
|
||||||
#[error("Worker source identity is invalid: {0}")]
|
#[error("Worker source identity is invalid: {0}")]
|
||||||
WorkerSourceIdentity(String),
|
WorkerSourceIdentity(String),
|
||||||
#[error("workspace identity error: {0}")]
|
#[error("workspace identity error: {0}")]
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use std::collections::{BTreeMap, VecDeque};
|
use std::collections::{BTreeMap, VecDeque};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use worker_runtime::identity::WorkerRef;
|
use worker_runtime::identity::{RuntimeWorkerRef, WorkerRef};
|
||||||
use worker_runtime::observation::{WorkerObservationCursor, WorkerObservationEvent};
|
use worker_runtime::observation::{WorkerObservationCursor, WorkerObservationEvent};
|
||||||
|
|
||||||
use axum::http::StatusCode;
|
use axum::http::StatusCode;
|
||||||
@@ -15,8 +15,7 @@ use tokio_tungstenite::tungstenite::{Error as TungsteniteError, Message as Tungs
|
|||||||
/// Backend-private source for a runtime worker observation stream.
|
/// Backend-private source for a runtime worker observation stream.
|
||||||
#[derive(Clone, PartialEq, Eq)]
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
pub struct RuntimeObservationSourceConfig {
|
pub struct RuntimeObservationSourceConfig {
|
||||||
pub runtime_id: String,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub worker_id: String,
|
|
||||||
pub endpoint: String,
|
pub endpoint: String,
|
||||||
pub bearer_token: Option<String>,
|
pub bearer_token: Option<String>,
|
||||||
}
|
}
|
||||||
@@ -24,8 +23,8 @@ pub struct RuntimeObservationSourceConfig {
|
|||||||
impl std::fmt::Debug for RuntimeObservationSourceConfig {
|
impl std::fmt::Debug for RuntimeObservationSourceConfig {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
f.debug_struct("RuntimeObservationSourceConfig")
|
f.debug_struct("RuntimeObservationSourceConfig")
|
||||||
.field("runtime_id", &self.runtime_id)
|
.field("runtime_id", &self.worker.runtime_id)
|
||||||
.field("worker_id", &self.worker_id)
|
.field("worker_id", &self.worker.worker_id)
|
||||||
.field("endpoint", &"<backend-private>")
|
.field("endpoint", &"<backend-private>")
|
||||||
.field(
|
.field(
|
||||||
"bearer_token",
|
"bearer_token",
|
||||||
@@ -37,8 +36,7 @@ impl std::fmt::Debug for RuntimeObservationSourceConfig {
|
|||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct EmbeddedRuntimeObservationSource {
|
pub struct EmbeddedRuntimeObservationSource {
|
||||||
pub runtime_id: String,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub worker_id: String,
|
|
||||||
pub runtime: worker_runtime::Runtime,
|
pub runtime: worker_runtime::Runtime,
|
||||||
pub worker_ref: WorkerRef,
|
pub worker_ref: WorkerRef,
|
||||||
}
|
}
|
||||||
@@ -60,15 +58,15 @@ impl RuntimeObservationSource {
|
|||||||
|
|
||||||
pub fn runtime_id(&self) -> &str {
|
pub fn runtime_id(&self) -> &str {
|
||||||
match self {
|
match self {
|
||||||
Self::RemoteWs(config) => &config.runtime_id,
|
Self::RemoteWs(config) => &config.worker.runtime_id,
|
||||||
Self::Embedded(source) => &source.runtime_id,
|
Self::Embedded(source) => &source.worker.runtime_id,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn worker_id(&self) -> &str {
|
pub fn worker_id(&self) -> &str {
|
||||||
match self {
|
match self {
|
||||||
Self::RemoteWs(config) => &config.worker_id,
|
Self::RemoteWs(config) => &config.worker.worker_id,
|
||||||
Self::Embedded(source) => &source.worker_id,
|
Self::Embedded(source) => &source.worker.worker_id,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -78,8 +76,8 @@ impl std::fmt::Debug for RuntimeObservationSource {
|
|||||||
match self {
|
match self {
|
||||||
Self::RemoteWs(config) => formatter
|
Self::RemoteWs(config) => formatter
|
||||||
.debug_struct("RemoteRuntimeObservationSource")
|
.debug_struct("RemoteRuntimeObservationSource")
|
||||||
.field("runtime_id", &config.runtime_id)
|
.field("runtime_id", &config.worker.runtime_id)
|
||||||
.field("worker_id", &config.worker_id)
|
.field("worker_id", &config.worker.worker_id)
|
||||||
.field("endpoint", &"<backend-private>")
|
.field("endpoint", &"<backend-private>")
|
||||||
.field(
|
.field(
|
||||||
"bearer_token",
|
"bearer_token",
|
||||||
@@ -88,8 +86,8 @@ impl std::fmt::Debug for RuntimeObservationSource {
|
|||||||
.finish(),
|
.finish(),
|
||||||
Self::Embedded(source) => formatter
|
Self::Embedded(source) => formatter
|
||||||
.debug_struct("EmbeddedRuntimeObservationSource")
|
.debug_struct("EmbeddedRuntimeObservationSource")
|
||||||
.field("runtime_id", &source.runtime_id)
|
.field("runtime_id", &source.worker.runtime_id)
|
||||||
.field("worker_id", &source.worker_id)
|
.field("worker_id", &source.worker.worker_id)
|
||||||
.finish(),
|
.finish(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -98,8 +96,8 @@ impl std::fmt::Debug for RuntimeObservationSource {
|
|||||||
/// Event consumed from a Runtime-owned worker observation WebSocket.
|
/// Event consumed from a Runtime-owned worker observation WebSocket.
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct RuntimeObservationUpstreamEvent {
|
pub struct RuntimeObservationUpstreamEvent {
|
||||||
pub runtime_id: String,
|
#[serde(flatten)]
|
||||||
pub worker_id: String,
|
pub worker: RuntimeWorkerRef,
|
||||||
pub runtime_event_id: String,
|
pub runtime_event_id: String,
|
||||||
pub payload: protocol::Event,
|
pub payload: protocol::Event,
|
||||||
}
|
}
|
||||||
@@ -132,11 +130,7 @@ impl ObservationProxyError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
type ObservationKey = RuntimeWorkerRef;
|
||||||
struct ObservationKey {
|
|
||||||
runtime_id: String,
|
|
||||||
worker_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Backend-owned in-memory v0 observation proxy state.
|
/// Backend-owned in-memory v0 observation proxy state.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -156,15 +150,7 @@ impl BackendObservationProxy {
|
|||||||
pub fn new(sources: Vec<RuntimeObservationSourceConfig>) -> Self {
|
pub fn new(sources: Vec<RuntimeObservationSourceConfig>) -> Self {
|
||||||
let sources = sources
|
let sources = sources
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|source| {
|
.map(|source| (source.worker.clone(), source))
|
||||||
(
|
|
||||||
ObservationKey {
|
|
||||||
runtime_id: source.runtime_id.clone(),
|
|
||||||
worker_id: source.worker_id.clone(),
|
|
||||||
},
|
|
||||||
source,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect();
|
.collect();
|
||||||
Self {
|
Self {
|
||||||
sources: Arc::new(sources),
|
sources: Arc::new(sources),
|
||||||
@@ -173,19 +159,16 @@ impl BackendObservationProxy {
|
|||||||
|
|
||||||
pub fn source(
|
pub fn source(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
) -> Result<RuntimeObservationSource, ObservationProxyError> {
|
) -> Result<RuntimeObservationSource, ObservationProxyError> {
|
||||||
self.sources
|
self.sources
|
||||||
.get(&ObservationKey {
|
.get(worker)
|
||||||
runtime_id: runtime_id.to_string(),
|
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
})
|
|
||||||
.cloned()
|
.cloned()
|
||||||
.map(RuntimeObservationSource::remote_ws)
|
.map(RuntimeObservationSource::remote_ws)
|
||||||
.ok_or_else(|| {
|
.ok_or_else(|| {
|
||||||
ObservationProxyError::WorkerNotFound(format!(
|
ObservationProxyError::WorkerNotFound(format!(
|
||||||
"worker {worker_id} is not registered for runtime {runtime_id}"
|
"worker {} is not registered for runtime {}",
|
||||||
|
worker.worker_id, worker.runtime_id
|
||||||
))
|
))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -209,8 +192,7 @@ fn map_runtime_connect_error(error: TungsteniteError) -> ObservationProxyError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub struct RuntimeWsObservationClient {
|
pub struct RuntimeWsObservationClient {
|
||||||
runtime_id: String,
|
worker: RuntimeWorkerRef,
|
||||||
worker_id: String,
|
|
||||||
stream: tokio_tungstenite::WebSocketStream<
|
stream: tokio_tungstenite::WebSocketStream<
|
||||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||||
>,
|
>,
|
||||||
@@ -240,8 +222,7 @@ impl RuntimeWsObservationClient {
|
|||||||
.await
|
.await
|
||||||
.map_err(map_runtime_connect_error)?;
|
.map_err(map_runtime_connect_error)?;
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
runtime_id: source.runtime_id.clone(),
|
worker: source.worker.clone(),
|
||||||
worker_id: source.worker_id.clone(),
|
|
||||||
stream,
|
stream,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -291,8 +272,7 @@ impl RuntimeWsObservationClient {
|
|||||||
))
|
))
|
||||||
})?;
|
})?;
|
||||||
return Ok(RuntimeObservationUpstreamEvent {
|
return Ok(RuntimeObservationUpstreamEvent {
|
||||||
runtime_id: self.runtime_id.clone(),
|
worker: self.worker.clone(),
|
||||||
worker_id: self.worker_id.clone(),
|
|
||||||
runtime_event_id: "protocol".to_string(),
|
runtime_event_id: "protocol".to_string(),
|
||||||
payload,
|
payload,
|
||||||
});
|
});
|
||||||
@@ -330,8 +310,7 @@ impl RuntimeObservationClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub struct EmbeddedObservationClient {
|
pub struct EmbeddedObservationClient {
|
||||||
runtime_id: String,
|
worker: RuntimeWorkerRef,
|
||||||
worker_id: String,
|
|
||||||
worker_ref: WorkerRef,
|
worker_ref: WorkerRef,
|
||||||
cursor: WorkerObservationCursor,
|
cursor: WorkerObservationCursor,
|
||||||
receiver: tokio::sync::broadcast::Receiver<WorkerObservationEvent>,
|
receiver: tokio::sync::broadcast::Receiver<WorkerObservationEvent>,
|
||||||
@@ -346,7 +325,7 @@ impl EmbeddedObservationClient {
|
|||||||
.map_err(|err| {
|
.map_err(|err| {
|
||||||
ObservationProxyError::WorkerNotFound(format!(
|
ObservationProxyError::WorkerNotFound(format!(
|
||||||
"embedded Worker '{}' is not observable: {err}",
|
"embedded Worker '{}' is not observable: {err}",
|
||||||
source.worker_id
|
source.worker.worker_id
|
||||||
))
|
))
|
||||||
})?;
|
})?;
|
||||||
let receiver = source
|
let receiver = source
|
||||||
@@ -355,7 +334,7 @@ impl EmbeddedObservationClient {
|
|||||||
.map_err(|err| {
|
.map_err(|err| {
|
||||||
ObservationProxyError::WorkerNotFound(format!(
|
ObservationProxyError::WorkerNotFound(format!(
|
||||||
"embedded Worker '{}' observation subscription is unavailable: {err}",
|
"embedded Worker '{}' observation subscription is unavailable: {err}",
|
||||||
source.worker_id
|
source.worker.worker_id
|
||||||
))
|
))
|
||||||
})?;
|
})?;
|
||||||
let mut queued = VecDeque::new();
|
let mut queued = VecDeque::new();
|
||||||
@@ -365,12 +344,11 @@ impl EmbeddedObservationClient {
|
|||||||
.map_err(|err| {
|
.map_err(|err| {
|
||||||
ObservationProxyError::WorkerNotFound(format!(
|
ObservationProxyError::WorkerNotFound(format!(
|
||||||
"embedded Worker '{}' snapshot is unavailable: {err}",
|
"embedded Worker '{}' snapshot is unavailable: {err}",
|
||||||
source.worker_id
|
source.worker.worker_id
|
||||||
))
|
))
|
||||||
})?;
|
})?;
|
||||||
queued.push_back(RuntimeObservationUpstreamEvent {
|
queued.push_back(RuntimeObservationUpstreamEvent {
|
||||||
runtime_id: source.runtime_id.clone(),
|
worker: source.worker.clone(),
|
||||||
worker_id: source.worker_id.clone(),
|
|
||||||
runtime_event_id: "snapshot".to_string(),
|
runtime_event_id: "snapshot".to_string(),
|
||||||
payload: snapshot,
|
payload: snapshot,
|
||||||
});
|
});
|
||||||
@@ -380,19 +358,14 @@ impl EmbeddedObservationClient {
|
|||||||
.map_err(|err| {
|
.map_err(|err| {
|
||||||
ObservationProxyError::RuntimeUnavailable(format!(
|
ObservationProxyError::RuntimeUnavailable(format!(
|
||||||
"embedded Worker '{}' observation cursor is unavailable: {err}",
|
"embedded Worker '{}' observation cursor is unavailable: {err}",
|
||||||
source.worker_id
|
source.worker.worker_id
|
||||||
))
|
))
|
||||||
})?
|
})?
|
||||||
{
|
{
|
||||||
queued.push_back(Self::map_event(
|
queued.push_back(Self::map_event(&source.worker, event));
|
||||||
&source.runtime_id,
|
|
||||||
&source.worker_id,
|
|
||||||
event,
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
runtime_id: source.runtime_id.clone(),
|
worker: source.worker.clone(),
|
||||||
worker_id: source.worker_id.clone(),
|
|
||||||
worker_ref: source.worker_ref.clone(),
|
worker_ref: source.worker_ref.clone(),
|
||||||
cursor,
|
cursor,
|
||||||
receiver,
|
receiver,
|
||||||
@@ -418,7 +391,7 @@ impl EmbeddedObservationClient {
|
|||||||
"embedded runtime emitted a malformed cursor".into(),
|
"embedded runtime emitted a malformed cursor".into(),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
return Ok(Self::map_event(&self.runtime_id, &self.worker_id, event));
|
return Ok(Self::map_event(&self.worker, event));
|
||||||
}
|
}
|
||||||
Ok(_) => continue,
|
Ok(_) => continue,
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
|
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
|
||||||
@@ -436,13 +409,11 @@ impl EmbeddedObservationClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn map_event(
|
fn map_event(
|
||||||
runtime_id: &str,
|
worker: &RuntimeWorkerRef,
|
||||||
worker_id: &str,
|
|
||||||
event: WorkerObservationEvent,
|
event: WorkerObservationEvent,
|
||||||
) -> RuntimeObservationUpstreamEvent {
|
) -> RuntimeObservationUpstreamEvent {
|
||||||
RuntimeObservationUpstreamEvent {
|
RuntimeObservationUpstreamEvent {
|
||||||
runtime_id: runtime_id.to_string(),
|
worker: worker.clone(),
|
||||||
worker_id: worker_id.to_string(),
|
|
||||||
runtime_event_id: event.cursor.clone(),
|
runtime_event_id: event.cursor.clone(),
|
||||||
payload: event.payload,
|
payload: event.payload,
|
||||||
}
|
}
|
||||||
@@ -455,8 +426,7 @@ mod tests {
|
|||||||
|
|
||||||
fn sensitive_source() -> RuntimeObservationSourceConfig {
|
fn sensitive_source() -> RuntimeObservationSourceConfig {
|
||||||
RuntimeObservationSourceConfig {
|
RuntimeObservationSourceConfig {
|
||||||
runtime_id: "remote-runtime".to_string(),
|
worker: RuntimeWorkerRef::new("remote-runtime", "worker-1"),
|
||||||
worker_id: "worker-1".to_string(),
|
|
||||||
endpoint: "wss://remote.example.invalid/private/workers/worker-1/protocol/ws"
|
endpoint: "wss://remote.example.invalid/private/workers/worker-1/protocol/ws"
|
||||||
.to_string(),
|
.to_string(),
|
||||||
bearer_token: Some("top-secret-bearer-token".to_string()),
|
bearer_token: Some("top-secret-bearer-token".to_string()),
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ use chrono::{Duration, Utc};
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use worker_runtime::identity::WorkerId;
|
use worker_runtime::identity::RuntimeWorkerRef;
|
||||||
use worker_runtime::profile_archive::ProfileSourceArchive;
|
use worker_runtime::profile_archive::ProfileSourceArchive;
|
||||||
use worker_runtime::resource::{
|
use worker_runtime::resource::{
|
||||||
BackendResourceClient, BackendResourceError, BackendResourceFetchRequest,
|
BackendResourceClient, BackendResourceError, BackendResourceFetchRequest,
|
||||||
@@ -17,10 +17,17 @@ pub struct BackendResourceBroker {
|
|||||||
resources: Arc<Mutex<HashMap<String, StoredResource>>>,
|
resources: Arc<Mutex<HashMap<String, StoredResource>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
pub enum BackendResourceTarget<'a> {
|
||||||
|
Workspace,
|
||||||
|
Runtime(&'a str),
|
||||||
|
Worker(&'a RuntimeWorkerRef),
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct StoredResource {
|
struct StoredResource {
|
||||||
runtime_id: Option<String>,
|
runtime_id: Option<String>,
|
||||||
worker_id: Option<String>,
|
worker: Option<RuntimeWorkerRef>,
|
||||||
handle: BackendResourceHandle,
|
handle: BackendResourceHandle,
|
||||||
archive: ProfileSourceArchive,
|
archive: ProfileSourceArchive,
|
||||||
}
|
}
|
||||||
@@ -29,11 +36,17 @@ impl BackendResourceBroker {
|
|||||||
pub fn issue_profile_source_archive_handle(
|
pub fn issue_profile_source_archive_handle(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: impl Into<String>,
|
workspace_id: impl Into<String>,
|
||||||
runtime_id: Option<&str>,
|
target: BackendResourceTarget<'_>,
|
||||||
worker_id: Option<&WorkerId>,
|
|
||||||
archive: ProfileSourceArchive,
|
archive: ProfileSourceArchive,
|
||||||
) -> BackendResourceHandle {
|
) -> BackendResourceHandle {
|
||||||
let workspace_id = workspace_id.into();
|
let workspace_id = workspace_id.into();
|
||||||
|
let (runtime_id, worker) = match target {
|
||||||
|
BackendResourceTarget::Workspace => (None, None),
|
||||||
|
BackendResourceTarget::Runtime(runtime_id) => (Some(runtime_id.to_string()), None),
|
||||||
|
BackendResourceTarget::Worker(worker) => {
|
||||||
|
(Some(worker.runtime_id.clone()), Some(worker.clone()))
|
||||||
|
}
|
||||||
|
};
|
||||||
let nonce = Uuid::now_v7().to_string();
|
let nonce = Uuid::now_v7().to_string();
|
||||||
let audit_correlation_id = format!("resource-fetch-{nonce}");
|
let audit_correlation_id = format!("resource-fetch-{nonce}");
|
||||||
let expires_at = Utc::now() + Duration::minutes(15);
|
let expires_at = Utc::now() + Duration::minutes(15);
|
||||||
@@ -41,8 +54,8 @@ impl BackendResourceBroker {
|
|||||||
kind: BackendResourceKind::ProfileSourceArchive,
|
kind: BackendResourceKind::ProfileSourceArchive,
|
||||||
workspace_id: workspace_id.clone(),
|
workspace_id: workspace_id.clone(),
|
||||||
scope_id: Some("workspace-profile-source".to_string()),
|
scope_id: Some("workspace-profile-source".to_string()),
|
||||||
runtime_id: runtime_id.map(|id| id.to_string()),
|
runtime_id: runtime_id.clone(),
|
||||||
worker_id: worker_id.map(|id| id.to_string()),
|
worker_id: worker.as_ref().map(|worker| worker.worker_id.clone()),
|
||||||
resource_id: archive.reference.id.clone(),
|
resource_id: archive.reference.id.clone(),
|
||||||
digest: archive.reference.digest.clone(),
|
digest: archive.reference.digest.clone(),
|
||||||
operation: BackendResourceOperation::FetchArchive,
|
operation: BackendResourceOperation::FetchArchive,
|
||||||
@@ -57,8 +70,8 @@ impl BackendResourceBroker {
|
|||||||
profile_source_graph: Some(archive.reference.source_graph.clone()),
|
profile_source_graph: Some(archive.reference.source_graph.clone()),
|
||||||
};
|
};
|
||||||
let stored = StoredResource {
|
let stored = StoredResource {
|
||||||
runtime_id: runtime_id.map(|id| id.to_string()),
|
runtime_id,
|
||||||
worker_id: worker_id.map(|id| id.to_string()),
|
worker,
|
||||||
handle: handle.clone(),
|
handle: handle.clone(),
|
||||||
archive,
|
archive,
|
||||||
};
|
};
|
||||||
@@ -117,8 +130,10 @@ impl BackendResourceBroker {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(expected_worker_id) = stored.worker_id.as_deref() {
|
if let Some(expected_worker) = stored.worker.as_ref() {
|
||||||
if Some(expected_worker_id) != request.worker_id.as_deref() {
|
if expected_worker.runtime_id != request.runtime_id
|
||||||
|
|| Some(expected_worker.worker_id.as_str()) != request.worker_id.as_deref()
|
||||||
|
{
|
||||||
return Err(BackendResourceError::Unauthorized {
|
return Err(BackendResourceError::Unauthorized {
|
||||||
message: "worker id does not match resource handle".to_string(),
|
message: "worker id does not match resource handle".to_string(),
|
||||||
});
|
});
|
||||||
@@ -167,7 +182,6 @@ fn verify_handle_shape(handle: &BackendResourceHandle) -> Result<(), BackendReso
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use worker_runtime::identity::WorkerId;
|
|
||||||
use worker_runtime::profile_archive::{
|
use worker_runtime::profile_archive::{
|
||||||
ProfileSourceArchive, ProfileSourceArchiveRef, ProfileSourceGraphSummary, sha256_hex,
|
ProfileSourceArchive, ProfileSourceArchiveRef, ProfileSourceGraphSummary, sha256_hex,
|
||||||
};
|
};
|
||||||
@@ -215,13 +229,13 @@ mod tests {
|
|||||||
fn request(
|
fn request(
|
||||||
handle: BackendResourceHandle,
|
handle: BackendResourceHandle,
|
||||||
runtime_id: &str,
|
runtime_id: &str,
|
||||||
worker_id: Option<&WorkerId>,
|
worker_id: Option<&str>,
|
||||||
) -> BackendResourceFetchRequest {
|
) -> BackendResourceFetchRequest {
|
||||||
BackendResourceFetchRequest {
|
BackendResourceFetchRequest {
|
||||||
audit_correlation_id: handle.audit_correlation_id.clone(),
|
audit_correlation_id: handle.audit_correlation_id.clone(),
|
||||||
handle,
|
handle,
|
||||||
runtime_id: runtime_id.to_string(),
|
runtime_id: runtime_id.to_string(),
|
||||||
worker_id: worker_id.map(|id| id.to_string()),
|
worker_id: worker_id.map(str::to_string),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -231,8 +245,7 @@ mod tests {
|
|||||||
let runtime_id = "runtime-test";
|
let runtime_id = "runtime-test";
|
||||||
let handle = broker.issue_profile_source_archive_handle(
|
let handle = broker.issue_profile_source_archive_handle(
|
||||||
"workspace-test",
|
"workspace-test",
|
||||||
Some(runtime_id),
|
BackendResourceTarget::Runtime(runtime_id),
|
||||||
None,
|
|
||||||
archive(),
|
archive(),
|
||||||
);
|
);
|
||||||
let response = broker
|
let response = broker
|
||||||
@@ -253,8 +266,7 @@ mod tests {
|
|||||||
let runtime_a = "runtime-a";
|
let runtime_a = "runtime-a";
|
||||||
let handle = broker.issue_profile_source_archive_handle(
|
let handle = broker.issue_profile_source_archive_handle(
|
||||||
"workspace-test",
|
"workspace-test",
|
||||||
Some(runtime_a),
|
BackendResourceTarget::Runtime(runtime_a),
|
||||||
None,
|
|
||||||
archive(),
|
archive(),
|
||||||
);
|
);
|
||||||
let err = broker
|
let err = broker
|
||||||
@@ -267,16 +279,15 @@ mod tests {
|
|||||||
fn broker_rejects_worker_mismatch() {
|
fn broker_rejects_worker_mismatch() {
|
||||||
let broker = BackendResourceBroker::default();
|
let broker = BackendResourceBroker::default();
|
||||||
let runtime_id = "runtime-test";
|
let runtime_id = "runtime-test";
|
||||||
let worker_a = WorkerId::new(1);
|
let worker_a = RuntimeWorkerRef::new(runtime_id, "1");
|
||||||
let worker_b = WorkerId::new(2);
|
let worker_b = RuntimeWorkerRef::new(runtime_id, "2");
|
||||||
let handle = broker.issue_profile_source_archive_handle(
|
let handle = broker.issue_profile_source_archive_handle(
|
||||||
"workspace-test",
|
"workspace-test",
|
||||||
Some(runtime_id),
|
BackendResourceTarget::Worker(&worker_a),
|
||||||
Some(&worker_a),
|
|
||||||
archive(),
|
archive(),
|
||||||
);
|
);
|
||||||
let err = broker
|
let err = broker
|
||||||
.fetch_profile_source_archive(request(handle, &runtime_id, Some(&worker_b)))
|
.fetch_profile_source_archive(request(handle, runtime_id, Some(&worker_b.worker_id)))
|
||||||
.unwrap_err();
|
.unwrap_err();
|
||||||
assert!(matches!(err, BackendResourceError::Unauthorized { .. }));
|
assert!(matches!(err, BackendResourceError::Unauthorized { .. }));
|
||||||
}
|
}
|
||||||
@@ -287,8 +298,7 @@ mod tests {
|
|||||||
let runtime_id = "runtime-test";
|
let runtime_id = "runtime-test";
|
||||||
let handle = broker.issue_profile_source_archive_handle(
|
let handle = broker.issue_profile_source_archive_handle(
|
||||||
"workspace-test",
|
"workspace-test",
|
||||||
Some(runtime_id),
|
BackendResourceTarget::Runtime(runtime_id),
|
||||||
None,
|
|
||||||
archive(),
|
archive(),
|
||||||
);
|
);
|
||||||
broker
|
broker
|
||||||
@@ -313,8 +323,7 @@ mod tests {
|
|||||||
let runtime_id = "runtime-test";
|
let runtime_id = "runtime-test";
|
||||||
let mut handle = broker.issue_profile_source_archive_handle(
|
let mut handle = broker.issue_profile_source_archive_handle(
|
||||||
"workspace-test",
|
"workspace-test",
|
||||||
Some(runtime_id),
|
BackendResourceTarget::Runtime(runtime_id),
|
||||||
None,
|
|
||||||
archive(),
|
archive(),
|
||||||
);
|
);
|
||||||
handle.scope_id = Some("tampered-scope".to_string());
|
handle.scope_id = Some("tampered-scope".to_string());
|
||||||
@@ -331,8 +340,7 @@ mod tests {
|
|||||||
let archive = archive_with_len((DEFAULT_PROFILE_SOURCE_ARCHIVE_MAX_BYTES + 1) as usize);
|
let archive = archive_with_len((DEFAULT_PROFILE_SOURCE_ARCHIVE_MAX_BYTES + 1) as usize);
|
||||||
let mut handle = broker.issue_profile_source_archive_handle(
|
let mut handle = broker.issue_profile_source_archive_handle(
|
||||||
"workspace-test",
|
"workspace-test",
|
||||||
Some(runtime_id),
|
BackendResourceTarget::Runtime(runtime_id),
|
||||||
None,
|
|
||||||
archive,
|
archive,
|
||||||
);
|
);
|
||||||
handle.max_bytes = DEFAULT_PROFILE_SOURCE_ARCHIVE_MAX_BYTES + 1024;
|
handle.max_bytes = DEFAULT_PROFILE_SOURCE_ARCHIVE_MAX_BYTES + 1024;
|
||||||
|
|||||||
@@ -418,6 +418,8 @@ async fn run_embedded_connection(
|
|||||||
update = update_receiver.recv() => {
|
update = update_receiver.recv() => {
|
||||||
let Some((selector, subject_revision, payload)) = update else { return; };
|
let Some((selector, subject_revision, payload)) = update else { return; };
|
||||||
if let Some(entry) = entries.get_mut(&selector) {
|
if let Some(entry) = entries.get_mut(&selector) {
|
||||||
|
entry.snapshot_revision = entry.snapshot_revision.saturating_add(1);
|
||||||
|
apply_event_to_cached_snapshot(&mut entry.snapshot, &payload);
|
||||||
broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Event { connection_generation: generation, subject_revision, payload });
|
broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Event { connection_generation: generation, subject_revision, payload });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -431,6 +433,31 @@ async fn run_embedded_connection(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_event_to_cached_snapshot(
|
||||||
|
snapshot: &mut SubscriptionSnapshot,
|
||||||
|
payload: &SubscriptionEventPayload,
|
||||||
|
) {
|
||||||
|
let SubscriptionSnapshot::Workers { workers } = snapshot else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match payload {
|
||||||
|
SubscriptionEventPayload::WorkerUpserted { worker } => {
|
||||||
|
if let Some(existing) = workers
|
||||||
|
.iter_mut()
|
||||||
|
.find(|existing| existing.worker_id == worker.worker_id)
|
||||||
|
{
|
||||||
|
*existing = worker.clone();
|
||||||
|
} else {
|
||||||
|
workers.push(worker.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SubscriptionEventPayload::WorkerRemoved { worker_id, .. } => {
|
||||||
|
workers.retain(|worker| worker.worker_id != *worker_id);
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn project_snapshot_runtime(
|
fn project_snapshot_runtime(
|
||||||
mut snapshot: SubscriptionSnapshot,
|
mut snapshot: SubscriptionSnapshot,
|
||||||
runtime_id: &str,
|
runtime_id: &str,
|
||||||
@@ -737,6 +764,10 @@ async fn handle_frame(
|
|||||||
}
|
}
|
||||||
*revision = subject_revision;
|
*revision = subject_revision;
|
||||||
}
|
}
|
||||||
|
if let Some((snapshot_revision, snapshot)) = entry.snapshot.as_mut() {
|
||||||
|
*snapshot_revision = snapshot_revision.saturating_add(1);
|
||||||
|
apply_event_to_cached_snapshot(snapshot, &payload);
|
||||||
|
}
|
||||||
broadcast(
|
broadcast(
|
||||||
&mut entry.downstreams,
|
&mut entry.downstreams,
|
||||||
BrokerSubscriptionEvent::Event {
|
BrokerSubscriptionEvent::Event {
|
||||||
|
|||||||
@@ -36,6 +36,13 @@ impl WorkerExecutionBackend for TestExecutionBackend {
|
|||||||
WorkerExecutionRunState::Busy,
|
WorkerExecutionRunState::Busy,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||||
|
WorkerExecutionResult::accepted(
|
||||||
|
WorkerExecutionOperation::Stop,
|
||||||
|
WorkerExecutionRunState::Stopped,
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const TOKEN: &str = "runtime-subscription-test-token";
|
const TOKEN: &str = "runtime-subscription-test-token";
|
||||||
@@ -156,7 +163,7 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
|||||||
.unwrap(),
|
.unwrap(),
|
||||||
};
|
};
|
||||||
let mut first = broker.subscribe("runtime-test", selector.clone()).unwrap();
|
let mut first = broker.subscribe("runtime-test", selector.clone()).unwrap();
|
||||||
let mut second = broker.subscribe("runtime-test", selector).unwrap();
|
let mut second = broker.subscribe("runtime-test", selector.clone()).unwrap();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
next_snapshot(&mut first).await,
|
next_snapshot(&mut first).await,
|
||||||
BrokerSubscriptionEvent::Snapshot { .. }
|
BrokerSubscriptionEvent::Snapshot { .. }
|
||||||
@@ -188,6 +195,16 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
|||||||
} if worker.state == SubscriptionWorkerState::Running
|
} if worker.state == SubscriptionWorkerState::Running
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap();
|
||||||
|
let BrokerSubscriptionEvent::Snapshot { snapshot, .. } = next_snapshot(&mut late).await else {
|
||||||
|
panic!("expected cached snapshot for late subscriber");
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
snapshot,
|
||||||
|
SubscriptionSnapshot::Workers { workers }
|
||||||
|
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running)
|
||||||
|
));
|
||||||
|
drop(late);
|
||||||
|
|
||||||
drop(first);
|
drop(first);
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
@@ -312,5 +329,53 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
|
|||||||
assert!(matches!(next_event(&mut subscription).await,
|
assert!(matches!(next_event(&mut subscription).await,
|
||||||
BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. }
|
BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. }
|
||||||
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") && worker.state == SubscriptionWorkerState::Running));
|
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") && worker.state == SubscriptionWorkerState::Running));
|
||||||
|
let mut late = broker
|
||||||
|
.subscribe(
|
||||||
|
"embedded-worker-runtime",
|
||||||
|
EventSubscriptionSelector::RuntimeWorkers,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let BrokerSubscriptionEvent::Snapshot { snapshot, .. } = next_snapshot(&mut late).await else {
|
||||||
|
panic!("expected cached embedded snapshot for late subscriber");
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
snapshot,
|
||||||
|
SubscriptionSnapshot::Workers { workers }
|
||||||
|
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running)
|
||||||
|
));
|
||||||
|
|
||||||
|
runtime
|
||||||
|
.stop_worker(&worker.worker_ref, Some("done".to_string()))
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
next_event(&mut subscription).await,
|
||||||
|
BrokerSubscriptionEvent::Event {
|
||||||
|
payload: SubscriptionEventPayload::WorkerUpserted { .. },
|
||||||
|
..
|
||||||
|
}
|
||||||
|
));
|
||||||
|
runtime.delete_worker(&worker.worker_ref).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
next_event(&mut subscription).await,
|
||||||
|
BrokerSubscriptionEvent::Event {
|
||||||
|
payload: SubscriptionEventPayload::WorkerRemoved { .. },
|
||||||
|
..
|
||||||
|
}
|
||||||
|
));
|
||||||
|
let mut after_remove = broker
|
||||||
|
.subscribe(
|
||||||
|
"embedded-worker-runtime",
|
||||||
|
EventSubscriptionSelector::RuntimeWorkers,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let BrokerSubscriptionEvent::Snapshot { snapshot, .. } = next_snapshot(&mut after_remove).await
|
||||||
|
else {
|
||||||
|
panic!("expected cached embedded snapshot after remove");
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
snapshot,
|
||||||
|
SubscriptionSnapshot::Workers { workers }
|
||||||
|
if workers.iter().all(|candidate| candidate.worker_id.as_str() != worker.worker_ref.worker_id.to_string())
|
||||||
|
));
|
||||||
server.abort();
|
server.abort();
|
||||||
}
|
}
|
||||||
|
|||||||
+2107
-690
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -8,6 +8,7 @@ use protocol::subscription::{
|
|||||||
SubscriptionResponse, SubscriptionSnapshot, SubscriptionTerminationCode, SubscriptionWorker,
|
SubscriptionResponse, SubscriptionSnapshot, SubscriptionTerminationCode, SubscriptionWorker,
|
||||||
};
|
};
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
|
use worker_runtime::identity::RuntimeWorkerRef;
|
||||||
|
|
||||||
use crate::runtime_subscription::{BrokerSubscriptionEvent, RuntimeSubscriptionBroker};
|
use crate::runtime_subscription::{BrokerSubscriptionEvent, RuntimeSubscriptionBroker};
|
||||||
use crate::server::{WorkspaceApi, connect_workspace_worker_protocol};
|
use crate::server::{WorkspaceApi, connect_workspace_worker_protocol};
|
||||||
@@ -81,13 +82,8 @@ pub(crate) async fn serve_workspace_subscription(api: WorkspaceApi, socket: WebS
|
|||||||
worker_id,
|
worker_id,
|
||||||
runtime_id: Some(runtime_id),
|
runtime_id: Some(runtime_id),
|
||||||
} => {
|
} => {
|
||||||
match connect_workspace_worker_protocol(
|
let worker = RuntimeWorkerRef::new(&runtime_id, worker_id.as_str());
|
||||||
&api,
|
match connect_workspace_worker_protocol(&api, &worker).await {
|
||||||
&runtime_id,
|
|
||||||
worker_id.as_str(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(connection) => {
|
Ok(connection) => {
|
||||||
let methods = connection.methods.clone();
|
let methods = connection.methods.clone();
|
||||||
let task = tokio::spawn(run_worker_protocol(
|
let task = tokio::spawn(run_worker_protocol(
|
||||||
@@ -323,16 +319,15 @@ async fn run_workspace_workers(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut revisions = HashMap::<String, u64>::new();
|
let mut revisions = HashMap::<RuntimeWorkerRef, u64>::new();
|
||||||
let mut initial_workers = workers
|
let mut initial_workers = Vec::new();
|
||||||
.values_mut()
|
for (runtime_id, runtime) in &mut workers {
|
||||||
.flat_map(|runtime| runtime.values_mut())
|
for worker in runtime.values_mut() {
|
||||||
.map(|worker| {
|
let worker_ref = RuntimeWorkerRef::new(runtime_id, worker.worker_id.as_str());
|
||||||
let key = worker_key(worker.runtime_id.as_deref(), worker.worker_id.as_str());
|
worker.subject_revision = next_revision(&mut revisions, &worker_ref);
|
||||||
worker.subject_revision = next_revision(&mut revisions, &key);
|
initial_workers.push(worker.clone());
|
||||||
worker.clone()
|
}
|
||||||
})
|
}
|
||||||
.collect::<Vec<_>>();
|
|
||||||
sort_workers(&mut initial_workers);
|
sort_workers(&mut initial_workers);
|
||||||
if send_frame(
|
if send_frame(
|
||||||
&outbound,
|
&outbound,
|
||||||
@@ -359,8 +354,8 @@ async fn run_workspace_workers(
|
|||||||
BrokerSubscriptionEvent::Snapshot { snapshot, .. } => {
|
BrokerSubscriptionEvent::Snapshot { snapshot, .. } => {
|
||||||
let removed = workers.remove(&runtime_id).unwrap_or_default();
|
let removed = workers.remove(&runtime_id).unwrap_or_default();
|
||||||
for worker in removed.values() {
|
for worker in removed.values() {
|
||||||
let key = worker_key(Some(&runtime_id), worker.worker_id.as_str());
|
let worker_ref = RuntimeWorkerRef::new(&runtime_id, worker.worker_id.as_str());
|
||||||
let revision = next_revision(&mut revisions, &key);
|
let revision = next_revision(&mut revisions, &worker_ref);
|
||||||
if send_event(
|
if send_event(
|
||||||
&outbound,
|
&outbound,
|
||||||
&subscription_id,
|
&subscription_id,
|
||||||
@@ -379,8 +374,9 @@ async fn run_workspace_workers(
|
|||||||
install_snapshot(&mut workers, &runtime_id, snapshot);
|
install_snapshot(&mut workers, &runtime_id, snapshot);
|
||||||
if let Some(current) = workers.get_mut(&runtime_id) {
|
if let Some(current) = workers.get_mut(&runtime_id) {
|
||||||
for worker in current.values_mut() {
|
for worker in current.values_mut() {
|
||||||
let key = worker_key(Some(&runtime_id), worker.worker_id.as_str());
|
let worker_ref =
|
||||||
let revision = next_revision(&mut revisions, &key);
|
RuntimeWorkerRef::new(&runtime_id, worker.worker_id.as_str());
|
||||||
|
let revision = next_revision(&mut revisions, &worker_ref);
|
||||||
worker.subject_revision = revision;
|
worker.subject_revision = revision;
|
||||||
if send_event(
|
if send_event(
|
||||||
&outbound,
|
&outbound,
|
||||||
@@ -401,8 +397,8 @@ async fn run_workspace_workers(
|
|||||||
BrokerSubscriptionEvent::Event { payload, .. } => match payload {
|
BrokerSubscriptionEvent::Event { payload, .. } => match payload {
|
||||||
SubscriptionEventPayload::WorkerUpserted { mut worker } => {
|
SubscriptionEventPayload::WorkerUpserted { mut worker } => {
|
||||||
worker.runtime_id = Some(runtime_id.clone());
|
worker.runtime_id = Some(runtime_id.clone());
|
||||||
let key = worker_key(Some(&runtime_id), worker.worker_id.as_str());
|
let worker_ref = RuntimeWorkerRef::new(&runtime_id, worker.worker_id.as_str());
|
||||||
let revision = next_revision(&mut revisions, &key);
|
let revision = next_revision(&mut revisions, &worker_ref);
|
||||||
worker.subject_revision = revision;
|
worker.subject_revision = revision;
|
||||||
workers
|
workers
|
||||||
.entry(runtime_id)
|
.entry(runtime_id)
|
||||||
@@ -425,8 +421,8 @@ async fn run_workspace_workers(
|
|||||||
.entry(runtime_id.clone())
|
.entry(runtime_id.clone())
|
||||||
.or_default()
|
.or_default()
|
||||||
.remove(worker_id.as_str());
|
.remove(worker_id.as_str());
|
||||||
let key = worker_key(Some(&runtime_id), worker_id.as_str());
|
let worker_ref = RuntimeWorkerRef::new(&runtime_id, worker_id.as_str());
|
||||||
let revision = next_revision(&mut revisions, &key);
|
let revision = next_revision(&mut revisions, &worker_ref);
|
||||||
if send_event(
|
if send_event(
|
||||||
&outbound,
|
&outbound,
|
||||||
&subscription_id,
|
&subscription_id,
|
||||||
@@ -534,14 +530,11 @@ async fn send_frame(
|
|||||||
.map_err(|_| ())
|
.map_err(|_| ())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn next_revision(revisions: &mut HashMap<String, u64>, key: &str) -> u64 {
|
fn next_revision(revisions: &mut HashMap<RuntimeWorkerRef, u64>, worker: &RuntimeWorkerRef) -> u64 {
|
||||||
let revision = revisions.entry(key.to_string()).or_insert(0);
|
let revision = revisions.entry(worker.clone()).or_insert(0);
|
||||||
*revision = revision.saturating_add(1);
|
*revision = revision.saturating_add(1);
|
||||||
*revision
|
*revision
|
||||||
}
|
}
|
||||||
fn worker_key(runtime_id: Option<&str>, worker_id: &str) -> String {
|
|
||||||
format!("{}:{worker_id}", runtime_id.unwrap_or_default())
|
|
||||||
}
|
|
||||||
fn sort_workers(workers: &mut [SubscriptionWorker]) {
|
fn sort_workers(workers: &mut [SubscriptionWorker]) {
|
||||||
workers.sort_by(|left, right| {
|
workers.sort_by(|left, right| {
|
||||||
left.runtime_id
|
left.runtime_id
|
||||||
|
|||||||
@@ -65,9 +65,9 @@ name = "MCP_UPSTREAM_TOKEN"
|
|||||||
|
|
||||||
Local stdio MCP servers are ordinary local executables running with the user's OS permissions. Yoi's feature flags, Plugin permissions, and MCP config validation are not an operating-system sandbox and cannot prevent filesystem/network/process side effects once a later lifecycle implementation chooses to spawn a configured server.
|
Local stdio MCP servers are ordinary local executables running with the user's OS permissions. Yoi's feature flags, Plugin permissions, and MCP config validation are not an operating-system sandbox and cannot prevent filesystem/network/process side effects once a later lifecycle implementation chooses to spawn a configured server.
|
||||||
|
|
||||||
## Spawned Workers
|
## SubWorkers
|
||||||
|
|
||||||
`SpawnWorker.profile` is optional and resolves through defaults when omitted. The only concrete capability delegation in the tool call is `SpawnWorker.scope`, and it must be a subset of the parent's effective scope.
|
`SubWorkerSpawn.profile` is optional and resolves through defaults when omitted. The only concrete capability delegation in the tool call is `SubWorkerSpawn.scope`, and it must be a subset of the parent Worker's effective scope.
|
||||||
|
|
||||||
`inherit` derives reusable settings from the parent's resolved Manifest while replacing child identity and delegated scope. It should not blindly reuse the parent's original Profile source or runtime state.
|
`inherit` derives reusable settings from the parent's resolved Manifest while replacing child identity and delegated scope. It should not blindly reuse the parent's original Profile source or runtime state.
|
||||||
|
|
||||||
|
|||||||
@@ -253,7 +253,7 @@ Unless explicitly authorized otherwise, final merge, cleanup, design-boundary de
|
|||||||
|
|
||||||
Before closing, verify concrete evidence:
|
Before closing, verify concrete evidence:
|
||||||
|
|
||||||
- child Worker output via `ReadWorkerOutput`;
|
- SubWorker output via `SubWorkerReadOutput`;
|
||||||
- worktree state and diff;
|
- worktree state and diff;
|
||||||
- validation command output;
|
- validation command output;
|
||||||
- review result;
|
- review result;
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
# LAN の HTTP アクセスで `crypto.randomUUID()` が WebSocket 接続開始を阻害する
|
||||||
|
|
||||||
|
## 概要
|
||||||
|
|
||||||
|
Workspace WebUI の Vite dev server を `--host` 付きで LAN に公開し、別端末から
|
||||||
|
`http://192.168.1.32:5173` にアクセスすると、Worker Console の protocol 状態が
|
||||||
|
`connecting` のまま進まなかった。
|
||||||
|
|
||||||
|
原因は WebSocket や Vite proxy ではなく、WebSocket multiplexing の相関 ID 生成に
|
||||||
|
`crypto.randomUUID()` を直接使っていることである。`http://localhost` はブラウザから
|
||||||
|
potentially trustworthy origin として扱われる一方、LAN IP 上の平文 HTTP は secure
|
||||||
|
context ではない。`crypto.randomUUID()` は secure context 限定なので、LAN アクセスでは
|
||||||
|
接続処理が WebSocket の生成前に例外終了する。
|
||||||
|
|
||||||
|
## 発生経路
|
||||||
|
|
||||||
|
Worker Console の `connectProtocolTransport` は、最初に `protocolState` を
|
||||||
|
`"connecting"` に設定してから `WorkspaceMultiplexer.subscribe()` を呼ぶ。
|
||||||
|
|
||||||
|
- `web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte`
|
||||||
|
- `protocolState = "connecting"`
|
||||||
|
- 直後に `workspaceMultiplexer(...).subscribe(...)`
|
||||||
|
- `web/workspace/src/lib/workspace/multiplexer.ts`
|
||||||
|
- `subscribe()` の先頭で `const clientId = crypto.randomUUID()`
|
||||||
|
- その後の `#ensureConnected()` で初めて `new WebSocket(url)` を実行する
|
||||||
|
|
||||||
|
このため `crypto.randomUUID()` が利用できないブラウザでは、画面状態だけが
|
||||||
|
`connecting` に更新された後に同期例外が発生し、WebSocket request 自体が送信されない。
|
||||||
|
接続失敗を示す `closed` や diagnostic にも遷移しないため、見た目からは WebSocket の
|
||||||
|
接続待ちに見える。
|
||||||
|
|
||||||
|
同じ multiplexer では `crypto.randomUUID()` を次の3用途に使っている。
|
||||||
|
|
||||||
|
1. ブラウザ内で subscription を区別する `clientId`
|
||||||
|
2. `subscribe_events` request と response を対応付ける `request_id`
|
||||||
|
3. `unsubscribe_events` の `request_id`
|
||||||
|
|
||||||
|
いずれも暗号化や認証のためではなく、ローカル Map のキーまたはプロトコル上の相関 ID
|
||||||
|
である。secure-context-only API を要求する必要性はない。
|
||||||
|
|
||||||
|
## 切り分け結果
|
||||||
|
|
||||||
|
開発ホスト上から次の条件で `/api/w/<workspace-id>/protocol/ws` に WebSocket upgrade を
|
||||||
|
送ると、backend への直接接続と Vite の `5173` proxy 経由の両方で
|
||||||
|
`101 Switching Protocols` が返った。
|
||||||
|
|
||||||
|
- `Host: 192.168.1.32:5173`
|
||||||
|
- `Origin: http://192.168.1.32:5173`
|
||||||
|
|
||||||
|
したがって、現在の Vite 設定にある `proxy["/api"].ws = true` と、loopback に bind した
|
||||||
|
backend への proxy はこの現象の直接原因ではない。
|
||||||
|
|
||||||
|
## 改善案
|
||||||
|
|
||||||
|
- UI 全体で使う ID 生成 helper を用意し、secure context に依存しない実装にする。
|
||||||
|
`crypto.getRandomValues()` から UUID v4 相当を生成する方法で十分である。
|
||||||
|
- `WorkspaceMultiplexer.subscribe()` の同期初期化失敗を Console の diagnostic/state に
|
||||||
|
反映し、初期値の `connecting` に留まらないようにする。
|
||||||
|
- LAN 上の平文 HTTP を開発時の対応経路とするなら、insecure context から subscription
|
||||||
|
初期化できることを regression test として固定する。
|
||||||
|
- 本番相当の LAN 公開では HTTPS を使う。Passkey/WebAuthn も secure context を要求し、
|
||||||
|
現在の開発設定は `rp_id = localhost` なので、LAN origin を正式対応する場合は auth の
|
||||||
|
origin/RP ID 設計も別途必要になる。
|
||||||
|
|
||||||
|
## 補足
|
||||||
|
|
||||||
|
`--host` は Vite の listener を LAN に bind するだけであり、配信 origin を secure
|
||||||
|
context に変えるものではない。localhost で正常に動くことだけでは、LAN IP の HTTP
|
||||||
|
アクセスでも同じブラウザ API が利用できることを証明できない。
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# ENOSPC で Session JSONL の末尾が UTF-8 途中切れになる
|
||||||
|
|
||||||
|
## 観測
|
||||||
|
|
||||||
|
`Companion1` (`worker-runtime-30`) の直近 Segment
|
||||||
|
`019fce34-eb04-7420-9cc5-0e7e12f5bd67` は、Workdir の Edit tool が
|
||||||
|
`no storage space` で失敗した後、会話ログ `.jsonl` の末尾が `0xe3` 1 byte
|
||||||
|
だけで終わっていた。これは 3 byte UTF-8 文字の先頭 byte であり、実際の末尾は
|
||||||
|
「Workdir のストレー」の次の文字の途中で切れていた。
|
||||||
|
|
||||||
|
同 Segment の `.trace.jsonl` は valid UTF-8 だった。再開時の
|
||||||
|
`stream did not contain valid UTF-8` は provider stream ではなく、Session log を
|
||||||
|
`fs::read_to_string` した際のエラーだった。
|
||||||
|
|
||||||
|
## 原因
|
||||||
|
|
||||||
|
`FsStore::append_line` は JSON 本文と改行を別々に `write_all` し、ENOSPC で
|
||||||
|
部分書き込みになっても元の file length へ戻していなかった。reader も file 全体を
|
||||||
|
UTF-8 String として読むため、newline に到達していない未コミット末尾だけで Segment
|
||||||
|
全体を復元不能にしていた。
|
||||||
|
|
||||||
|
さらに Engine の history append callback と SystemItem committer は、Store error を
|
||||||
|
warning にして drop していた。このため disk 上の history を更新できなくても memory
|
||||||
|
上の history と tool loop が先へ進み得た。
|
||||||
|
|
||||||
|
## 改善
|
||||||
|
|
||||||
|
- newline を JSONL record の commit marker とする。
|
||||||
|
- reader は newline 未到達の末尾を未コミット record として無視する。
|
||||||
|
- 次回 append 前に未コミット末尾を最後の newline まで truncate する。
|
||||||
|
- append の partial write は append 開始時の file length へ rollback する。
|
||||||
|
- repair/write/rollback は `FsStore` clone 間で直列化する。
|
||||||
|
- Engine history append を fallible にし、Store write 成功前には item を memory history
|
||||||
|
に入れない。
|
||||||
|
- tool call の永続化に失敗した turn は tool 実行前に停止する。
|
||||||
|
- SystemItem の commit failure も transient context injection にせず turn error にする。
|
||||||
|
- `Invoke` 後に terminal run record が無い Segment は restore 時に interrupted とする。
|
||||||
|
これにより crash/ENOSPC 後の dangling tool call は新しい user turn の前に閉じられ、
|
||||||
|
side effect を無条件に再実行しない。
|
||||||
|
|
||||||
|
## 境界
|
||||||
|
|
||||||
|
この修正は process interruption と ENOSPC による trailing partial record を対象にする。
|
||||||
|
newline 済み record 内部の破損は silent recovery せず `StoreError::Corrupt` のまま扱う。
|
||||||
|
また append ごとの `fsync` は追加していないため、突然の電源断に対する block-level
|
||||||
|
durability まで保証するものではない。
|
||||||
@@ -25,7 +25,8 @@ feature = {
|
|||||||
task = { enabled = true; };
|
task = { enabled = true; };
|
||||||
memory = { enabled = true; };
|
memory = { enabled = true; };
|
||||||
web = { enabled = true; };
|
web = { enabled = true; };
|
||||||
workers = { enabled = true; };
|
sub_worker = { enabled = true; };
|
||||||
|
worker = { enabled = false; };
|
||||||
objective = { enabled = true; };
|
objective = { enabled = true; };
|
||||||
ticket = { enabled = true; authoring = true; thread = true; };
|
ticket = { enabled = true; authoring = true; thread = true; };
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ import "./base.dcdl" // {
|
|||||||
task = { enabled = true; };
|
task = { enabled = true; };
|
||||||
memory = { enabled = true; };
|
memory = { enabled = true; };
|
||||||
web = { enabled = true; };
|
web = { enabled = true; };
|
||||||
workers = { enabled = false; };
|
sub_worker = { enabled = true; };
|
||||||
|
worker = { enabled = false; };
|
||||||
ticket = { enabled = true; thread = true; };
|
ticket = { enabled = true; thread = true; };
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ import "./base.dcdl" // {
|
|||||||
task = { enabled = true; };
|
task = { enabled = true; };
|
||||||
memory = { enabled = true; };
|
memory = { enabled = true; };
|
||||||
web = { enabled = true; };
|
web = { enabled = true; };
|
||||||
workers = { enabled = true; };
|
sub_worker = { enabled = true; };
|
||||||
|
worker = { enabled = true; };
|
||||||
ticket = { enabled = true; authoring = true; thread = true; };
|
ticket = { enabled = true; authoring = true; thread = true; };
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ import "./base.dcdl" // {
|
|||||||
task = { enabled = true; };
|
task = { enabled = true; };
|
||||||
memory = { enabled = true; };
|
memory = { enabled = true; };
|
||||||
web = { enabled = true; };
|
web = { enabled = true; };
|
||||||
workers = { enabled = false; };
|
sub_worker = { enabled = false; };
|
||||||
|
worker = { enabled = false; };
|
||||||
ticket = { enabled = true; authoring = true; thread = true; intake = true; };
|
ticket = { enabled = true; authoring = true; thread = true; intake = true; };
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ import "./base.dcdl" // {
|
|||||||
task = { enabled = false; };
|
task = { enabled = false; };
|
||||||
memory = { enabled = true; staging = true; };
|
memory = { enabled = true; staging = true; };
|
||||||
web = { enabled = false; };
|
web = { enabled = false; };
|
||||||
workers = { enabled = false; };
|
sub_worker = { enabled = false; };
|
||||||
|
worker = { enabled = false; };
|
||||||
objective = { enabled = false; };
|
objective = { enabled = false; };
|
||||||
ticket = { enabled = false; thread = false; };
|
ticket = { enabled = false; thread = false; };
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -7,7 +7,9 @@ import "./base.dcdl" // {
|
|||||||
task = { enabled = true; };
|
task = { enabled = true; };
|
||||||
memory = { enabled = true; };
|
memory = { enabled = true; };
|
||||||
web = { enabled = true; };
|
web = { enabled = true; };
|
||||||
workers = { enabled = true; };
|
sub_worker = { enabled = true; };
|
||||||
|
worker = { enabled = true; };
|
||||||
|
manage_workdir = { enabled = true; };
|
||||||
ticket = { enabled = true; thread = true; orchestration_control = true; };
|
ticket = { enabled = true; thread = true; orchestration_control = true; };
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ import "./base.dcdl" // {
|
|||||||
task = { enabled = true; };
|
task = { enabled = true; };
|
||||||
memory = { enabled = true; };
|
memory = { enabled = true; };
|
||||||
web = { enabled = true; };
|
web = { enabled = true; };
|
||||||
workers = { enabled = false; };
|
sub_worker = { enabled = false; };
|
||||||
|
worker = { enabled = false; };
|
||||||
ticket = { enabled = true; thread = true; };
|
ticket = { enabled = true; thread = true; };
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
---
|
---
|
||||||
## Worker orchestration
|
## SubWorker orchestration
|
||||||
|
|
||||||
When Worker-management tools are available, spawned Worker notifications are background signals for the parent to handle at a natural stopping point. Do not ignore routine follow-up, but do not interrupt the current user request unnecessarily.
|
When SubWorker-management tools are available, SubWorker notifications are background signals for the parent Worker to handle at a natural stopping point. Do not ignore routine follow-up, but do not interrupt the current user request unnecessarily.
|
||||||
|
|
||||||
The parent does not need to keep a turn open or call tools solely to wait for a notification. Do not use `sleep` or polling loops just to wait for Worker output; if there is no useful immediate work, return control and handle the child when notified or when the user next asks.
|
The parent Worker does not need to keep a turn open or call tools solely to wait for a notification. Do not use `sleep` or polling loops just to wait for SubWorker output; if there is no useful immediate work, return control and handle the SubWorker when notified or when the user next asks.
|
||||||
|
|
||||||
Before treating delegated work as complete, read the child output and inspect concrete evidence such as worktree state, diff, and test results. Notifications are hints, not proof of completion.
|
Before treating delegated SubWorker work as complete, read the SubWorker output and inspect concrete evidence such as worktree state, diff, and test results. Notifications are hints, not proof of completion.
|
||||||
|
|
||||||
Peer Workers made visible by reciprocal metadata registration are not spawned children. Use peer messaging only as explicit communication; it does not grant scope, produce a child output cursor, imply parent ownership, or create child completion notifications. Peer sends require a live peer and do not auto-restore stopped peers.
|
Peer Workers made visible by reciprocal metadata registration are not spawned children. Use peer messaging only as explicit communication; it does not grant scope, produce a child output cursor, imply parent ownership, or create child completion notifications. Peer sends require a live peer and do not auto-restore stopped peers.
|
||||||
|
|
||||||
|
|||||||
@@ -53,12 +53,12 @@ worker_orchestration_guidance_section = "{% include \"$yoi/common/worker-orchest
|
|||||||
|
|
||||||
ticket_event_companion_notice = "{% include \"$yoi/worker/ticket_event_companion_notice\" %}"
|
ticket_event_companion_notice = "{% include \"$yoi/worker/ticket_event_companion_notice\" %}"
|
||||||
|
|
||||||
spawn_worker_tool_description = """\
|
sub_worker_spawn_tool_description = """\
|
||||||
Spawn a new Worker process to work on a delegated task. The spawner's write scope is reduced by the scope passed here; the spawned Worker receives its own socket and starts running `task` immediately. The spawned Worker outlives the spawner's current turn and can be contacted again through its socket path.
|
Spawn a parent-owned Internal SubWorker session to split context for a delegated task. The parent Worker's write scope is reduced by the scope passed here; the Internal SubWorker starts running `task` immediately without creating a Runtime Worker record, OS process, PID, or Unix socket. It remains available for follow-up turns until explicitly stopped or its parent exits.
|
||||||
|
|
||||||
Optional `cwd`: when provided, it is the child process/tool default working directory only. It must be an absolute existing directory covered by the child's delegated readable scope, and it does not change workspace/Profile/memory/Ticket roots or grant authority.
|
Optional `cwd`: when provided, it is the Internal SubWorker's tool default working directory only. It must be an absolute existing directory covered by the child's delegated readable scope, and it does not change workspace/Profile/memory/Ticket roots or grant authority. `name` must be unique among this Worker's direct children.
|
||||||
|
|
||||||
Profile selection: `profile` may be omitted or set to `default` to use the effective child default profile, set to `inherit` to derive reusable child configuration from this Worker, or set to one of the registry selectors below. Raw/path profile selectors are not accepted by SpawnWorker. `scope` is always the only delegated filesystem capability; profile scope is replaced by the explicit SpawnWorker scope.
|
Profile selection: `profile` may be omitted or set to `default` to use the effective child default profile, set to `inherit` to derive reusable child configuration from this Worker, or set to one of the registry selectors below. Raw/path profile selectors are not accepted by SubWorkerSpawn. `scope` is always the only delegated filesystem capability; profile scope is replaced by the explicit SubWorkerSpawn scope.
|
||||||
|
|
||||||
Default profile: {{ default_profile }}
|
Default profile: {{ default_profile }}
|
||||||
Special selector: inherit — derive reusable model/worker/tool policy from the spawner while replacing worker.name and scope.
|
Special selector: inherit — derive reusable model/worker/tool policy from the spawner while replacing worker.name and scope.
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
Workspace Orchestrator attention: authoritative Ticket state still contains queued work after the previous turn or after Server recovery.
|
||||||
|
|
||||||
|
Workspace: {{workspace_id}}
|
||||||
|
Remaining queued Tickets (bounded):
|
||||||
|
{{ticket_lines}}
|
||||||
|
{{omitted_line}}
|
||||||
|
Reread the listed Tickets, their relations, orchestration plans, current assignments, Workers, and Workdirs before acting. Continue only work already authorized by the human `ready -> queued` transition. Do not drain the queue automatically and do not create duplicate assignments, Workers, Workdirs, or merges. If no Ticket is currently actionable, record the durable waiting reason on the authoritative Ticket or orchestration plan and stop. Before implementation side effects, record the accepted `queued -> inprogress` transition.
|
||||||
@@ -6,7 +6,7 @@
|
|||||||
"dev": "deno run -A npm:vite@7.2.7 dev",
|
"dev": "deno run -A npm:vite@7.2.7 dev",
|
||||||
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
||||||
"check": "deno run -A npm:@sveltejs/kit@2.49.4 sync && deno run -A npm:svelte-check@4.3.4 --tsconfig ./tsconfig.json",
|
"check": "deno run -A npm:@sveltejs/kit@2.49.4 sync && deno run -A npm:svelte-check@4.3.4 --tsconfig ./tsconfig.json",
|
||||||
"test": "deno test --allow-read=src --allow-env=VSCODE_TEXTMATE_DEBUG src/lib/workspace/auth/model.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/markdown.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/sidebar/repository-nav.test.ts",
|
"test": "deno test --allow-read=src --allow-env=VSCODE_TEXTMATE_DEBUG src/lib/workspace/auth/model.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/markdown.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/sidebar/repository-nav.test.ts src/lib/workspace/tickets/ticket-panel.test.ts",
|
||||||
"build": "deno run -A npm:vite@7.2.7 build",
|
"build": "deno run -A npm:vite@7.2.7 build",
|
||||||
"preview": "deno run -A npm:vite@7.2.7 preview"
|
"preview": "deno run -A npm:vite@7.2.7 preview"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -108,9 +108,10 @@ export async function loadWorkspaceSkillDetail(
|
|||||||
export async function loadJson<T>(
|
export async function loadJson<T>(
|
||||||
fetchFn: typeof fetch,
|
fetchFn: typeof fetch,
|
||||||
path: string,
|
path: string,
|
||||||
|
init?: RequestInit,
|
||||||
): Promise<ApiResult<T>> {
|
): Promise<ApiResult<T>> {
|
||||||
try {
|
try {
|
||||||
const response = await fetchFn(path);
|
const response = await fetchFn(path, init);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const text = await response.text();
|
const text = await response.text();
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
import { parseSigilSegments } from "./composer-command.ts";
|
import {
|
||||||
|
buildComposerRequest,
|
||||||
|
parseSigilSegments,
|
||||||
|
} from "./composer-command.ts";
|
||||||
|
|
||||||
declare const Deno: { test(name: string, fn: () => void): void };
|
declare const Deno: { test(name: string, fn: () => void): void };
|
||||||
|
|
||||||
@@ -21,3 +24,14 @@ Deno.test("parseSigilSegments leaves hash sigils as plain text", () => {
|
|||||||
content: "ask #memory",
|
content: "ask #memory",
|
||||||
}]);
|
}]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Deno.test("notify command exposes the operation instead of a System-role input", () => {
|
||||||
|
assertEquals(buildComposerRequest(":notify reread the Ticket"), {
|
||||||
|
ok: true,
|
||||||
|
request: { kind: "notify", content: "reread the Ticket" },
|
||||||
|
});
|
||||||
|
assertEquals(buildComposerRequest(":system reread the Ticket"), {
|
||||||
|
ok: false,
|
||||||
|
message: "Unknown command: system. Type :help for available commands.",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import type { Segment } from "$lib/generated/protocol";
|
|||||||
|
|
||||||
export type WorkerConsoleInputKind =
|
export type WorkerConsoleInputKind =
|
||||||
| "user"
|
| "user"
|
||||||
| "system"
|
| "notify"
|
||||||
| "compact"
|
| "compact"
|
||||||
| "list_rewind_targets"
|
| "list_rewind_targets"
|
||||||
| "register_peer";
|
| "register_peer";
|
||||||
@@ -53,9 +53,9 @@ const COMMANDS: Record<string, CommandSpec> = {
|
|||||||
description:
|
description:
|
||||||
"Register another existing Worker as a reciprocal metadata peer.",
|
"Register another existing Worker as a reciprocal metadata peer.",
|
||||||
},
|
},
|
||||||
system: {
|
notify: {
|
||||||
usage: ":system <message>",
|
usage: ":notify <message>",
|
||||||
description: "Send an agent-visible system notification to the Worker.",
|
description: "Send an agent-visible notification to the Worker.",
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -125,12 +125,12 @@ function buildColonCommand(commandLine: string): ComposerCommandResult {
|
|||||||
request: { kind: "register_peer", content: argv[0] },
|
request: { kind: "register_peer", content: argv[0] },
|
||||||
notice: `peer metadata registration requested with \`${argv[0]}\``,
|
notice: `peer metadata registration requested with \`${argv[0]}\``,
|
||||||
};
|
};
|
||||||
case "system": {
|
case "notify": {
|
||||||
const message = commandLine.trim().slice(name.length).trimStart();
|
const message = commandLine.trim().slice(name.length).trimStart();
|
||||||
if (!message) {
|
if (!message) {
|
||||||
return invalidUsage("system");
|
return invalidUsage("notify");
|
||||||
}
|
}
|
||||||
return { ok: true, request: { kind: "system", content: message } };
|
return { ok: true, request: { kind: "notify", content: message } };
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return {
|
return {
|
||||||
@@ -158,7 +158,7 @@ function helpCommand(argv: string[]): ComposerCommandResult {
|
|||||||
notice: `command: ${name} — usage: ${spec.usage}. ${spec.description}`,
|
notice: `command: ${name} — usage: ${spec.usage}. ${spec.description}`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
const list = ["help", "noop", "compact", "rewind", "peer", "system"]
|
const list = ["help", "noop", "compact", "rewind", "peer", "notify"]
|
||||||
.map((command) => `${command} (${COMMANDS[command].usage})`)
|
.map((command) => `${command} (${COMMANDS[command].usage})`)
|
||||||
.join(", ");
|
.join(", ");
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ export const COLON_COMMAND_COMPLETIONS: ComposerCompletionEntry[] = [
|
|||||||
{ value: "rewind", description: "List rewind targets" },
|
{ value: "rewind", description: "List rewind targets" },
|
||||||
{ value: "rollback", description: "Alias for rewind" },
|
{ value: "rollback", description: "Alias for rewind" },
|
||||||
{ value: "peer", description: "Register metadata peer" },
|
{ value: "peer", description: "Register metadata peer" },
|
||||||
{ value: "system", description: "Send system notification" },
|
{ value: "notify", description: "Send Worker notification" },
|
||||||
];
|
];
|
||||||
|
|
||||||
export function completionTokenAt(
|
export function completionTokenAt(
|
||||||
|
|||||||
@@ -718,7 +718,7 @@ Deno.test("projectConsole preserves in-progress assistant protocol stream", () =
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
Deno.test("projectConsole keeps protocol lifecycle events out of the console surface", () => {
|
Deno.test("projectConsole hides lifecycle events and renders system items", () => {
|
||||||
const projection = projectConsole([
|
const projection = projectConsole([
|
||||||
{
|
{
|
||||||
eventId: "30",
|
eventId: "30",
|
||||||
@@ -743,12 +743,24 @@ Deno.test("projectConsole keeps protocol lifecycle events out of the console sur
|
|||||||
eventId: "34",
|
eventId: "34",
|
||||||
event: {
|
event: {
|
||||||
event: "system_item",
|
event: "system_item",
|
||||||
data: { item: { kind: "note", content: "internal" } },
|
data: {
|
||||||
|
item: {
|
||||||
|
kind: "notification",
|
||||||
|
message: "Ticket queued",
|
||||||
|
body: "Reread Ticket 00001KZ6TSGG5 before acting.",
|
||||||
|
},
|
||||||
|
},
|
||||||
} satisfies Event,
|
} satisfies Event,
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
|
|
||||||
assertEquals(projection.lines, []);
|
assertEquals(projection.lines.length, 1);
|
||||||
|
assertEquals(projection.lines[0].kind, "system");
|
||||||
|
assertEquals(projection.lines[0].title, "System · notification");
|
||||||
|
assertEquals(
|
||||||
|
projection.lines[0].body,
|
||||||
|
"Reread Ticket 00001KZ6TSGG5 before acting.",
|
||||||
|
);
|
||||||
assertEquals(projection.status, "running");
|
assertEquals(projection.status, "running");
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -859,6 +871,42 @@ Deno.test("projectConsole renders snapshot entries and in-flight output", () =>
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Deno.test("projectConsole restores system items from snapshot entries", () => {
|
||||||
|
const projection = projectConsole([{
|
||||||
|
eventId: "system-snapshot",
|
||||||
|
event: {
|
||||||
|
event: "snapshot",
|
||||||
|
data: {
|
||||||
|
entries: [{
|
||||||
|
kind: "system_item",
|
||||||
|
ts: 1,
|
||||||
|
item: {
|
||||||
|
kind: "notification",
|
||||||
|
message: "Worker completed",
|
||||||
|
body: "Child Worker coder-1 completed.",
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
greeting: {
|
||||||
|
worker_name: "Worker",
|
||||||
|
cwd: "/repo",
|
||||||
|
provider: "provider",
|
||||||
|
model: "model",
|
||||||
|
scope_summary: "bounded",
|
||||||
|
tools: [],
|
||||||
|
context_window: 100,
|
||||||
|
context_tokens: 20,
|
||||||
|
},
|
||||||
|
status: "idle",
|
||||||
|
},
|
||||||
|
} satisfies Event,
|
||||||
|
}]);
|
||||||
|
|
||||||
|
assertEquals(projection.lines.length, 1);
|
||||||
|
assertEquals(projection.lines[0].kind, "system");
|
||||||
|
assertEquals(projection.lines[0].title, "System · notification");
|
||||||
|
assertEquals(projection.lines[0].body, "Child Worker coder-1 completed.");
|
||||||
|
});
|
||||||
|
|
||||||
Deno.test("projectConsole reseeds visible rows from segment rotation", () => {
|
Deno.test("projectConsole reseeds visible rows from segment rotation", () => {
|
||||||
const projection = projectConsole([
|
const projection = projectConsole([
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -208,7 +208,7 @@ export function applyProtocolEvent(
|
|||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
case "system_item":
|
case "system_item":
|
||||||
// System items are protocol/internal context, not console output.
|
next.lines.push(systemItemLine(envelope.eventId, event.data.item));
|
||||||
break;
|
break;
|
||||||
case "text_delta":
|
case "text_delta":
|
||||||
appendStreaming(
|
appendStreaming(
|
||||||
@@ -397,6 +397,17 @@ function line(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function systemItemLine(eventId: string, item: unknown): ConsoleLine {
|
||||||
|
if (!isRecord(item)) {
|
||||||
|
return line(eventId, "system", "System item", jsonPreview(item));
|
||||||
|
}
|
||||||
|
const itemKind = stringField(item, "kind") ?? "item";
|
||||||
|
const title = `System · ${itemKind.replaceAll("_", " ")}`;
|
||||||
|
const body = stringField(item, "body") ?? stringField(item, "message") ??
|
||||||
|
stringField(item, "content") ?? jsonPreview(item);
|
||||||
|
return line(eventId, "system", title, body);
|
||||||
|
}
|
||||||
|
|
||||||
function upsertStatusLine(
|
function upsertStatusLine(
|
||||||
projection: ConsoleProjection,
|
projection: ConsoleProjection,
|
||||||
id: string,
|
id: string,
|
||||||
@@ -1140,6 +1151,9 @@ function applyLogEntry(
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
|
case "system_item":
|
||||||
|
projection.lines.push(systemItemLine(eventId, entry["item"]));
|
||||||
|
break;
|
||||||
case "assistant_item":
|
case "assistant_item":
|
||||||
case "tool_result":
|
case "tool_result":
|
||||||
applyLoggedItem(projection, eventId, entry["item"]);
|
applyLoggedItem(projection, eventId, entry["item"]);
|
||||||
|
|||||||
@@ -73,6 +73,42 @@
|
|||||||
justify-content: space-between;
|
justify-content: space-between;
|
||||||
gap: var(--space-4);
|
gap: var(--space-4);
|
||||||
}
|
}
|
||||||
|
.ticket-panel-controls {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-3);
|
||||||
|
}
|
||||||
|
.orchestrator-status {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.65rem;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: 0.65rem;
|
||||||
|
background: var(--bg-raised);
|
||||||
|
padding: 0.55rem 0.65rem;
|
||||||
|
}
|
||||||
|
.orchestrator-status > div {
|
||||||
|
display: grid;
|
||||||
|
gap: 0.08rem;
|
||||||
|
min-width: 5.5rem;
|
||||||
|
}
|
||||||
|
.orchestrator-status strong {
|
||||||
|
color: var(--text-strong);
|
||||||
|
font-size: 0.76rem;
|
||||||
|
}
|
||||||
|
.orchestrator-status span {
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-size: 0.68rem;
|
||||||
|
}
|
||||||
|
.orchestrator-status-dot {
|
||||||
|
width: 0.55rem;
|
||||||
|
height: 0.55rem;
|
||||||
|
border-radius: 50%;
|
||||||
|
background: #a75454;
|
||||||
|
}
|
||||||
|
.orchestrator-status[data-online="true"] .orchestrator-status-dot {
|
||||||
|
background: #43a66d;
|
||||||
|
}
|
||||||
.ticket-panel-summary {
|
.ticket-panel-summary {
|
||||||
display: grid;
|
display: grid;
|
||||||
justify-items: end;
|
justify-items: end;
|
||||||
|
|||||||
@@ -10,8 +10,15 @@ import type {
|
|||||||
|
|
||||||
declare const Deno: {
|
declare const Deno: {
|
||||||
test(name: string, fn: () => Promise<void> | void): void;
|
test(name: string, fn: () => Promise<void> | void): void;
|
||||||
|
readTextFile(path: string): Promise<string>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function assertIncludes(actual: string, expected: string): void {
|
||||||
|
if (!actual.includes(expected)) {
|
||||||
|
throw new Error(`expected source to include ${JSON.stringify(expected)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function assertEquals<T>(actual: T, expected: T): void {
|
function assertEquals<T>(actual: T, expected: T): void {
|
||||||
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
|
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
@@ -97,3 +104,20 @@ Deno.test("ticket worker launch uses the common Worker route and bounded Ticket
|
|||||||
"Work on Ticket 00001KYRRDVH9 as its reviewer.",
|
"Work on Ticket 00001KYRRDVH9 as its reviewer.",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Deno.test("ticket panel starts the Orchestrator explicitly and gates orchestration actions", async () => {
|
||||||
|
const panelSource = await Deno.readTextFile(
|
||||||
|
"src/routes/w/[workspaceId]/tickets/+page.svelte",
|
||||||
|
);
|
||||||
|
const detailSource = await Deno.readTextFile(
|
||||||
|
"src/routes/w/[workspaceId]/tickets/[ticketId]/+page.svelte",
|
||||||
|
);
|
||||||
|
|
||||||
|
assertIncludes(panelSource, 'workspaceApiPath(data.workspaceId, "/orchestrator")');
|
||||||
|
assertIncludes(panelSource, '{ method: "POST" }');
|
||||||
|
assertIncludes(panelSource, "Start Orchestrator");
|
||||||
|
assertIncludes(panelSource, "orchestrator.data?.online");
|
||||||
|
assertIncludes(detailSource, "{#if orchestratorOnline}");
|
||||||
|
assertIncludes(detailSource, "!orchestratorOnline");
|
||||||
|
assertIncludes(detailSource, "Orchestrator offline");
|
||||||
|
});
|
||||||
|
|||||||
@@ -12,6 +12,23 @@ export const TICKET_STATES = [
|
|||||||
export type TicketState = (typeof TICKET_STATES)[number];
|
export type TicketState = (typeof TICKET_STATES)[number];
|
||||||
export type TicketWorkerRole = "coder" | "reviewer";
|
export type TicketWorkerRole = "coder" | "reviewer";
|
||||||
|
|
||||||
|
export type WorkspaceOrchestratorStatus = {
|
||||||
|
workspace_id: string;
|
||||||
|
online: boolean;
|
||||||
|
disposition: string;
|
||||||
|
worker?: {
|
||||||
|
runtime_id: string;
|
||||||
|
worker_id: string;
|
||||||
|
state: string;
|
||||||
|
display_name: string;
|
||||||
|
} | null;
|
||||||
|
diagnostics: Array<{
|
||||||
|
code: string;
|
||||||
|
severity: string;
|
||||||
|
message: string;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
const LANE_DEFINITIONS = [
|
const LANE_DEFINITIONS = [
|
||||||
{
|
{
|
||||||
id: "ready-planning",
|
id: "ready-planning",
|
||||||
|
|||||||
+1
-1
@@ -428,7 +428,7 @@
|
|||||||
],
|
],
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
case "system":
|
case "notify":
|
||||||
return {
|
return {
|
||||||
method: "notify",
|
method: "notify",
|
||||||
params: { message: request.content, auto_run: true },
|
params: { message: request.content, auto_run: true },
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { untrack } from "svelte";
|
import { untrack } from "svelte";
|
||||||
import type { ApiResult } from "$lib/workspace/api/http";
|
import type { ApiResult } from "$lib/workspace/api/http";
|
||||||
import { ticketLanes } from "$lib/workspace/tickets/ticket-panel";
|
import { loadJson, workspaceApiPath } from "$lib/workspace/api/http";
|
||||||
|
import {
|
||||||
|
ticketLanes,
|
||||||
|
type WorkspaceOrchestratorStatus,
|
||||||
|
} from "$lib/workspace/tickets/ticket-panel";
|
||||||
import type {
|
import type {
|
||||||
TicketListResponse,
|
TicketListResponse,
|
||||||
TicketSummary,
|
TicketSummary,
|
||||||
@@ -11,13 +15,29 @@
|
|||||||
data: {
|
data: {
|
||||||
workspaceId: string;
|
workspaceId: string;
|
||||||
tickets: ApiResult<TicketListResponse>;
|
tickets: ApiResult<TicketListResponse>;
|
||||||
|
orchestrator: ApiResult<WorkspaceOrchestratorStatus>;
|
||||||
};
|
};
|
||||||
}>();
|
}>();
|
||||||
|
|
||||||
const initialTickets = untrack(() => data.tickets.data?.items ?? []);
|
const initialTickets = untrack(() => data.tickets.data?.items ?? []);
|
||||||
let tickets = $state<TicketSummary[]>(initialTickets);
|
let tickets = $state<TicketSummary[]>(initialTickets);
|
||||||
|
let orchestrator = $state<ApiResult<WorkspaceOrchestratorStatus>>(
|
||||||
|
untrack(() => data.orchestrator),
|
||||||
|
);
|
||||||
|
let orchestratorStarting = $state(false);
|
||||||
let lanes = $derived(ticketLanes(tickets));
|
let lanes = $derived(ticketLanes(tickets));
|
||||||
|
|
||||||
|
async function startOrchestrator() {
|
||||||
|
if (orchestratorStarting || orchestrator.data?.online) return;
|
||||||
|
orchestratorStarting = true;
|
||||||
|
orchestrator = await loadJson<WorkspaceOrchestratorStatus>(
|
||||||
|
fetch,
|
||||||
|
workspaceApiPath(data.workspaceId, "/orchestrator"),
|
||||||
|
{ method: "POST" },
|
||||||
|
);
|
||||||
|
orchestratorStarting = false;
|
||||||
|
}
|
||||||
|
|
||||||
function prettyDate(value?: string | null): string {
|
function prettyDate(value?: string | null): string {
|
||||||
if (!value) return "—";
|
if (!value) return "—";
|
||||||
const date = new Date(value);
|
const date = new Date(value);
|
||||||
@@ -36,12 +56,41 @@
|
|||||||
Plan, route, review, and close work without leaving the workspace.
|
Plan, route, review, and close work without leaving the workspace.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="ticket-panel-controls">
|
||||||
|
<div class="orchestrator-status" data-online={orchestrator.data?.online ?? false}>
|
||||||
|
<span class="orchestrator-status-dot"></span>
|
||||||
|
<div>
|
||||||
|
<strong>Orchestrator</strong>
|
||||||
|
<span>{orchestrator.data?.online ? "Online" : "Offline"}</span>
|
||||||
|
</div>
|
||||||
|
{#if !orchestrator.data?.online}
|
||||||
|
<button
|
||||||
|
class="workspace-primary-button"
|
||||||
|
type="button"
|
||||||
|
disabled={orchestratorStarting}
|
||||||
|
onclick={startOrchestrator}
|
||||||
|
>
|
||||||
|
{orchestratorStarting ? "Starting…" : "Start Orchestrator"}
|
||||||
|
</button>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
<div class="ticket-panel-summary" aria-label="Ticket summary">
|
<div class="ticket-panel-summary" aria-label="Ticket summary">
|
||||||
<strong>{tickets.length}</strong>
|
<strong>{tickets.length}</strong>
|
||||||
<span>tickets</span>
|
<span>tickets</span>
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
|
{#if orchestrator.error}
|
||||||
|
<p class="workspace-callout is-error">
|
||||||
|
Orchestrator status: {orchestrator.error}
|
||||||
|
</p>
|
||||||
|
{:else if !orchestrator.data?.online}
|
||||||
|
<p class="workspace-callout">
|
||||||
|
Orchestration actions are unavailable until the embedded Orchestrator is online.
|
||||||
|
</p>
|
||||||
|
{/if}
|
||||||
|
|
||||||
<section class="ticket-kanban" aria-label="Ticket workflow board">
|
<section class="ticket-kanban" aria-label="Ticket workflow board">
|
||||||
{#each lanes as lane (lane.id)}
|
{#each lanes as lane (lane.id)}
|
||||||
<section class="ticket-lane" data-state={lane.id}>
|
<section class="ticket-lane" data-state={lane.id}>
|
||||||
|
|||||||
@@ -1,15 +1,23 @@
|
|||||||
import { loadJson, workspaceApiPath } from "$lib/workspace/api/http";
|
import { loadJson, workspaceApiPath } from "$lib/workspace/api/http";
|
||||||
|
import type { WorkspaceOrchestratorStatus } from "$lib/workspace/tickets/ticket-panel";
|
||||||
import type { TicketListResponse } from "$lib/workspace/sidebar/types";
|
import type { TicketListResponse } from "$lib/workspace/sidebar/types";
|
||||||
import type { PageLoad } from "./$types";
|
import type { PageLoad } from "./$types";
|
||||||
|
|
||||||
export const load = (async ({ fetch, params }) => {
|
export const load = (async ({ fetch, params }) => {
|
||||||
const tickets = await loadJson<TicketListResponse>(
|
const [tickets, orchestrator] = await Promise.all([
|
||||||
|
loadJson<TicketListResponse>(
|
||||||
fetch,
|
fetch,
|
||||||
`${workspaceApiPath(params.workspaceId, "/tickets")}?limit=1000`,
|
`${workspaceApiPath(params.workspaceId, "/tickets")}?limit=1000`,
|
||||||
);
|
),
|
||||||
|
loadJson<WorkspaceOrchestratorStatus>(
|
||||||
|
fetch,
|
||||||
|
workspaceApiPath(params.workspaceId, "/orchestrator"),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
workspaceId: params.workspaceId,
|
workspaceId: params.workspaceId,
|
||||||
tickets,
|
tickets,
|
||||||
|
orchestrator,
|
||||||
};
|
};
|
||||||
}) satisfies PageLoad;
|
}) satisfies PageLoad;
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
relationLabel,
|
relationLabel,
|
||||||
TICKET_STATES,
|
TICKET_STATES,
|
||||||
ticketWorkerLaunchHref,
|
ticketWorkerLaunchHref,
|
||||||
|
type WorkspaceOrchestratorStatus,
|
||||||
} from "$lib/workspace/tickets/ticket-panel";
|
} from "$lib/workspace/tickets/ticket-panel";
|
||||||
import type { ApiResult } from "$lib/workspace/api/http";
|
import type { ApiResult } from "$lib/workspace/api/http";
|
||||||
import type {
|
import type {
|
||||||
@@ -22,6 +23,7 @@
|
|||||||
ticketId: string;
|
ticketId: string;
|
||||||
ticket: ApiResult<TicketDetail>;
|
ticket: ApiResult<TicketDetail>;
|
||||||
repositories: ApiResult<RepositoryListResponse>;
|
repositories: ApiResult<RepositoryListResponse>;
|
||||||
|
orchestrator: ApiResult<WorkspaceOrchestratorStatus>;
|
||||||
};
|
};
|
||||||
}>();
|
}>();
|
||||||
|
|
||||||
@@ -29,6 +31,7 @@
|
|||||||
const loadedTicket = initialData.ticket.data;
|
const loadedTicket = initialData.ticket.data;
|
||||||
if (!loadedTicket) throw new Error(initialData.ticket.error ?? "ticket load failed");
|
if (!loadedTicket) throw new Error(initialData.ticket.error ?? "ticket load failed");
|
||||||
const loadedRepositories = initialData.repositories.data;
|
const loadedRepositories = initialData.repositories.data;
|
||||||
|
const orchestratorOnline = initialData.orchestrator.data?.online ?? false;
|
||||||
|
|
||||||
let ticket = $state<TicketDetail>(loadedTicket);
|
let ticket = $state<TicketDetail>(loadedTicket);
|
||||||
let editing = $state(false);
|
let editing = $state(false);
|
||||||
@@ -270,11 +273,19 @@
|
|||||||
<p class="ticket-assignment-line">
|
<p class="ticket-assignment-line">
|
||||||
Assigned to <strong>{ticket.assignee ?? "Unassigned"}</strong>
|
Assigned to <strong>{ticket.assignee ?? "Unassigned"}</strong>
|
||||||
</p>
|
</p>
|
||||||
<p>The common launch flow carries a short canonical Ticket message and the target below.</p>
|
{#if orchestratorOnline}
|
||||||
|
<p>The Orchestrator is online. Start a role-specific Worker with the Ticket target below.</p>
|
||||||
<div class="ticket-role-actions">
|
<div class="ticket-role-actions">
|
||||||
<a class="workspace-primary-button" href={ticketWorkerLaunchHref(data.workspaceId, ticket, "coder")}>Coder</a>
|
<a class="workspace-primary-button" href={ticketWorkerLaunchHref(data.workspaceId, ticket, "coder")}>Coder</a>
|
||||||
<a class="workspace-secondary-button" href={ticketWorkerLaunchHref(data.workspaceId, ticket, "reviewer")}>Reviewer</a>
|
<a class="workspace-secondary-button" href={ticketWorkerLaunchHref(data.workspaceId, ticket, "reviewer")}>Reviewer</a>
|
||||||
</div>
|
</div>
|
||||||
|
{:else}
|
||||||
|
<p class="workspace-callout">Start the Workspace Orchestrator from the Ticket panel before launching Ticket Workers.</p>
|
||||||
|
<div class="ticket-role-actions">
|
||||||
|
<button class="workspace-primary-button" type="button" disabled>Coder</button>
|
||||||
|
<button class="workspace-secondary-button" type="button" disabled>Reviewer</button>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section class="ticket-control-card">
|
<section class="ticket-control-card">
|
||||||
@@ -309,8 +320,8 @@
|
|||||||
</button>
|
</button>
|
||||||
</form>
|
</form>
|
||||||
{#if ticket.state === "ready"}
|
{#if ticket.state === "ready"}
|
||||||
<button class="workspace-primary-button ticket-queue-button" type="button" disabled={busy === "queue"} onclick={() => mutate("queue", "/queue", {})}>
|
<button class="workspace-primary-button ticket-queue-button" type="button" disabled={busy === "queue" || !orchestratorOnline} onclick={() => mutate("queue", "/queue", {})}>
|
||||||
{busy === "queue" ? "Queueing…" : "Queue ticket"}
|
{busy === "queue" ? "Queueing…" : orchestratorOnline ? "Queue ticket" : "Orchestrator offline"}
|
||||||
</button>
|
</button>
|
||||||
{/if}
|
{/if}
|
||||||
</section>
|
</section>
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { loadJson, workspaceApiPath } from "$lib/workspace/api/http";
|
import { loadJson, workspaceApiPath } from "$lib/workspace/api/http";
|
||||||
|
import type { WorkspaceOrchestratorStatus } from "$lib/workspace/tickets/ticket-panel";
|
||||||
import type {
|
import type {
|
||||||
RepositoryListResponse,
|
RepositoryListResponse,
|
||||||
TicketDetail,
|
TicketDetail,
|
||||||
@@ -6,7 +7,7 @@ import type {
|
|||||||
import type { PageLoad } from "./$types";
|
import type { PageLoad } from "./$types";
|
||||||
|
|
||||||
export const load = (async ({ fetch, params }) => {
|
export const load = (async ({ fetch, params }) => {
|
||||||
const [ticket, repositories] = await Promise.all([
|
const [ticket, repositories, orchestrator] = await Promise.all([
|
||||||
loadJson<TicketDetail>(
|
loadJson<TicketDetail>(
|
||||||
fetch,
|
fetch,
|
||||||
workspaceApiPath(
|
workspaceApiPath(
|
||||||
@@ -18,6 +19,10 @@ export const load = (async ({ fetch, params }) => {
|
|||||||
fetch,
|
fetch,
|
||||||
workspaceApiPath(params.workspaceId, "/repositories"),
|
workspaceApiPath(params.workspaceId, "/repositories"),
|
||||||
),
|
),
|
||||||
|
loadJson<WorkspaceOrchestratorStatus>(
|
||||||
|
fetch,
|
||||||
|
workspaceApiPath(params.workspaceId, "/orchestrator"),
|
||||||
|
),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -25,5 +30,6 @@ export const load = (async ({ fetch, params }) => {
|
|||||||
ticketId: params.ticketId,
|
ticketId: params.ticketId,
|
||||||
ticket,
|
ticket,
|
||||||
repositories,
|
repositories,
|
||||||
|
orchestrator,
|
||||||
};
|
};
|
||||||
}) satisfies PageLoad;
|
}) satisfies PageLoad;
|
||||||
|
|||||||
Reference in New Issue
Block a user