Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: Podのステータスを厳密にし、同期漏れを防ぐ

Browse Source
This commit is contained in:
Hare
2026-05-04 12:55:11 +09:00
parent a0771608b1
commit 560c23bc75
24 changed files with 641 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/pod/src/compact/prune.rs
+22 -26
View File
@@ -49,33 +49,29 @@ impl<C: LlmClient, St: Store> Pod<C, St> {
let metrics = self.metrics_tracker_handle();
let usage_tracker = self.usage_tracker_handle();
let observer: PruneObserver = Box::new(move |eval| {
match &eval.decision {
PruneDecision::Fired { .. } => {
let correlation_id = uuid::Uuid::now_v7().to_string();
let mut metric = Metric::now("prune.fire")
.with_value(eval.estimated_savings as f64)
.with_correlation_id(&correlation_id)
.with_dimension("candidate_count", eval.candidate_count.to_string());
if let Some(border) = eval.border_turn {
metric = metric.with_dimension("border_turn", border.to_string());
}
metrics.push(metric);
usage_tracker.note_correlation_id(correlation_id);
}
PruneDecision::SkippedNoCandidates => {
metrics.push(
Metric::now("prune.skip").with_dimension("reason", "no_candidates"),
);
}
PruneDecision::SkippedBelowMinSavings => {
metrics.push(
Metric::now("prune.skip")
.with_dimension("reason", "below_min_savings")
.with_dimension("candidate_count", eval.candidate_count.to_string())
.with_value(eval.estimated_savings as f64),
);
let observer: PruneObserver = Box::new(move |eval| match &eval.decision {
PruneDecision::Fired { .. } => {
let correlation_id = uuid::Uuid::now_v7().to_string();
let mut metric = Metric::now("prune.fire")
.with_value(eval.estimated_savings as f64)
.with_correlation_id(&correlation_id)
.with_dimension("candidate_count", eval.candidate_count.to_string());
if let Some(border) = eval.border_turn {
metric = metric.with_dimension("border_turn", border.to_string());
}
metrics.push(metric);
usage_tracker.note_correlation_id(correlation_id);
}
PruneDecision::SkippedNoCandidates => {
metrics.push(Metric::now("prune.skip").with_dimension("reason", "no_candidates"));
}
PruneDecision::SkippedBelowMinSavings => {
metrics.push(
Metric::now("prune.skip")
.with_dimension("reason", "below_min_savings")
.with_dimension("candidate_count", eval.candidate_count.to_string())
.with_value(eval.estimated_savings as f64),
);
}
});
crates/pod/src/controller.rs
+164 -155
View File
@@ -12,13 +12,15 @@ use crate::ipc::notify_buffer::NotifyBuffer;
use crate::ipc::server::SocketServer;
use crate::pod::{Pod, PodError, PodRunResult};
use crate::runtime::dir::RuntimeDir;
use crate::shared_state::{PodSharedState, PodStatus};
use crate::shared_state::PodSharedState;
use crate::spawn::comm_tools::{
list_pods_tool, read_pod_output_tool, send_to_pod_tool, stop_pod_tool,
};
use crate::spawn::registry::SpawnedPodRegistry;
use crate::spawn::tool::spawn_pod_tool;
use protocol::{AlertLevel, AlertSource, ErrorCode, Event, Method, RunResult, TurnResult};
use protocol::{
AlertLevel, AlertSource, ErrorCode, Event, Method, PodStatus, RunResult, TurnResult,
};
fn is_system_message_item(item: &Item) -> bool {
matches!(
@@ -63,6 +65,75 @@ impl PodHandle {
}
}
async fn set_controller_status(
shared_state: &Arc<PodSharedState>,
runtime_dir: &RuntimeDir,
event_tx: &broadcast::Sender<Event>,
status: PodStatus,
) {
shared_state.set_status(status);
let _ = runtime_dir.write_status(shared_state).await;
let _ = event_tx.send(Event::Status { status });
}
async fn run_post_run_jobs<C, St>(pod: &mut Pod<C, St>, alerter: &Alerter)
where
C: LlmClient,
St: Store,
{
if let Err(e) = pod.try_post_run_extract().await {
tracing::warn!(error = %e, "Post-run memory extract error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory extract error: {e}"),
);
}
if let Err(e) = pod.try_post_run_consolidate().await {
tracing::warn!(error = %e, "Post-run memory consolidate error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory consolidate error: {e}"),
);
}
if let Err(e) = pod.try_post_run_compact().await {
tracing::warn!(error = %e, "Post-run compaction error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Compactor,
format!("post-run compaction error: {e}"),
);
}
}
async fn finish_controller_run<C, St>(
pod: &mut Pod<C, St>,
shared_state: &Arc<PodSharedState>,
runtime_dir: &RuntimeDir,
event_tx: &broadcast::Sender<Event>,
alerter: &Alerter,
new_status: PodStatus,
) where
C: LlmClient,
St: Store,
{
if new_status == PodStatus::Busy {
run_post_run_jobs(pod, alerter).await;
}
let items = pod.worker().history().to_vec();
shared_state.update_history(items);
shared_state.set_user_segments(pod.user_segments().to_vec());
let final_status = if new_status == PodStatus::Busy {
PodStatus::Idle
} else {
new_status
};
set_controller_status(shared_state, runtime_dir, event_tx, final_status).await;
let _ = runtime_dir.write_history(shared_state).await;
}
// ---------------------------------------------------------------------------
// PodController — actor that owns a Pod
// ---------------------------------------------------------------------------
@@ -414,8 +485,13 @@ impl PodController {
let _ = event_tx.send(Event::UserMessage {
segments: input.clone(),
});
shared_state.set_status(PodStatus::Running);
let _ = runtime_dir.write_status(&shared_state).await;
set_controller_status(
&shared_state,
&runtime_dir,
&event_tx,
PodStatus::Running,
)
.await;
let run_future = async {
if was_paused {
@@ -430,6 +506,7 @@ impl PodController {
&event_tx,
&cancel_tx,
&shared_state,
&runtime_dir,
&notify_buffer,
self_parent_socket.as_ref(),
&spawner_name,
@@ -437,39 +514,15 @@ impl PodController {
)
.await;
if new_status == PodStatus::Idle {
if let Err(e) = pod.try_post_run_extract().await {
tracing::warn!(error = %e, "Post-run memory extract error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory extract error: {e}"),
);
}
if let Err(e) = pod.try_post_run_consolidate().await {
tracing::warn!(error = %e, "Post-run memory consolidate error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory consolidate error: {e}"),
);
}
if let Err(e) = pod.try_post_run_compact().await {
tracing::warn!(error = %e, "Post-run compaction error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Compactor,
format!("post-run compaction error: {e}"),
);
}
}
let items = pod.worker().history().to_vec();
shared_state.update_history(items);
shared_state.set_user_segments(pod.user_segments().to_vec());
shared_state.set_status(new_status);
let _ = runtime_dir.write_status(&shared_state).await;
let _ = runtime_dir.write_history(&shared_state).await;
finish_controller_run(
&mut pod,
&shared_state,
&runtime_dir,
&event_tx,
&alerter,
new_status,
)
.await;
if shutdown {
let _ = event_tx.send(Event::Shutdown);
@@ -482,17 +535,23 @@ impl PodController {
message: message.clone(),
});
pod.push_notify(message);
if shared_state.get_status() != PodStatus::Idle {
// RUNNING / Paused: the buffer push is the
// entire operation; the in-flight turn (or
// next Resume) will drain the buffer at its
// next pre_llm_request.
let status = shared_state.get_status();
if status != PodStatus::Idle {
// RUNNING / Paused / Busy: the buffer push is the
// entire operation; an in-flight turn (or the
// next Resume/Run after Busy) will drain the buffer
// at its next pre_llm_request.
continue;
}
// IDLE: auto-start a turn so the LLM sees the
// buffered notification(s) without a human Run.
shared_state.set_status(PodStatus::Running);
let _ = runtime_dir.write_status(&shared_state).await;
set_controller_status(
&shared_state,
&runtime_dir,
&event_tx,
PodStatus::Running,
)
.await;
let (new_status, shutdown) = run_with_cancel_support(
pod.run_for_notification(),
@@ -500,6 +559,7 @@ impl PodController {
&event_tx,
&cancel_tx,
&shared_state,
&runtime_dir,
&notify_buffer,
self_parent_socket.as_ref(),
&spawner_name,
@@ -507,39 +567,15 @@ impl PodController {
)
.await;
if new_status == PodStatus::Idle {
if let Err(e) = pod.try_post_run_extract().await {
tracing::warn!(error = %e, "Post-run memory extract error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory extract error: {e}"),
);
}
if let Err(e) = pod.try_post_run_consolidate().await {
tracing::warn!(error = %e, "Post-run memory consolidate error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory consolidate error: {e}"),
);
}
if let Err(e) = pod.try_post_run_compact().await {
tracing::warn!(error = %e, "Post-run compaction error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Compactor,
format!("post-run compaction error: {e}"),
);
}
}
let items = pod.worker().history().to_vec();
shared_state.update_history(items);
shared_state.set_user_segments(pod.user_segments().to_vec());
shared_state.set_status(new_status);
let _ = runtime_dir.write_status(&shared_state).await;
let _ = runtime_dir.write_history(&shared_state).await;
finish_controller_run(
&mut pod,
&shared_state,
&runtime_dir,
&event_tx,
&alerter,
new_status,
)
.await;
if shutdown {
let _ = event_tx.send(Event::Shutdown);
@@ -555,8 +591,13 @@ impl PodController {
});
continue;
}
shared_state.set_status(PodStatus::Running);
let _ = runtime_dir.write_status(&shared_state).await;
set_controller_status(
&shared_state,
&runtime_dir,
&event_tx,
PodStatus::Running,
)
.await;
let (new_status, shutdown) = run_with_cancel_support(
pod.resume(),
@@ -564,6 +605,7 @@ impl PodController {
&event_tx,
&cancel_tx,
&shared_state,
&runtime_dir,
&notify_buffer,
self_parent_socket.as_ref(),
&spawner_name,
@@ -571,39 +613,15 @@ impl PodController {
)
.await;
if new_status == PodStatus::Idle {
if let Err(e) = pod.try_post_run_extract().await {
tracing::warn!(error = %e, "Post-run memory extract error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory extract error: {e}"),
);
}
if let Err(e) = pod.try_post_run_consolidate().await {
tracing::warn!(error = %e, "Post-run memory consolidate error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory consolidate error: {e}"),
);
}
if let Err(e) = pod.try_post_run_compact().await {
tracing::warn!(error = %e, "Post-run compaction error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Compactor,
format!("post-run compaction error: {e}"),
);
}
}
let items = pod.worker().history().to_vec();
shared_state.update_history(items);
shared_state.set_user_segments(pod.user_segments().to_vec());
shared_state.set_status(new_status);
let _ = runtime_dir.write_status(&shared_state).await;
let _ = runtime_dir.write_history(&shared_state).await;
finish_controller_run(
&mut pod,
&shared_state,
&runtime_dir,
&event_tx,
&alerter,
new_status,
)
.await;
if shutdown {
let _ = event_tx.send(Event::Shutdown);
@@ -619,11 +637,12 @@ impl PodController {
}
Method::Pause => {
// Already paused → idempotent no-op. Otherwise
// the Pod is Idle (Running turns go through
// `run_with_cancel_support`, not this outer
// match), so there is nothing to pause.
if shared_state.get_status() != PodStatus::Paused {
// Already paused or post-run busy → idempotent no-op.
// Otherwise the Pod is Idle (Running turns go through
// `run_with_cancel_support`, not this outer match), so
// there is nothing to pause.
let status = shared_state.get_status();
if !matches!(status, PodStatus::Paused | PodStatus::Busy) {
let _ = event_tx.send(Event::Error {
code: ErrorCode::NotRunning,
message: "Pod is not running".into(),
@@ -666,8 +685,13 @@ impl PodController {
// notification is not stranded. Matches the
// `Method::Notify` idle path.
if shared_state.get_status() == PodStatus::Idle {
shared_state.set_status(PodStatus::Running);
let _ = runtime_dir.write_status(&shared_state).await;
set_controller_status(
&shared_state,
&runtime_dir,
&event_tx,
PodStatus::Running,
)
.await;
let (new_status, shutdown) = run_with_cancel_support(
pod.run_for_notification(),
@@ -675,6 +699,7 @@ impl PodController {
&event_tx,
&cancel_tx,
&shared_state,
&runtime_dir,
&notify_buffer,
self_parent_socket.as_ref(),
&spawner_name,
@@ -682,39 +707,15 @@ impl PodController {
)
.await;
if new_status == PodStatus::Idle {
if let Err(e) = pod.try_post_run_extract().await {
tracing::warn!(error = %e, "Post-run memory extract error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory extract error: {e}"),
);
}
if let Err(e) = pod.try_post_run_consolidate().await {
tracing::warn!(error = %e, "Post-run memory consolidate error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Pod,
format!("post-run memory consolidate error: {e}"),
);
}
if let Err(e) = pod.try_post_run_compact().await {
tracing::warn!(error = %e, "Post-run compaction error");
alerter.alert(
AlertLevel::Warn,
AlertSource::Compactor,
format!("post-run compaction error: {e}"),
);
}
}
let items = pod.worker().history().to_vec();
shared_state.update_history(items);
shared_state.set_user_segments(pod.user_segments().to_vec());
shared_state.set_status(new_status);
let _ = runtime_dir.write_status(&shared_state).await;
let _ = runtime_dir.write_history(&shared_state).await;
finish_controller_run(
&mut pod,
&shared_state,
&runtime_dir,
&event_tx,
&alerter,
new_status,
)
.await;
if shutdown {
let _ = event_tx.send(Event::Shutdown);
@@ -766,6 +767,7 @@ async fn run_with_cancel_support<F>(
event_tx: &broadcast::Sender<Event>,
cancel_tx: &mpsc::Sender<()>,
shared_state: &Arc<PodSharedState>,
runtime_dir: &RuntimeDir,
notify_buffer: &NotifyBuffer,
parent_socket: Option<&std::path::PathBuf>,
self_name: &str,
@@ -784,11 +786,18 @@ where
return match result {
Ok(r) => {
let (status, run_result) = match r {
PodRunResult::Finished => (PodStatus::Idle, RunResult::Finished),
PodRunResult::Finished => (PodStatus::Busy, RunResult::Finished),
PodRunResult::Paused => (PodStatus::Paused, RunResult::Paused),
PodRunResult::LimitReached => (PodStatus::Idle, RunResult::LimitReached),
PodRunResult::LimitReached => (PodStatus::Busy, RunResult::LimitReached),
};
let _ = event_tx.send(Event::RunEnd { result: run_result });
if status == PodStatus::Busy {
shared_state.set_status(PodStatus::Busy);
let _ = runtime_dir.write_status(shared_state).await;
let _ = event_tx.send(Event::Status {
status: PodStatus::Busy,
});
}
if matches!(run_result, RunResult::Finished) {
crate::ipc::event::fire_and_forget(
parent_socket.cloned(),
@@ -822,7 +831,7 @@ where
message,
},
);
(PodStatus::Idle, shutdown_requested)
(PodStatus::Busy, shutdown_requested)
}
};
}
crates/pod/src/ipc/interceptor.rs
+1 -1
View File
@@ -17,9 +17,9 @@ use llm_worker::interceptor::{
Interceptor, PostToolAction, PreRequestAction, PreToolAction, PromptAction, ToolCallInfo,
ToolResultInfo, TurnEndAction,
};
use tracing::warn;
use llm_worker::tool::ToolOutput;
use tracing::info;
use tracing::warn;
use crate::compact::state::CompactState;
use crate::hook::{
crates/pod/src/ipc/server.rs
+2
View File
@@ -159,10 +159,12 @@ async fn handle_connection(stream: tokio::net::UnixStream, handle: PodHandle) {
})
.collect();
let greeting = handle.shared_state.greeting.clone();
let status = handle.shared_state.get_status();
if writer
.write(&Event::History {
items: values,
greeting,
status,
})
.await
.is_err()
crates/pod/src/lib.rs
+2 -2
View File
@@ -26,7 +26,7 @@ pub use pod::{Pod, PodError, PodRunResult, apply_worker_manifest};
pub use prompt::catalog::{CatalogError, PodPrompt, PromptCatalog};
pub use prompt::loader::PromptLoader;
pub use prompt::system::{SystemPromptContext, SystemPromptError, SystemPromptTemplate};
pub use protocol::{ErrorCode, Event, Method, TurnResult};
pub use protocol::{ErrorCode, Event, Method, PodStatus, TurnResult};
pub use provider::{ProviderError, build_client};
pub use runtime::dir::RuntimeDir;
pub use shared_state::{PodSharedState, PodStatus};
pub use shared_state::PodSharedState;
crates/pod/src/runtime/dir.rs
+2 -1
View File
@@ -131,7 +131,8 @@ pub fn default_base() -> Result<PathBuf, io::Error> {
#[cfg(test)]
mod tests {
use super::*;
use crate::shared_state::{PodSharedState, PodStatus};
use crate::shared_state::PodSharedState;
use protocol::PodStatus;
fn test_state() -> PodSharedState {
PodSharedState::new(
crates/pod/src/shared_state.rs
+3 -11
View File
@@ -1,8 +1,8 @@
use std::sync::{OnceLock, RwLock};
use llm_worker::llm_client::types::Item;
use protocol::Segment;
use serde::{Deserialize, Serialize};
use protocol::{PodStatus, Segment};
use serde_json::json;
use session_store::SessionId;
use crate::fs_view::PodFsView;
@@ -39,14 +39,6 @@ pub struct PodSharedState {
workflows: OnceLock<Vec<WorkflowCandidate>>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PodStatus {
Idle,
Running,
Paused,
}
impl PodSharedState {
pub fn new(
pod_name: String,
@@ -138,7 +130,7 @@ impl PodSharedState {
/// Serialize status as JSON.
pub fn status_json(&self) -> String {
let status = self.get_status();
serde_json::json!({
json!({
"state": status,
"session_id": self.session_id.to_string(),
"pod_name": self.pod_name,