runtime: canonicalize Worker aggregates

This commit is contained in:
2026-08-11 22:38:29 +09:00
parent 33d3d1a43f
commit 34f8949e85
16 changed files with 2013 additions and 201 deletions
+29 -4
View File
@@ -211,7 +211,7 @@ impl WorkerController {
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
Self::spawn_inner(worker, runtime_base, false).await
Self::spawn_inner(worker, runtime_base, false, None).await
}
/// Spawn a Worker owned by `worker-runtime`.
@@ -227,20 +227,37 @@ impl WorkerController {
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
Self::spawn_inner(worker, runtime_base, true).await
Self::spawn_inner(worker, runtime_base, true, None).await
}
/// Spawn into an exact persistent `runs/<generation>` directory.
pub async fn spawn_runtime_managed_run<C, St>(
worker: Worker<C, St>,
run_dir: &Path,
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
let parent = run_dir
.parent()
.ok_or_else(|| std::io::Error::other("run path has no parent"))?;
Self::spawn_inner(worker, parent, true, Some(run_dir)).await
}
async fn spawn_inner<C, St>(
worker: Worker<C, St>,
runtime_base: &Path,
runtime_managed: bool,
runtime_run: Option<&Path>,
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
let session = worker.workdir_session().cloned();
let result = Self::spawn_initialized(worker, runtime_base, runtime_managed).await;
let result =
Self::spawn_initialized(worker, runtime_base, runtime_managed, runtime_run).await;
if result.is_err()
&& let Some(session) = session
&& let Err(error) = session.close().await
@@ -254,6 +271,7 @@ impl WorkerController {
mut worker: Worker<C, St>,
runtime_base: &Path,
runtime_managed: bool,
runtime_run: Option<&Path>,
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
where
C: LlmClient + Clone + 'static,
@@ -273,7 +291,9 @@ impl WorkerController {
// the spawn-tool factories need its socket path, and before the
// initial status/history writes consume the greeting we build
// after registration is complete.
let runtime_dir = Arc::new(if runtime_managed {
let runtime_dir = Arc::new(if let Some(run_dir) = runtime_run {
RuntimeDir::create_worker_run(run_dir).await?
} else if runtime_managed {
RuntimeDir::create_transient(runtime_base, &worker.manifest().worker.name).await?
} else {
RuntimeDir::create(runtime_base, &worker.manifest().worker.name).await?
@@ -1298,6 +1318,11 @@ async fn controller_loop<C, St>(
}
}
drop(_socket_server);
if let Err(error) = runtime_dir.close_socket().await {
tracing::warn!(%error, "Worker runtime socket cleanup failed");
}
// Background memory jobs own extract/consolidate workers after a
// turn completes. Join them before closing the Workdir session so no
// Worker-owned task can outlive its operation attachment.
+47 -2
View File
@@ -42,6 +42,8 @@ pub struct SpawnedWorkerRecord {
pub struct RuntimeDir {
path: PathBuf,
write_legacy_snapshots: bool,
preserve_on_drop: bool,
socket_file_name: &'static str,
}
impl RuntimeDir {
@@ -56,6 +58,8 @@ impl RuntimeDir {
Ok(Self {
path,
write_legacy_snapshots: true,
preserve_on_drop: false,
socket_file_name: "sock",
})
}
@@ -69,6 +73,36 @@ impl RuntimeDir {
Ok(Self {
path,
write_legacy_snapshots: false,
preserve_on_drop: false,
socket_file_name: "sock",
})
}
/// Create an exact, persistent generation-scoped Worker run directory.
/// Existing directories are rejected so stale artifacts cannot be reused.
pub async fn create_worker_run(path: &Path) -> Result<Self, io::Error> {
let parent = path
.parent()
.ok_or_else(|| io::Error::other("run path has no parent"))?;
fs::create_dir_all(parent).await?;
fs::create_dir(path).await?;
fs::create_dir(path.join("artifacts")).await?;
fs::create_dir(path.join("spawned")).await?;
for log in ["worker.out.log", "worker.err.log"] {
let file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(path.join(log))
.await?;
file.sync_all().await?;
}
std::fs::File::open(path)?.sync_all()?;
std::fs::File::open(parent)?.sync_all()?;
Ok(Self {
path: path.to_path_buf(),
write_legacy_snapshots: false,
preserve_on_drop: true,
socket_file_name: "worker.sock",
})
}
@@ -116,13 +150,24 @@ impl RuntimeDir {
/// that only know the worker name (e.g. the TUI's attach flow)
/// predict the same path via [`manifest::paths::worker_socket_path`].
pub fn socket_path(&self) -> PathBuf {
self.path.join("sock")
self.path.join(self.socket_file_name)
}
pub async fn close_socket(&self) -> Result<(), io::Error> {
match fs::remove_file(self.socket_path()).await {
Ok(()) => Ok(()),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error),
}
}
}
impl Drop for RuntimeDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.path);
let _ = std::fs::remove_file(self.socket_path());
if !self.preserve_on_drop {
let _ = std::fs::remove_dir_all(&self.path);
}
}
}
@@ -120,15 +120,15 @@ pub fn adopt_allocation(
/// The Worker's in-memory `segment_id` can change underneath the
/// allocation in two normal places:
///
/// - `Worker::compact` mints a fresh session and swaps it in.
/// - `session_store::ensure_head_or_fork` auto-forks when another
/// - `Worker::compact` mints a fresh Segment in the same Session.
/// - `session_store::ensure_head_or_fork` auto-forks within that Session when another
/// writer has advanced the store head behind our back.
///
/// Both paths must call this so subsequent [`lookup_segment`] queries
/// find the live session id, not the old one. Without this update a
/// find the live Segment id, not the old one. Without this update a
/// concurrent `restore_from_manifest(new_id)` would see "no live
/// writer" and proceed to register a competing allocation on the
/// session this Worker just moved into.
/// Segment lineage this Worker just moved into.
///
/// The lock is opened once and the allocation is rewritten inside the
/// guard, so the segment_id collision check is atomic with the
+2 -2
View File
@@ -3067,13 +3067,13 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
/// Compact the current session by summarising history via a
/// disposable Engine, then replacing history with
/// `[summary, ...recent_turns]` and creating a new session.
/// `[summary, ...recent_turns]` in a new Segment of the same Session.
///
/// The summary Engine uses:
/// - `compaction.model` from the manifest if configured, or
/// - a clone of the main LlmClient via `clone_boxed()`.
///
/// Returns the new session ID.
/// Returns the new Segment ID. The Worker keeps its Session ID.
pub async fn compact(&mut self, retained_tokens: u64) -> Result<SegmentId, WorkerError> {
use crate::compact::worker::{
CompactWorkerContext, CompactWorkerInterceptor, add_reference_tool,
+7 -3
View File
@@ -421,7 +421,11 @@ async fn pre_run_compact_success_broadcasts_start_and_done() {
// Drain run events so only compact events remain in `rx`.
let _ = drain(&mut rx);
let session_before = worker.session_id();
let segment_before = worker.segment_id();
worker.try_pre_run_compact().await;
assert_eq!(worker.session_id(), session_before);
assert_ne!(worker.segment_id(), segment_before);
let events = drain(&mut rx);
let kinds: Vec<&str> = events
@@ -442,7 +446,7 @@ async fn pre_run_compact_success_broadcasts_start_and_done() {
"unexpected CompactFailed in {kinds:?}"
);
// CompactDone carries the new session id.
// CompactDone carries the new Segment ID; the Session ID is unchanged.
let new_id_in_event = events.iter().find_map(|e| match e {
Event::CompactDone { new_segment_id } => Some(*new_segment_id),
_ => None,
@@ -583,11 +587,11 @@ async fn compact_resets_extract_pointer_so_extract_can_fire_again() {
);
// Compact runs. Without the fix the in-memory pointer would still
// reference the old session's history_len.
// reference the old Segment's history_len.
worker.try_pre_run_compact().await;
assert!(
worker.extract_pointer().is_none(),
"extract_pointer must be reset to None after compact (matches cold-restore on the new session)"
"extract_pointer must be reset to None after compact (matches cold-restore on the new Segment)"
);
}