runtime: canonicalize Worker aggregates
This commit is contained in:
@@ -211,7 +211,7 @@ impl WorkerController {
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Self::spawn_inner(worker, runtime_base, false).await
|
||||
Self::spawn_inner(worker, runtime_base, false, None).await
|
||||
}
|
||||
|
||||
/// Spawn a Worker owned by `worker-runtime`.
|
||||
@@ -227,20 +227,37 @@ impl WorkerController {
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Self::spawn_inner(worker, runtime_base, true).await
|
||||
Self::spawn_inner(worker, runtime_base, true, None).await
|
||||
}
|
||||
|
||||
/// Spawn into an exact persistent `runs/<generation>` directory.
|
||||
pub async fn spawn_runtime_managed_run<C, St>(
|
||||
worker: Worker<C, St>,
|
||||
run_dir: &Path,
|
||||
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
let parent = run_dir
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::other("run path has no parent"))?;
|
||||
Self::spawn_inner(worker, parent, true, Some(run_dir)).await
|
||||
}
|
||||
|
||||
async fn spawn_inner<C, St>(
|
||||
worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
runtime_managed: bool,
|
||||
runtime_run: Option<&Path>,
|
||||
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
let session = worker.workdir_session().cloned();
|
||||
let result = Self::spawn_initialized(worker, runtime_base, runtime_managed).await;
|
||||
let result =
|
||||
Self::spawn_initialized(worker, runtime_base, runtime_managed, runtime_run).await;
|
||||
if result.is_err()
|
||||
&& let Some(session) = session
|
||||
&& let Err(error) = session.close().await
|
||||
@@ -254,6 +271,7 @@ impl WorkerController {
|
||||
mut worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
runtime_managed: bool,
|
||||
runtime_run: Option<&Path>,
|
||||
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
@@ -273,7 +291,9 @@ impl WorkerController {
|
||||
// the spawn-tool factories need its socket path, and before the
|
||||
// initial status/history writes consume the greeting we build
|
||||
// after registration is complete.
|
||||
let runtime_dir = Arc::new(if runtime_managed {
|
||||
let runtime_dir = Arc::new(if let Some(run_dir) = runtime_run {
|
||||
RuntimeDir::create_worker_run(run_dir).await?
|
||||
} else if runtime_managed {
|
||||
RuntimeDir::create_transient(runtime_base, &worker.manifest().worker.name).await?
|
||||
} else {
|
||||
RuntimeDir::create(runtime_base, &worker.manifest().worker.name).await?
|
||||
@@ -1298,6 +1318,11 @@ async fn controller_loop<C, St>(
|
||||
}
|
||||
}
|
||||
|
||||
drop(_socket_server);
|
||||
if let Err(error) = runtime_dir.close_socket().await {
|
||||
tracing::warn!(%error, "Worker runtime socket cleanup failed");
|
||||
}
|
||||
|
||||
// Background memory jobs own extract/consolidate workers after a
|
||||
// turn completes. Join them before closing the Workdir session so no
|
||||
// Worker-owned task can outlive its operation attachment.
|
||||
|
||||
@@ -42,6 +42,8 @@ pub struct SpawnedWorkerRecord {
|
||||
pub struct RuntimeDir {
|
||||
path: PathBuf,
|
||||
write_legacy_snapshots: bool,
|
||||
preserve_on_drop: bool,
|
||||
socket_file_name: &'static str,
|
||||
}
|
||||
|
||||
impl RuntimeDir {
|
||||
@@ -56,6 +58,8 @@ impl RuntimeDir {
|
||||
Ok(Self {
|
||||
path,
|
||||
write_legacy_snapshots: true,
|
||||
preserve_on_drop: false,
|
||||
socket_file_name: "sock",
|
||||
})
|
||||
}
|
||||
|
||||
@@ -69,6 +73,36 @@ impl RuntimeDir {
|
||||
Ok(Self {
|
||||
path,
|
||||
write_legacy_snapshots: false,
|
||||
preserve_on_drop: false,
|
||||
socket_file_name: "sock",
|
||||
})
|
||||
}
|
||||
|
||||
/// Create an exact, persistent generation-scoped Worker run directory.
|
||||
/// Existing directories are rejected so stale artifacts cannot be reused.
|
||||
pub async fn create_worker_run(path: &Path) -> Result<Self, io::Error> {
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::other("run path has no parent"))?;
|
||||
fs::create_dir_all(parent).await?;
|
||||
fs::create_dir(path).await?;
|
||||
fs::create_dir(path.join("artifacts")).await?;
|
||||
fs::create_dir(path.join("spawned")).await?;
|
||||
for log in ["worker.out.log", "worker.err.log"] {
|
||||
let file = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(path.join(log))
|
||||
.await?;
|
||||
file.sync_all().await?;
|
||||
}
|
||||
std::fs::File::open(path)?.sync_all()?;
|
||||
std::fs::File::open(parent)?.sync_all()?;
|
||||
Ok(Self {
|
||||
path: path.to_path_buf(),
|
||||
write_legacy_snapshots: false,
|
||||
preserve_on_drop: true,
|
||||
socket_file_name: "worker.sock",
|
||||
})
|
||||
}
|
||||
|
||||
@@ -116,13 +150,24 @@ impl RuntimeDir {
|
||||
/// that only know the worker name (e.g. the TUI's attach flow)
|
||||
/// predict the same path via [`manifest::paths::worker_socket_path`].
|
||||
pub fn socket_path(&self) -> PathBuf {
|
||||
self.path.join("sock")
|
||||
self.path.join(self.socket_file_name)
|
||||
}
|
||||
|
||||
pub async fn close_socket(&self) -> Result<(), io::Error> {
|
||||
match fs::remove_file(self.socket_path()).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RuntimeDir {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.path);
|
||||
let _ = std::fs::remove_file(self.socket_path());
|
||||
if !self.preserve_on_drop {
|
||||
let _ = std::fs::remove_dir_all(&self.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -120,15 +120,15 @@ pub fn adopt_allocation(
|
||||
/// The Worker's in-memory `segment_id` can change underneath the
|
||||
/// allocation in two normal places:
|
||||
///
|
||||
/// - `Worker::compact` mints a fresh session and swaps it in.
|
||||
/// - `session_store::ensure_head_or_fork` auto-forks when another
|
||||
/// - `Worker::compact` mints a fresh Segment in the same Session.
|
||||
/// - `session_store::ensure_head_or_fork` auto-forks within that Session when another
|
||||
/// writer has advanced the store head behind our back.
|
||||
///
|
||||
/// Both paths must call this so subsequent [`lookup_segment`] queries
|
||||
/// find the live session id, not the old one. Without this update a
|
||||
/// find the live Segment id, not the old one. Without this update a
|
||||
/// concurrent `restore_from_manifest(new_id)` would see "no live
|
||||
/// writer" and proceed to register a competing allocation on the
|
||||
/// session this Worker just moved into.
|
||||
/// Segment lineage this Worker just moved into.
|
||||
///
|
||||
/// The lock is opened once and the allocation is rewritten inside the
|
||||
/// guard, so the segment_id collision check is atomic with the
|
||||
|
||||
@@ -3067,13 +3067,13 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
|
||||
|
||||
/// Compact the current session by summarising history via a
|
||||
/// disposable Engine, then replacing history with
|
||||
/// `[summary, ...recent_turns]` and creating a new session.
|
||||
/// `[summary, ...recent_turns]` in a new Segment of the same Session.
|
||||
///
|
||||
/// The summary Engine uses:
|
||||
/// - `compaction.model` from the manifest if configured, or
|
||||
/// - a clone of the main LlmClient via `clone_boxed()`.
|
||||
///
|
||||
/// Returns the new session ID.
|
||||
/// Returns the new Segment ID. The Worker keeps its Session ID.
|
||||
pub async fn compact(&mut self, retained_tokens: u64) -> Result<SegmentId, WorkerError> {
|
||||
use crate::compact::worker::{
|
||||
CompactWorkerContext, CompactWorkerInterceptor, add_reference_tool,
|
||||
|
||||
@@ -421,7 +421,11 @@ async fn pre_run_compact_success_broadcasts_start_and_done() {
|
||||
// Drain run events so only compact events remain in `rx`.
|
||||
let _ = drain(&mut rx);
|
||||
|
||||
let session_before = worker.session_id();
|
||||
let segment_before = worker.segment_id();
|
||||
worker.try_pre_run_compact().await;
|
||||
assert_eq!(worker.session_id(), session_before);
|
||||
assert_ne!(worker.segment_id(), segment_before);
|
||||
|
||||
let events = drain(&mut rx);
|
||||
let kinds: Vec<&str> = events
|
||||
@@ -442,7 +446,7 @@ async fn pre_run_compact_success_broadcasts_start_and_done() {
|
||||
"unexpected CompactFailed in {kinds:?}"
|
||||
);
|
||||
|
||||
// CompactDone carries the new session id.
|
||||
// CompactDone carries the new Segment ID; the Session ID is unchanged.
|
||||
let new_id_in_event = events.iter().find_map(|e| match e {
|
||||
Event::CompactDone { new_segment_id } => Some(*new_segment_id),
|
||||
_ => None,
|
||||
@@ -583,11 +587,11 @@ async fn compact_resets_extract_pointer_so_extract_can_fire_again() {
|
||||
);
|
||||
|
||||
// Compact runs. Without the fix the in-memory pointer would still
|
||||
// reference the old session's history_len.
|
||||
// reference the old Segment's history_len.
|
||||
worker.try_pre_run_compact().await;
|
||||
assert!(
|
||||
worker.extract_pointer().is_none(),
|
||||
"extract_pointer must be reset to None after compact (matches cold-restore on the new session)"
|
||||
"extract_pointer must be reset to None after compact (matches cold-restore on the new Segment)"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user