Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

refactor: make runtime worker state authoritative

Browse Source
This commit is contained in:
Keisuke Hirata 2026-07-28 19:21:09 +09:00
parent 6114cc9018
commit 7a1b5e97c1
No known key found for this signature in database
11 changed files with 530 additions and 1266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
crates/worker-runtime/src/catalog.rs
View File

@ -1,4 +1,3 @@
use crate::execution::WorkerExecutionStatus;
use crate::identity::{WorkerId, WorkerRef}; use crate::identity::{WorkerId, WorkerRef};
use crate::interaction::WorkerInput; use crate::interaction::WorkerInput;
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef}; use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef};
@ -211,14 +210,16 @@ pub struct CreateWorkerRequest {
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub enum WorkerStatus { pub enum WorkerStatus {
Idle,
Running, Running,
Paused,
Stopped, Stopped,
Cancelled, Cancelled,
} }
impl WorkerStatus { impl WorkerStatus {
pub fn is_active(self) -> bool { pub fn is_active(self) -> bool {
matches!(self, Self::Running) matches!(self, Self::Idle | Self::Running | Self::Paused)
} }
} }
@ -228,7 +229,8 @@ pub struct WorkerSummary {
pub worker_ref: WorkerRef, pub worker_ref: WorkerRef,
pub worker_id: WorkerId, pub worker_id: WorkerId,
pub status: WorkerStatus, pub status: WorkerStatus,
pub execution: WorkerExecutionStatus, #[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>,
pub profile: ProfileSelector, pub profile: ProfileSelector,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub display_name: Option<String>, pub display_name: Option<String>,
@ -244,7 +246,8 @@ pub struct WorkerDetail {
pub worker_ref: WorkerRef, pub worker_ref: WorkerRef,
pub worker_id: WorkerId, pub worker_id: WorkerId,
pub status: WorkerStatus, pub status: WorkerStatus,
pub execution: WorkerExecutionStatus, #[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>,
pub profile: ProfileSelector, pub profile: ProfileSelector,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub display_name: Option<String>, pub display_name: Option<String>,

220
crates/worker-runtime/src/execution.rs
View File

@ -11,44 +11,6 @@ use serde::{Deserialize, Serialize};
use std::fmt; use std::fmt;
use std::sync::Arc; use std::sync::Arc;
/// Persisted execution lifecycle visible through Worker catalog/detail responses.
///
/// This is intentionally a worker lifecycle projection, not a transport/backend
/// handle state. Runtime restart boundaries invalidate live handles, so a
/// persisted `alive` worker is restored on startup; restore deferral keeps the
/// worker `stopped`, while structural restore failure marks it `corrupted`.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkerExecutionBackendKind {
/// Restoreable persisted state exists, but no live execution handle is attached.
#[default]
#[serde(alias = "unconnected", alias = "stale")]
Stopped,
/// A live execution handle is currently attached. Legacy `connected` maps here.
#[serde(alias = "connected")]
Alive,
/// Persisted execution state is structurally invalid and cannot be restored.
Corrupted,
}
/// Durable, non-authority execution binding projection.
///
/// This records only enough identity to restore through the same backend kind.
/// It is not a live handle and must not contain sockets, paths, credentials, or
/// provider-private authority.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerExecutionBindingIdentity {
pub backend_id: String,
}
impl WorkerExecutionBindingIdentity {
pub fn from_handle(handle: &WorkerExecutionHandle) -> Self {
Self {
backend_id: handle.backend_id.clone(),
}
}
}
/// Current execution-side run state for a Worker. /// Current execution-side run state for a Worker.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
@ -155,113 +117,6 @@ impl WorkerExecutionResult {
} }
} }
/// Execution status surfaced in Worker summary/detail responses.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerExecutionStatus {
pub backend: WorkerExecutionBackendKind,
pub run_state: WorkerExecutionRunState,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub binding: Option<WorkerExecutionBindingIdentity>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub restore_dry_check: Option<WorkerRestoreDryCheck>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum WorkerRestoreDryCheckStatus {
Valid,
Invalid,
Unavailable,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkerRestoreDryCheck {
pub status: WorkerRestoreDryCheckStatus,
pub code: String,
pub message: String,
}
impl WorkerRestoreDryCheck {
pub fn valid(message: impl Into<String>) -> Self {
Self {
status: WorkerRestoreDryCheckStatus::Valid,
code: "restore_dry_check_valid".to_string(),
message: message.into(),
}
}
pub fn invalid(code: impl Into<String>, message: impl Into<String>) -> Self {
Self {
status: WorkerRestoreDryCheckStatus::Invalid,
code: code.into(),
message: message.into(),
}
}
pub fn unavailable(code: impl Into<String>, message: impl Into<String>) -> Self {
Self {
status: WorkerRestoreDryCheckStatus::Unavailable,
code: code.into(),
message: message.into(),
}
}
}
impl WorkerExecutionStatus {
pub fn stopped() -> Self {
Self::default()
}
pub fn alive(run_state: WorkerExecutionRunState) -> Self {
Self {
backend: WorkerExecutionBackendKind::Alive,
run_state,
binding: None,
working_directory: None,
restore_dry_check: None,
}
}
pub fn stopped_from(mut previous: Self) -> Self {
previous.backend = WorkerExecutionBackendKind::Stopped;
previous.run_state = WorkerExecutionRunState::Stopped;
previous.restore_dry_check = None;
previous
}
pub fn corrupted(mut previous: Self) -> Self {
previous.backend = WorkerExecutionBackendKind::Corrupted;
previous.run_state = WorkerExecutionRunState::Errored;
previous.restore_dry_check = None;
previous
}
pub fn with_binding(mut self, binding: WorkerExecutionBindingIdentity) -> Self {
self.binding = Some(binding);
self
}
pub fn with_working_directory(mut self, status: WorkingDirectoryStatus) -> Self {
self.working_directory = Some(status);
self
}
pub fn with_result(mut self, result: WorkerExecutionResult) -> Self {
self.run_state = result.run_state;
self
}
pub fn with_restore_dry_check(
mut self,
restore_dry_check: Option<WorkerRestoreDryCheck>,
) -> Self {
self.restore_dry_check = restore_dry_check;
self
}
}
/// Opaque per-Worker execution handle returned by a backend. /// Opaque per-Worker execution handle returned by a backend.
/// ///
/// The handle is a typed token for routing calls back into the same backend. It /// The handle is a typed token for routing calls back into the same backend. It
@ -362,23 +217,13 @@ pub struct WorkerExecutionSpawnRequest {
pub config_bundle: Option<ConfigBundle>, pub config_bundle: Option<ConfigBundle>,
} }
/// Request passed to a [`WorkerExecutionBackend`] when validating a restore without side effects.
#[derive(Clone, Debug)]
pub struct WorkerExecutionRestoreDryRequest {
pub worker_ref: WorkerRef,
pub request: crate::catalog::CreateWorkerRequest,
pub previous_execution: WorkerExecutionStatus,
pub working_directory: Option<WorkingDirectoryBinding>,
pub config_bundle: Option<ConfigBundle>,
}
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker. /// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct WorkerExecutionRestoreRequest { pub struct WorkerExecutionRestoreRequest {
pub worker_ref: WorkerRef, pub worker_ref: WorkerRef,
pub request: crate::catalog::CreateWorkerRequest, pub request: crate::catalog::CreateWorkerRequest,
pub context: WorkerExecutionContext, pub context: WorkerExecutionContext,
pub previous_execution: WorkerExecutionStatus, pub previous_working_directory: Option<WorkingDirectoryStatus>,
pub working_directory: Option<WorkingDirectoryBinding>, pub working_directory: Option<WorkingDirectoryBinding>,
pub config_bundle: Option<ConfigBundle>, pub config_bundle: Option<ConfigBundle>,
} }
@ -414,16 +259,6 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult; fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult;
fn dry_restore_worker(
&self,
_request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
WorkerRestoreDryCheck::unavailable(
"restore_dry_check_unsupported",
"execution backend does not support side-effect-free restore validation",
)
}
fn restore_worker( fn restore_worker(
&self, &self,
_request: WorkerExecutionRestoreRequest, _request: WorkerExecutionRestoreRequest,
@ -536,10 +371,6 @@ impl WorkerExecutionBackendRef {
}) })
} }
pub(crate) fn backend_id(&self) -> &str {
&self.backend_id
}
pub(crate) fn spawn_worker( pub(crate) fn spawn_worker(
&self, &self,
request: WorkerExecutionSpawnRequest, request: WorkerExecutionSpawnRequest,
@ -547,13 +378,6 @@ impl WorkerExecutionBackendRef {
self.backend.spawn_worker(request) self.backend.spawn_worker(request)
} }
pub(crate) fn dry_restore_worker(
&self,
request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
self.backend.dry_restore_worker(request)
}
pub(crate) fn restore_worker( pub(crate) fn restore_worker(
&self, &self,
request: WorkerExecutionRestoreRequest, request: WorkerExecutionRestoreRequest,
@ -635,45 +459,3 @@ impl fmt::Debug for WorkerExecutionBackendRef {
.finish_non_exhaustive() .finish_non_exhaustive()
} }
} }
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn execution_backend_kind_accepts_legacy_values() {
let connected: WorkerExecutionStatus = serde_json::from_value(json!({
"backend": "connected",
"run_state": "idle",
"binding": { "backend_id": "worker-crate" }
}))
.unwrap();
assert_eq!(connected.backend, WorkerExecutionBackendKind::Alive);
let stale: WorkerExecutionStatus = serde_json::from_value(json!({
"backend": "stale",
"run_state": "stopped",
"binding": { "backend_id": "worker-crate" }
}))
.unwrap();
assert_eq!(stale.backend, WorkerExecutionBackendKind::Stopped);
let unconnected: WorkerExecutionStatus = serde_json::from_value(json!({
"backend": "unconnected",
"run_state": "stopped"
}))
.unwrap();
assert_eq!(unconnected.backend, WorkerExecutionBackendKind::Stopped);
}
#[test]
fn execution_status_serializes_without_last_result() {
let status = WorkerExecutionStatus::alive(WorkerExecutionRunState::Idle).with_result(
WorkerExecutionResult::rejected(WorkerExecutionOperation::Input, "transient"),
);
let serialized = serde_json::to_value(status).unwrap();
assert_eq!(serialized["backend"], "alive");
assert!(serialized.get("last_result").is_none());
}
}

142
crates/worker-runtime/src/fs_store.rs
View File

@ -1,12 +1,9 @@
use crate::catalog::{CreateWorkerRequest, WorkerStatus}; use crate::catalog::{CreateWorkerRequest, WorkingDirectoryStatus};
use crate::config_bundle::ConfigBundle; use crate::config_bundle::ConfigBundle;
use crate::diagnostics::{DiagnosticSeverity, RuntimeDiagnostic}; use crate::diagnostics::{DiagnosticSeverity, RuntimeDiagnostic};
use crate::error::RuntimeError; use crate::error::RuntimeError;
use crate::execution::WorkerExecutionStatus;
use crate::identity::{WorkerId, WorkerRef}; use crate::identity::{WorkerId, WorkerRef};
use crate::management::{RuntimeBackendKind, RuntimeLimits, RuntimeStatus}; use crate::management::{RuntimeBackendKind, RuntimeLimits, RuntimeStatus};
#[cfg(feature = "ws-server")]
use crate::observation::WorkerObservationEvent;
use crate::observation::{EventCursor, RuntimeEvent, RuntimeEventBatch}; use crate::observation::{EventCursor, RuntimeEvent, RuntimeEventBatch};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::BTreeMap; use std::collections::BTreeMap;
@ -21,8 +18,7 @@ const EVENTS_FILE: &str = "events.jsonl";
const WORKERS_DIR: &str = "workers"; const WORKERS_DIR: &str = "workers";
const LEGACY_RUNTIMES_DIR: &str = "runtimes"; const LEGACY_RUNTIMES_DIR: &str = "runtimes";
const WORKER_FILE: &str = "worker.json"; const WORKER_FILE: &str = "worker.json";
#[cfg(feature = "ws-server")] const LEGACY_OBSERVATIONS_FILE: &str = "observations.jsonl";
const OBSERVATIONS_FILE: &str = "observations.jsonl";
static NEXT_TMP_SEQUENCE: AtomicU64 = AtomicU64::new(1); static NEXT_TMP_SEQUENCE: AtomicU64 = AtomicU64::new(1);
@ -153,11 +149,7 @@ impl FsRuntimeStore {
&WorkerSnapshot::from_persisted(worker), &WorkerSnapshot::from_persisted(worker),
"write worker snapshot", "write worker snapshot",
)?; )?;
#[cfg(feature = "ws-server")] remove_legacy_observations(&worker_dir);
ensure_file_exists(
&worker_dir.join(OBSERVATIONS_FILE),
"create observations log",
)?;
Ok(()) Ok(())
} }
@ -180,19 +172,6 @@ impl FsRuntimeStore {
append_json_line(&self.events_path(), event, "append event") append_json_line(&self.events_path(), event, "append event")
} }
#[cfg(feature = "ws-server")]
pub(crate) fn append_worker_observation_event(
&self,
event: &WorkerObservationEvent,
) -> Result<(), RuntimeError> {
self.ensure_worker_ref(&event.worker_ref)?;
append_json_line(
&self.observations_path(&event.worker_ref.worker_id),
event,
"append worker observation",
)
}
pub(crate) fn load_runtime_state(&self) -> Result<PersistedRuntimeState, RuntimeError> { pub(crate) fn load_runtime_state(&self) -> Result<PersistedRuntimeState, RuntimeError> {
let runtime_path = self.runtime_path(); let runtime_path = self.runtime_path();
let mut snapshot: RuntimeSnapshot = read_json(&runtime_path, "read runtime snapshot")?; let mut snapshot: RuntimeSnapshot = read_json(&runtime_path, "read runtime snapshot")?;
@ -229,9 +208,6 @@ impl FsRuntimeStore {
})?; })?;
worker_dirs.sort_by_key(|entry| entry.path()); worker_dirs.sort_by_key(|entry| entry.path());
#[cfg(feature = "ws-server")]
let mut observation_events = Vec::new();
for entry in worker_dirs { for entry in worker_dirs {
let path = entry.path(); let path = entry.path();
if !path.is_dir() { if !path.is_dir() {
@ -263,43 +239,7 @@ impl FsRuntimeStore {
); );
continue; continue;
} }
#[cfg(feature = "ws-server")] remove_legacy_observations(&path);
let worker_observations = match read_json_lines::<WorkerObservationEvent>(
&path.join(OBSERVATIONS_FILE),
"read worker observations",
) {
Ok(events) => events,
Err(_error) => {
record_worker_load_diagnostic(
&mut snapshot,
Some(worker_snapshot.worker_ref.clone()),
"ignored worker with unreadable observations while loading runtime store",
);
continue;
}
};
#[cfg(feature = "ws-server")]
let mut observations_valid = true;
#[cfg(feature = "ws-server")]
for event in &worker_observations {
if self.ensure_worker_ref(&event.worker_ref).is_err()
|| event.worker_ref.worker_id != worker_snapshot.worker_id
{
observations_valid = false;
break;
}
}
#[cfg(feature = "ws-server")]
if !observations_valid {
record_worker_load_diagnostic(
&mut snapshot,
Some(worker_snapshot.worker_ref.clone()),
"ignored worker with invalid observations while loading runtime store",
);
continue;
}
#[cfg(feature = "ws-server")]
observation_events.extend(worker_observations);
let worker = worker_snapshot.into_persisted(); let worker = worker_snapshot.into_persisted();
if workers.insert(worker.worker_id.clone(), worker).is_some() { if workers.insert(worker.worker_id.clone(), worker).is_some() {
record_worker_load_diagnostic( record_worker_load_diagnostic(
@ -310,15 +250,7 @@ impl FsRuntimeStore {
} }
} }
#[cfg(feature = "ws-server")] Ok(snapshot.into_persisted(events, workers))
observation_events.sort_by_key(|event| event.sequence);
Ok(snapshot.into_persisted(
events,
workers,
#[cfg(feature = "ws-server")]
observation_events,
))
} }
fn ensure_worker_ref(&self, _worker_ref: &WorkerRef) -> Result<(), RuntimeError> { fn ensure_worker_ref(&self, _worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
@ -336,10 +268,12 @@ impl FsRuntimeStore {
fn worker_dir(&self, worker_id: &WorkerId) -> PathBuf { fn worker_dir(&self, worker_id: &WorkerId) -> PathBuf {
self.root.join(WORKERS_DIR).join(worker_id.to_string()) self.root.join(WORKERS_DIR).join(worker_id.to_string())
} }
}
#[cfg(feature = "ws-server")] fn remove_legacy_observations(worker_dir: &Path) {
fn observations_path(&self, worker_id: &WorkerId) -> PathBuf { let path = worker_dir.join(LEGACY_OBSERVATIONS_FILE);
self.worker_dir(worker_id).join(OBSERVATIONS_FILE) if path.is_file() {
let _ = fs::remove_file(path);
} }
} }
@ -360,8 +294,6 @@ pub(crate) struct PersistedRuntimeState {
pub(crate) workers: BTreeMap<WorkerId, PersistedWorkerRecord>, pub(crate) workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
pub(crate) config_bundles: BTreeMap<String, ConfigBundle>, pub(crate) config_bundles: BTreeMap<String, ConfigBundle>,
pub(crate) events: Vec<RuntimeEvent>, pub(crate) events: Vec<RuntimeEvent>,
#[cfg(feature = "ws-server")]
pub(crate) observation_events: Vec<WorkerObservationEvent>,
pub(crate) diagnostics: Vec<RuntimeDiagnostic>, pub(crate) diagnostics: Vec<RuntimeDiagnostic>,
} }
@ -369,9 +301,8 @@ pub(crate) struct PersistedRuntimeState {
pub(crate) struct PersistedWorkerRecord { pub(crate) struct PersistedWorkerRecord {
pub(crate) worker_ref: WorkerRef, pub(crate) worker_ref: WorkerRef,
pub(crate) worker_id: WorkerId, pub(crate) worker_id: WorkerId,
pub(crate) status: WorkerStatus,
pub(crate) request: CreateWorkerRequest, pub(crate) request: CreateWorkerRequest,
pub(crate) execution: WorkerExecutionStatus, pub(crate) working_directory: Option<WorkingDirectoryStatus>,
pub(crate) last_event_id: u64, pub(crate) last_event_id: u64,
} }
@ -447,7 +378,6 @@ impl RuntimeSnapshot {
self, self,
events: Vec<RuntimeEvent>, events: Vec<RuntimeEvent>,
workers: BTreeMap<WorkerId, PersistedWorkerRecord>, workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
#[cfg(feature = "ws-server")] observation_events: Vec<WorkerObservationEvent>,
) -> PersistedRuntimeState { ) -> PersistedRuntimeState {
PersistedRuntimeState { PersistedRuntimeState {
display_name: self.display_name, display_name: self.display_name,
@ -459,8 +389,6 @@ impl RuntimeSnapshot {
workers, workers,
config_bundles: self.config_bundles, config_bundles: self.config_bundles,
events, events,
#[cfg(feature = "ws-server")]
observation_events,
diagnostics: self.diagnostics, diagnostics: self.diagnostics,
} }
} }
@ -471,22 +399,31 @@ struct WorkerSnapshot {
schema_version: u32, schema_version: u32,
worker_ref: WorkerRef, worker_ref: WorkerRef,
worker_id: WorkerId, worker_id: WorkerId,
status: WorkerStatus,
request: CreateWorkerRequest, request: CreateWorkerRequest,
#[serde(default = "WorkerExecutionStatus::stopped")] #[serde(default, skip_serializing_if = "Option::is_none")]
execution: WorkerExecutionStatus, working_directory: Option<WorkingDirectoryStatus>,
/// One-way migration input for schema-v1 snapshots. New snapshots never
/// write the removed execution projection.
#[serde(default, rename = "execution", skip_serializing)]
legacy_execution: Option<LegacyWorkerExecutionProjection>,
last_event_id: u64, last_event_id: u64,
} }
#[derive(Clone, Debug, Deserialize)]
struct LegacyWorkerExecutionProjection {
#[serde(default)]
working_directory: Option<WorkingDirectoryStatus>,
}
impl WorkerSnapshot { impl WorkerSnapshot {
fn from_persisted(worker: &PersistedWorkerRecord) -> Self { fn from_persisted(worker: &PersistedWorkerRecord) -> Self {
Self { Self {
schema_version: SCHEMA_VERSION, schema_version: SCHEMA_VERSION,
worker_ref: worker.worker_ref.clone(), worker_ref: worker.worker_ref.clone(),
worker_id: worker.worker_id.clone(), worker_id: worker.worker_id.clone(),
status: worker.status,
request: worker.request.clone(), request: worker.request.clone(),
execution: worker.execution.clone(), working_directory: worker.working_directory.clone(),
legacy_execution: None,
last_event_id: worker.last_event_id, last_event_id: worker.last_event_id,
} }
} }
@ -519,9 +456,11 @@ impl WorkerSnapshot {
PersistedWorkerRecord { PersistedWorkerRecord {
worker_ref: self.worker_ref, worker_ref: self.worker_ref,
worker_id: self.worker_id, worker_id: self.worker_id,
status: self.status,
request: self.request, request: self.request,
execution: self.execution, working_directory: self.working_directory.or_else(|| {
self.legacy_execution
.and_then(|execution| execution.working_directory)
}),
last_event_id: self.last_event_id, last_event_id: self.last_event_id,
} }
} }
@ -759,29 +698,6 @@ where
}) })
} }
fn ensure_file_exists(path: &Path, operation: &'static str) -> Result<(), RuntimeError> {
let parent = path.parent().ok_or_else(|| RuntimeError::StoreCorrupt {
operation,
path: path.to_path_buf(),
message: "path has no parent directory".to_string(),
})?;
fs::create_dir_all(parent).map_err(|source| RuntimeError::StoreIo {
operation,
path: parent.to_path_buf(),
source,
})?;
OpenOptions::new()
.create(true)
.append(true)
.open(path)
.and_then(|file| file.sync_all())
.map_err(|source| RuntimeError::StoreIo {
operation,
path: path.to_path_buf(),
source,
})
}
fn tmp_path_for(path: &Path) -> PathBuf { fn tmp_path_for(path: &Path) -> PathBuf {
let sequence = NEXT_TMP_SEQUENCE.fetch_add(1, Ordering::Relaxed); let sequence = NEXT_TMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
let file_name = path let file_name = path

66
crates/worker-runtime/src/http_server.rs
View File

@ -179,6 +179,7 @@ pub fn runtime_http_router_with_auth(
get(get_worker).delete(delete_worker), get(get_worker).delete(delete_worker),
) )
.route("/v1/workers/{worker_id}/input", post(send_worker_input)) .route("/v1/workers/{worker_id}/input", post(send_worker_input))
.route("/v1/workers/{worker_id}/restore", post(restore_worker))
.route( .route(
"/v1/workers/{worker_id}/completions", "/v1/workers/{worker_id}/completions",
post(worker_completions), post(worker_completions),
@ -467,6 +468,18 @@ async fn create_worker(
Ok(Json(RuntimeHttpWorkerResponse { worker })) Ok(Json(RuntimeHttpWorkerResponse { worker }))
} }
async fn restore_worker(
State(state): State<RuntimeHttpState>,
Path(worker_id): Path<String>,
) -> RestResult<RuntimeHttpWorkerResponse> {
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
let worker = state
.runtime
.restore_worker(&worker_ref)
.map_err(RuntimeHttpRestError::runtime)?;
Ok(Json(RuntimeHttpWorkerResponse { worker }))
}
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
async fn worker_protocol_ws( async fn worker_protocol_ws(
State(state): State<RuntimeHttpState>, State(state): State<RuntimeHttpState>,
@ -512,6 +525,22 @@ async fn worker_protocol_ws_session(
} }
}, },
}; };
// Observation cursors are process-local. After a Runtime restart (or
// bounded backlog expiry), the current Worker snapshot is authoritative
// and replay resumes from the current in-memory tail.
if runtime
.read_worker_observation_events(&worker_ref, cursor)
.is_err()
{
cursor = match runtime.worker_observation_cursor_now(&worker_ref) {
Ok(cursor) => cursor,
Err(error) => {
let event = protocol_error_event(error.to_string());
let _ = send_protocol_event(&mut socket, &event).await;
return;
}
};
}
let mut receiver = match runtime.subscribe_worker_observation() { let mut receiver = match runtime.subscribe_worker_observation() {
Ok(receiver) => receiver, Ok(receiver) => receiver,
@ -806,7 +835,7 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
if path.starts_with("/v1/config-bundles") || path.starts_with("/v1/working-directories") { if path.starts_with("/v1/config-bundles") || path.starts_with("/v1/working-directories") {
return Some("workers:create"); return Some("workers:create");
} }
if path.ends_with("/input") { if path.ends_with("/input") || path.ends_with("/restore") {
return Some("workers:input"); return Some("workers:input");
} }
if path.ends_with("/stop") || path.ends_with("/cancel") { if path.ends_with("/stop") || path.ends_with("/cancel") {
@ -948,14 +977,14 @@ pub enum RuntimeHttpServerError {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::catalog::{ConfigBundleRef, ProfileSelector}; use crate::catalog::{ConfigBundleRef, ProfileSelector, WorkerStatus};
use crate::config_bundle::{ use crate::config_bundle::{
ConfigBundle, ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor, ConfigBundle, ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
}; };
use crate::execution::{ use crate::execution::{
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
WorkerExecutionSpawnResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
}; };
use crate::management::RuntimeOptions; use crate::management::RuntimeOptions;
use axum::body::to_bytes; use axum::body::to_bytes;
@ -1038,6 +1067,17 @@ mod tests {
} }
} }
fn restore_worker(
&self,
request: WorkerExecutionRestoreRequest,
) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
run_state: WorkerExecutionRunState::Idle,
working_directory: request.previous_working_directory,
}
}
fn dispatch_input( fn dispatch_input(
&self, &self,
_handle: &WorkerExecutionHandle, _handle: &WorkerExecutionHandle,
@ -1156,6 +1196,24 @@ mod tests {
let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await; let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await;
assert_eq!(stop.ack.worker_ref, created.worker.worker_ref); assert_eq!(stop.ack.worker_ref, created.worker.worker_ref);
let response = empty_request(
app.clone(),
Method::POST,
&format!("/v1/workers/{}/restore", created.worker.worker_id),
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let restored: RuntimeHttpWorkerResponse = read_json(response).await;
assert_eq!(restored.worker.status, WorkerStatus::Idle);
let response = empty_request(
app.clone(),
Method::POST,
&format!("/v1/workers/{}/stop", created.worker.worker_id),
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let response = empty_request( let response = empty_request(
app.clone(), app.clone(),
Method::POST, Method::POST,

3
crates/worker-runtime/src/main.rs
View File

@ -82,8 +82,7 @@ fn build_runtime(config: &ProcessConfig) -> Result<Runtime, ProcessError> {
let fs_paths = config.resolved_fs_paths(); let fs_paths = config.resolved_fs_paths();
let mut factory = ProfileRuntimeWorkerFactory::new(fs_paths.worker_dir.join("worker-root")) let mut factory = ProfileRuntimeWorkerFactory::new(fs_paths.worker_dir.join("worker-root"))
.with_store_dir(fs_paths.worker_dir.join("sessions")) .with_store_dir(fs_paths.worker_dir.join("sessions"))
.with_worker_metadata_dir(fs_paths.worker_dir.join("metadata")) .with_worker_metadata_dir(fs_paths.worker_dir.join("metadata"));
.with_runtime_base_dir(fs_paths.worker_dir.join("runtime"));
if let Some(endpoint) = config.backend_resource_endpoint.clone() { if let Some(endpoint) = config.backend_resource_endpoint.clone() {
factory = factory.with_resource_client(Arc::new( factory = factory.with_resource_client(Arc::new(
worker_runtime::resource::HttpBackendResourceClient::new( worker_runtime::resource::HttpBackendResourceClient::new(

792
crates/worker-runtime/src/runtime.rs
View File

File diff suppressed because it is too large Load Diff

266
crates/worker-runtime/src/worker_backend.rs
View File

@ -10,7 +10,7 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::future::Future; use std::future::Future;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, mpsc}; use std::sync::{Arc, Mutex, mpsc};
use std::time::Duration; use std::time::Duration;
@ -20,9 +20,8 @@ use crate::catalog::{
}; };
use crate::execution::{ use crate::execution::{
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
WorkerExecutionRestoreDryRequest, WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
WorkerRestoreDryCheck,
}; };
use crate::interaction::{WorkerInput, WorkerInputKind}; use crate::interaction::{WorkerInput, WorkerInputKind};
use crate::resource::{BackendResourceClient, ProfileSourceArchiveCache}; use crate::resource::{BackendResourceClient, ProfileSourceArchiveCache};
@ -33,7 +32,7 @@ use async_trait::async_trait;
use manifest::paths; use manifest::paths;
use protocol::{Method, Segment, WorkerStatus}; use protocol::{Method, Segment, WorkerStatus};
use session_store::FsStore; use session_store::FsStore;
use session_store::{CombinedStore, FsWorkerStore, Store, WorkerMetadataStore}; use session_store::{CombinedStore, FsWorkerStore};
use tokio::runtime::Runtime; use tokio::runtime::Runtime;
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
use tokio::sync::broadcast; use tokio::sync::broadcast;
@ -47,6 +46,42 @@ use worker::{
const DEFAULT_BACKEND_ID: &str = "worker-crate"; const DEFAULT_BACKEND_ID: &str = "worker-crate";
const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10); const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
static NEXT_RUNTIME_ARTIFACT_ROOT: AtomicU64 = AtomicU64::new(1);
#[derive(Clone)]
enum RuntimeArtifactRoot {
Owned(Arc<OwnedRuntimeArtifactRoot>),
External(PathBuf),
}
impl RuntimeArtifactRoot {
fn owned() -> Self {
let sequence = NEXT_RUNTIME_ARTIFACT_ROOT.fetch_add(1, Ordering::Relaxed);
Self::Owned(Arc::new(OwnedRuntimeArtifactRoot {
path: std::env::temp_dir().join(format!(
"yoi-worker-runtime-artifacts-{}-{sequence}",
std::process::id()
)),
}))
}
fn path(&self) -> &std::path::Path {
match self {
Self::Owned(root) => &root.path,
Self::External(path) => path,
}
}
}
struct OwnedRuntimeArtifactRoot {
path: PathBuf,
}
impl Drop for OwnedRuntimeArtifactRoot {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.path);
}
}
/// Factory seam used by [`WorkerRuntimeExecutionBackend`] to construct a real /// Factory seam used by [`WorkerRuntimeExecutionBackend`] to construct a real
/// controller-backed Worker for a Runtime catalog entry. /// controller-backed Worker for a Runtime catalog entry.
@ -61,16 +96,6 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static {
&self, &self,
request: WorkerExecutionRestoreRequest, request: WorkerExecutionRestoreRequest,
) -> Result<WorkerHandle, String>; ) -> Result<WorkerHandle, String>;
async fn dry_restore_controller(
&self,
_request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
WorkerRestoreDryCheck::unavailable(
"restore_dry_check_unsupported",
"runtime worker factory does not support side-effect-free restore validation",
)
}
} }
/// Production factory that resolves a normal Worker profile and spawns it under /// Production factory that resolves a normal Worker profile and spawns it under
@ -81,7 +106,7 @@ pub struct ProfileRuntimeWorkerFactory {
store_dir: Option<PathBuf>, store_dir: Option<PathBuf>,
worker_metadata_dir: Option<PathBuf>, worker_metadata_dir: Option<PathBuf>,
profile: Option<String>, profile: Option<String>,
runtime_base_dir: Option<PathBuf>, runtime_base_dir: RuntimeArtifactRoot,
resource_client: Option<Arc<dyn BackendResourceClient>>, resource_client: Option<Arc<dyn BackendResourceClient>>,
profile_archive_cache: Arc<ProfileSourceArchiveCache>, profile_archive_cache: Arc<ProfileSourceArchiveCache>,
} }
@ -94,7 +119,7 @@ impl ProfileRuntimeWorkerFactory {
store_dir: None, store_dir: None,
worker_metadata_dir: None, worker_metadata_dir: None,
profile: None, profile: None,
runtime_base_dir: None, runtime_base_dir: RuntimeArtifactRoot::owned(),
resource_client: None, resource_client: None,
profile_archive_cache: Arc::new(ProfileSourceArchiveCache::default()), profile_archive_cache: Arc::new(ProfileSourceArchiveCache::default()),
} }
@ -118,7 +143,7 @@ impl ProfileRuntimeWorkerFactory {
} }
pub fn with_runtime_base_dir(mut self, runtime_base_dir: impl Into<PathBuf>) -> Self { pub fn with_runtime_base_dir(mut self, runtime_base_dir: impl Into<PathBuf>) -> Self {
self.runtime_base_dir = Some(runtime_base_dir.into()); self.runtime_base_dir = RuntimeArtifactRoot::External(runtime_base_dir.into());
self self
} }
@ -146,10 +171,7 @@ impl ProfileRuntimeWorkerFactory {
} }
fn runtime_base_dir(&self) -> Result<PathBuf, String> { fn runtime_base_dir(&self) -> Result<PathBuf, String> {
self.runtime_base_dir Ok(self.runtime_base_dir.path().to_path_buf())
.clone()
.or_else(|| worker::runtime::dir::default_base().ok())
.ok_or_else(|| "could not resolve worker runtime directory".to_string())
} }
fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String { fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String {
@ -339,52 +361,6 @@ async fn fetch_profile_source_archive_http(
) )
} }
impl ProfileRuntimeWorkerFactory {
async fn try_dry_restore_controller(
&self,
request: WorkerExecutionRestoreDryRequest,
) -> Result<(), String> {
let WorkerExecutionRestoreDryRequest { worker_ref, .. } = request;
let store_dir = self.store_dir()?;
let session_store = FsStore::new(&store_dir).map_err(|err| {
format!(
"failed to initialize session store at {}: {err}",
store_dir.display()
)
})?;
let worker_metadata_dir = self.worker_metadata_dir(&store_dir);
let worker_metadata_store = FsWorkerStore::new(&worker_metadata_dir).map_err(|err| {
format!(
"failed to initialize worker metadata store at {}: {err}",
worker_metadata_dir.display()
)
})?;
let worker_name = Self::runtime_worker_name_for_ref(&worker_ref);
let active_segment = worker_metadata_store
.read_by_name(&worker_name)
.map_err(|err| format!("failed to read Worker metadata for {worker_name}: {err}"))?
.ok_or_else(|| format!("missing Worker metadata for {worker_name}"))?
.active
.ok_or_else(|| format!("Worker metadata for {worker_name} has no active segment"))?;
let active_segment_id = active_segment
.segment_id
.ok_or_else(|| format!("Worker metadata for {worker_name} has no active segment id"))?;
let raw_entries = session_store
.read_all(active_segment.session_id, active_segment_id)
.map_err(|err| {
format!(
"failed to read active segment {} for session {}: {err}",
active_segment_id, active_segment.session_id
)
})?;
let state = session_store::collect_state(&raw_entries);
if state.entries_count == 0 {
return Err(format!("active segment {active_segment_id} is empty"));
}
Ok(())
}
}
#[async_trait] #[async_trait]
impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
async fn spawn_controller( async fn spawn_controller(
@ -462,22 +438,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
.map_err(|err| format!("failed to create Worker from profile: {err}"))?; .map_err(|err| format!("failed to create Worker from profile: {err}"))?;
let runtime_base = self.runtime_base_dir()?; let runtime_base = self.runtime_base_dir()?;
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base) let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base)
.await .await
.map_err(|err| format!("failed to spawn Worker controller: {err}"))?; .map_err(|err| format!("failed to spawn Worker controller: {err}"))?;
Ok(handle) Ok(handle)
} }
async fn dry_restore_controller(
&self,
request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
match self.try_dry_restore_controller(request).await {
Ok(()) => WorkerRestoreDryCheck::valid("restore dry-check succeeded"),
Err(message) => WorkerRestoreDryCheck::invalid("restore_dry_check_failed", message),
}
}
async fn restore_controller( async fn restore_controller(
&self, &self,
request: WorkerExecutionRestoreRequest, request: WorkerExecutionRestoreRequest,
@ -557,7 +523,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
}; };
let runtime_base = self.runtime_base_dir()?; let runtime_base = self.runtime_base_dir()?;
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base) let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base)
.await .await
.map_err(|err| format!("failed to spawn restored Worker controller: {err}"))?; .map_err(|err| format!("failed to spawn restored Worker controller: {err}"))?;
Ok(handle) Ok(handle)
@ -968,80 +934,11 @@ where
) )
} }
fn dry_restore_worker(
&self,
mut request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
let working_directory = match request.previous_execution.working_directory.clone() {
Some(status) => {
let Some(materializer) = self.working_directory_materializer.as_ref() else {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
"persisted worker has a working directory binding, but no materializer is configured for this runtime backend",
);
};
let relative_cwd = request
.request
.working_directory
.as_ref()
.and_then(|working_directory| working_directory.relative_cwd.as_deref());
match materializer
.bind_working_directory(&status.summary.working_directory_id, relative_cwd)
{
Ok(binding) => Some(binding),
Err(error) => {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
error.to_string(),
);
}
}
}
None if request.request.working_directory_request.is_some() => {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
"persisted worker requested a working directory, but no persisted working directory binding is available to restore",
);
}
None if request.request.working_directory.is_some() => {
let Some(materializer) = self.working_directory_materializer.as_ref() else {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
"persisted worker has a working directory claim, but no materializer is configured for this runtime backend",
);
};
let working_directory =
request.request.working_directory.as_ref().expect("checked");
match materializer.bind_working_directory(
&working_directory.working_directory_id,
working_directory.relative_cwd.as_deref(),
) {
Ok(binding) => Some(binding),
Err(error) => {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
error.to_string(),
);
}
}
}
None => None,
};
request.working_directory = working_directory;
let factory = self.factory.clone();
self.run_on_adapter_runtime(
async move { Ok(factory.dry_restore_controller(request).await) },
)
.unwrap_or_else(|message| {
WorkerRestoreDryCheck::unavailable("restore_dry_check_unavailable", message)
})
}
fn restore_worker( fn restore_worker(
&self, &self,
mut request: WorkerExecutionRestoreRequest, mut request: WorkerExecutionRestoreRequest,
) -> WorkerExecutionSpawnResult { ) -> WorkerExecutionSpawnResult {
let working_directory = match request.previous_execution.working_directory.clone() { let working_directory = match request.previous_working_directory.clone() {
Some(status) => { Some(status) => {
let Some(materializer) = self.working_directory_materializer.as_ref() else { let Some(materializer) = self.working_directory_materializer.as_ref() else {
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected(
@ -1341,6 +1238,7 @@ mod tests {
use llm_engine::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent}; use llm_engine::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent};
use llm_engine::llm_client::{ClientError, LlmClient, Request}; use llm_engine::llm_client::{ClientError, LlmClient, Request};
use manifest::{Scope, WorkerManifest}; use manifest::{Scope, WorkerManifest};
use session_store::WorkerMetadataStore;
#[derive(Clone)] #[derive(Clone)]
struct MockClient { struct MockClient {
@ -1462,9 +1360,10 @@ mod tests {
) )
.await .await
.map_err(|err| err.to_string())?; .map_err(|err| err.to_string())?;
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &self.runtime_base) let (handle, _shutdown_rx) =
.await WorkerController::spawn_runtime_managed(worker, &self.runtime_base)
.map_err(|err| err.to_string())?; .await
.map_err(|err| err.to_string())?;
Ok(handle) Ok(handle)
} }
async fn restore_controller( async fn restore_controller(
@ -1689,60 +1588,6 @@ mod tests {
.expect("embedded archive should resolve without Backend resource client"); .expect("embedded archive should resolve without Backend resource client");
} }
#[tokio::test]
async fn dry_restore_validates_saved_worker_state_without_profile_source_resolution() {
let root = tempfile::tempdir().unwrap();
let store_dir = root.path().join("sessions");
let worker_metadata_dir = root.path().join("workers");
let worker_ref = WorkerRef::new(crate::identity::WorkerId::new(1));
let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref);
let session_id = session_store::new_session_id();
let segment_id = session_store::new_segment_id();
FsStore::new(&store_dir)
.unwrap()
.create_segment(
session_id,
segment_id,
&[session_store::LogEntry::Invoke {
ts: 1,
trigger: protocol::InvokeKind::UserSend,
}],
)
.unwrap();
FsWorkerStore::new(&worker_metadata_dir)
.unwrap()
.set_active(
&worker_name,
Some(session_store::WorkerActiveSegmentRef::active_segment(
session_id, segment_id,
)),
None,
)
.unwrap();
let request = create_request("restore");
let result = ProfileRuntimeWorkerFactory::new(root.path())
.with_store_dir(&store_dir)
.with_worker_metadata_dir(&worker_metadata_dir)
.dry_restore_controller(WorkerExecutionRestoreDryRequest {
worker_ref,
request,
previous_execution: crate::execution::WorkerExecutionStatus::alive(
WorkerExecutionRunState::Idle,
),
working_directory: None,
config_bundle: None,
})
.await;
assert_eq!(
result.status,
crate::execution::WorkerRestoreDryCheckStatus::Valid,
"{}",
result.message
);
}
#[tokio::test] #[tokio::test]
async fn restore_pending_worker_uses_saved_manifest_snapshot() { async fn restore_pending_worker_uses_saved_manifest_snapshot() {
let root = tempfile::tempdir().unwrap(); let root = tempfile::tempdir().unwrap();
@ -1793,9 +1638,7 @@ mod tests {
worker_ref: worker_ref.clone(), worker_ref: worker_ref.clone(),
request, request,
context: test_execution_context(worker_ref), context: test_execution_context(worker_ref),
previous_execution: crate::execution::WorkerExecutionStatus::alive( previous_working_directory: None,
WorkerExecutionRunState::Idle,
),
working_directory: None, working_directory: None,
config_bundle: None, config_bundle: None,
}) })
@ -1948,7 +1791,7 @@ mod tests {
); );
} }
assert!(detail.execution.working_directory.is_some()); assert!(detail.working_directory.is_some());
let cwds = observed_cwds.lock().unwrap(); let cwds = observed_cwds.lock().unwrap();
assert_eq!(cwds.len(), 1); assert_eq!(cwds.len(), 1);
let cwd = &cwds[0]; let cwd = &cwds[0];
@ -1991,7 +1834,6 @@ mod tests {
request.working_directory_request = Some(working_directory_request(repo.path())); request.working_directory_request = Some(working_directory_request(repo.path()));
let detail = runtime.create_worker(request).unwrap(); let detail = runtime.create_worker(request).unwrap();
let workdir_id = detail let workdir_id = detail
.execution
.working_directory .working_directory
.as_ref() .as_ref()
.unwrap() .unwrap()

35
crates/worker/src/controller.rs
View File

@ -193,8 +193,36 @@ pub struct WorkerController;
impl WorkerController { impl WorkerController {
pub async fn spawn<C, St>( pub async fn spawn<C, St>(
worker: Worker<C, St>,
runtime_base: &Path,
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
Self::spawn_inner(worker, runtime_base, false).await
}
/// Spawn a Worker owned by `worker-runtime`.
///
/// The controller still uses an ephemeral directory for Unix sockets and
/// tool spill artifacts, but does not write legacy pid/status/manifest
/// liveness projections.
pub async fn spawn_runtime_managed<C, St>(
worker: Worker<C, St>,
runtime_base: &Path,
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
Self::spawn_inner(worker, runtime_base, true).await
}
async fn spawn_inner<C, St>(
mut worker: Worker<C, St>, mut worker: Worker<C, St>,
runtime_base: &Path, runtime_base: &Path,
runtime_managed: bool,
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error> ) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
where where
C: LlmClient + Clone + 'static, C: LlmClient + Clone + 'static,
@ -214,8 +242,11 @@ impl WorkerController {
// the spawn-tool factories need its socket path, and before the // the spawn-tool factories need its socket path, and before the
// initial status/history writes consume the greeting we build // initial status/history writes consume the greeting we build
// after registration is complete. // after registration is complete.
let runtime_dir = let runtime_dir = Arc::new(if runtime_managed {
Arc::new(RuntimeDir::create(runtime_base, &worker.manifest().worker.name).await?); RuntimeDir::create_transient(runtime_base, &worker.manifest().worker.name).await?
} else {
RuntimeDir::create(runtime_base, &worker.manifest().worker.name).await?
});
let spawner_name = worker.manifest().worker.name.clone(); let spawner_name = worker.manifest().worker.name.clone();
let self_parent_socket = worker.callback_socket().cloned(); let self_parent_socket = worker.callback_socket().cloned();

42
crates/worker/src/runtime/dir.rs
View File

@ -41,6 +41,7 @@ pub struct SpawnedWorkerRecord {
/// The directory is removed on drop. /// The directory is removed on drop.
pub struct RuntimeDir { pub struct RuntimeDir {
path: PathBuf, path: PathBuf,
write_legacy_snapshots: bool,
} }
impl RuntimeDir { impl RuntimeDir {
@ -52,7 +53,23 @@ impl RuntimeDir {
let pid = std::process::id().to_string(); let pid = std::process::id().to_string();
fs::write(path.join("pid"), pid.as_bytes()).await?; fs::write(path.join("pid"), pid.as_bytes()).await?;
Ok(Self { path }) Ok(Self {
path,
write_legacy_snapshots: true,
})
}
/// Create an ephemeral Runtime-owned artifact directory.
///
/// Runtime-managed Workers keep their status in the owning Runtime and do
/// not materialize legacy pid/status/manifest projections.
pub async fn create_transient(base: &Path, worker_name: &str) -> Result<Self, io::Error> {
let path = base.join(worker_name);
fs::create_dir_all(&path).await?;
Ok(Self {
path,
write_legacy_snapshots: false,
})
} }
/// Create in the default base directory resolved via /// Create in the default base directory resolved via
@ -64,12 +81,18 @@ impl RuntimeDir {
/// Write status.json atomically. /// Write status.json atomically.
pub async fn write_status(&self, state: &WorkerSharedState) -> Result<(), io::Error> { pub async fn write_status(&self, state: &WorkerSharedState) -> Result<(), io::Error> {
if !self.write_legacy_snapshots {
return Ok(());
}
let content = state.status_json(); let content = state.status_json();
atomic_write(&self.path.join("status.json"), content.as_bytes()).await atomic_write(&self.path.join("status.json"), content.as_bytes()).await
} }
/// Write manifest.toml (typically once at startup). /// Write manifest.toml (typically once at startup).
pub async fn write_manifest(&self, toml: &str) -> Result<(), io::Error> { pub async fn write_manifest(&self, toml: &str) -> Result<(), io::Error> {
if !self.write_legacy_snapshots {
return Ok(());
}
atomic_write(&self.path.join("manifest.toml"), toml.as_bytes()).await atomic_write(&self.path.join("manifest.toml"), toml.as_bytes()).await
} }
@ -200,6 +223,23 @@ mod tests {
assert_eq!(content, "[engine]\nname = \"test\""); assert_eq!(content, "[engine]\nname = \"test\"");
} }
#[tokio::test]
async fn transient_directory_does_not_write_liveness_snapshots() {
let tmp = tempfile::tempdir().unwrap();
let rt = RuntimeDir::create_transient(tmp.path(), "runtime-worker")
.await
.unwrap();
rt.write_status(&test_state()).await.unwrap();
rt.write_manifest("[worker]\nname = \"runtime-worker\"")
.await
.unwrap();
assert!(!rt.path().join("pid").exists());
assert!(!rt.path().join("status.json").exists());
assert!(!rt.path().join("manifest.toml").exists());
}
#[tokio::test] #[tokio::test]
async fn write_spawned_workers_creates_file() { async fn write_spawned_workers_creates_file() {
use manifest::{Permission, ScopeRule}; use manifest::{Permission, ScopeRule};

199
crates/workspace-server/src/hosts.rs
View File

@ -26,9 +26,8 @@ use worker_runtime::config_bundle::{
ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor, ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
}; };
use worker_runtime::error::RuntimeError as EmbeddedRuntimeError; use worker_runtime::error::RuntimeError as EmbeddedRuntimeError;
use worker_runtime::execution::{ #[cfg(test)]
WorkerExecutionBackendKind, WorkerExecutionRunState, WorkerExecutionStatus, use worker_runtime::execution::WorkerExecutionRunState;
};
use worker_runtime::fs_store::FsRuntimeStoreOptions; use worker_runtime::fs_store::FsRuntimeStoreOptions;
use worker_runtime::http_server::{ use worker_runtime::http_server::{
RuntimeHttpConfigBundleAvailabilityResponse, RuntimeHttpConfigBundleSyncRequest, RuntimeHttpConfigBundleAvailabilityResponse, RuntimeHttpConfigBundleSyncRequest,
@ -1332,12 +1331,8 @@ impl EmbeddedWorkerRuntime {
Some(EmbeddedWorkerRef::new(EmbeddedWorkerId::parse(worker_id)?)) Some(EmbeddedWorkerRef::new(EmbeddedWorkerId::parse(worker_id)?))
} }
fn can_stop_embedded_worker( fn can_stop_embedded_worker(&self, status: EmbeddedWorkerStatus) -> bool {
&self, runtime_worker_can_stop(self.execution_enabled, status)
status: EmbeddedWorkerStatus,
execution: &worker_runtime::execution::WorkerExecutionStatus,
) -> bool {
runtime_worker_can_stop(self.execution_enabled, status, execution)
} }
fn map_worker_summary(&self, summary: worker_runtime::catalog::WorkerSummary) -> WorkerSummary { fn map_worker_summary(&self, summary: worker_runtime::catalog::WorkerSummary) -> WorkerSummary {
@ -1363,8 +1358,7 @@ impl EmbeddedWorkerRuntime {
visibility: "backend_internal".to_string(), visibility: "backend_internal".to_string(),
identity: "runtime_registry_worker".to_string(), identity: "runtime_registry_worker".to_string(),
}, },
state: embedded_worker_execution_status_label(summary.status, &summary.execution) state: embedded_worker_status_label(summary.status).to_string(),
.to_string(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@ -1373,15 +1367,11 @@ impl EmbeddedWorkerRuntime {
display_hint: "backend-internal worker-runtime Worker".to_string(), display_hint: "backend-internal worker-runtime Worker".to_string(),
}, },
capabilities: WorkerCapabilitySummary { capabilities: WorkerCapabilitySummary {
can_stop: self.can_stop_embedded_worker(summary.status, &summary.execution), can_stop: self.can_stop_embedded_worker(summary.status),
can_spawn_followup: false, can_spawn_followup: false,
}, },
working_directory: summary working_directory: summary.working_directory.map(|status| status.summary),
.execution diagnostics: embedded_worker_projection_diagnostics(),
.working_directory
.clone()
.map(|status| status.summary),
diagnostics: embedded_worker_projection_diagnostics(&summary.execution),
} }
} }
@ -1408,8 +1398,7 @@ impl EmbeddedWorkerRuntime {
visibility: "backend_internal".to_string(), visibility: "backend_internal".to_string(),
identity: "runtime_registry_worker".to_string(), identity: "runtime_registry_worker".to_string(),
}, },
state: embedded_worker_execution_status_label(detail.status, &detail.execution) state: embedded_worker_status_label(detail.status).to_string(),
.to_string(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@ -1418,15 +1407,11 @@ impl EmbeddedWorkerRuntime {
display_hint: "backend-internal worker-runtime Worker".to_string(), display_hint: "backend-internal worker-runtime Worker".to_string(),
}, },
capabilities: WorkerCapabilitySummary { capabilities: WorkerCapabilitySummary {
can_stop: self.can_stop_embedded_worker(detail.status, &detail.execution), can_stop: self.can_stop_embedded_worker(detail.status),
can_spawn_followup: false, can_spawn_followup: false,
}, },
working_directory: detail working_directory: detail.working_directory.map(|status| status.summary),
.execution diagnostics: embedded_worker_projection_diagnostics(),
.working_directory
.clone()
.map(|status| status.summary),
diagnostics: embedded_worker_projection_diagnostics(&detail.execution),
} }
} }
} }
@ -2289,8 +2274,7 @@ impl RemoteWorkerRuntime {
visibility: "remote_runtime".to_string(), visibility: "remote_runtime".to_string(),
identity: "runtime_registry_worker".to_string(), identity: "runtime_registry_worker".to_string(),
}, },
state: embedded_worker_execution_status_label(summary.status, &summary.execution) state: embedded_worker_status_label(summary.status).to_string(),
.to_string(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@ -2299,10 +2283,10 @@ impl RemoteWorkerRuntime {
display_hint: "Backend-proxied remote worker-runtime Worker".to_string(), display_hint: "Backend-proxied remote worker-runtime Worker".to_string(),
}, },
capabilities: WorkerCapabilitySummary { capabilities: WorkerCapabilitySummary {
can_stop: runtime_worker_can_stop(true, summary.status, &summary.execution), can_stop: runtime_worker_can_stop(true, summary.status),
can_spawn_followup: false, can_spawn_followup: false,
}, },
working_directory: summary.execution.working_directory.clone().map(|status| status.summary), working_directory: summary.working_directory.map(|status| status.summary),
diagnostics: vec![diagnostic( diagnostics: vec![diagnostic(
"remote_runtime_projection", "remote_runtime_projection",
DiagnosticSeverity::Info, DiagnosticSeverity::Info,
@ -2334,8 +2318,7 @@ impl RemoteWorkerRuntime {
visibility: "remote_runtime".to_string(), visibility: "remote_runtime".to_string(),
identity: "runtime_registry_worker".to_string(), identity: "runtime_registry_worker".to_string(),
}, },
state: embedded_worker_execution_status_label(detail.status, &detail.execution) state: embedded_worker_status_label(detail.status).to_string(),
.to_string(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@ -2344,10 +2327,10 @@ impl RemoteWorkerRuntime {
display_hint: "Backend-proxied remote worker-runtime Worker".to_string(), display_hint: "Backend-proxied remote worker-runtime Worker".to_string(),
}, },
capabilities: WorkerCapabilitySummary { capabilities: WorkerCapabilitySummary {
can_stop: runtime_worker_can_stop(true, detail.status, &detail.execution), can_stop: runtime_worker_can_stop(true, detail.status),
can_spawn_followup: false, can_spawn_followup: false,
}, },
working_directory: detail.execution.working_directory.clone().map(|status| status.summary), working_directory: detail.working_directory.map(|status| status.summary),
diagnostics: vec![diagnostic( diagnostics: vec![diagnostic(
"remote_runtime_projection", "remote_runtime_projection",
DiagnosticSeverity::Info, DiagnosticSeverity::Info,
@ -2814,94 +2797,26 @@ fn embedded_runtime_status_label(status: RuntimeStatus) -> &'static str {
} }
} }
fn runtime_worker_can_stop( fn runtime_worker_can_stop(execution_enabled: bool, status: EmbeddedWorkerStatus) -> bool {
execution_enabled: bool, execution_enabled && status.is_active()
status: EmbeddedWorkerStatus,
execution: &WorkerExecutionStatus,
) -> bool {
execution_enabled
&& status == EmbeddedWorkerStatus::Running
&& execution.backend == worker_runtime::execution::WorkerExecutionBackendKind::Alive
&& matches!(
execution.run_state,
WorkerExecutionRunState::Idle | WorkerExecutionRunState::Busy
)
} }
fn embedded_worker_status_label(status: EmbeddedWorkerStatus) -> &'static str { fn embedded_worker_status_label(status: EmbeddedWorkerStatus) -> &'static str {
match status { match status {
EmbeddedWorkerStatus::Idle => "idle",
EmbeddedWorkerStatus::Running => "running", EmbeddedWorkerStatus::Running => "running",
EmbeddedWorkerStatus::Paused => "paused",
EmbeddedWorkerStatus::Stopped => "stopped", EmbeddedWorkerStatus::Stopped => "stopped",
EmbeddedWorkerStatus::Cancelled => "cancelled", EmbeddedWorkerStatus::Cancelled => "cancelled",
} }
} }
fn embedded_worker_projection_diagnostics( fn embedded_worker_projection_diagnostics() -> Vec<RuntimeDiagnostic> {
execution: &WorkerExecutionStatus, vec![diagnostic(
) -> Vec<RuntimeDiagnostic> {
let mut diagnostics = vec![diagnostic(
"embedded_runtime_projection", "embedded_runtime_projection",
DiagnosticSeverity::Info, DiagnosticSeverity::Info,
"Worker identity is projected only as runtime_id plus worker_id; embedded runtime internals remain backend-private".to_string(), "Worker identity is projected only as runtime_id plus worker_id; embedded runtime internals remain backend-private".to_string(),
)]; )]
match execution.backend {
WorkerExecutionBackendKind::Stopped => diagnostics.push(diagnostic(
"embedded_worker_execution_stopped",
DiagnosticSeverity::Info,
"Worker execution is stopped; the runtime will restore it before dispatching input or protocol methods".to_string(),
)),
WorkerExecutionBackendKind::Corrupted => diagnostics.push(diagnostic(
"embedded_worker_execution_corrupted",
DiagnosticSeverity::Error,
"Worker execution state is corrupted and cannot be restored without repair".to_string(),
)),
WorkerExecutionBackendKind::Alive => {
if execution.run_state == WorkerExecutionRunState::Rejected {
diagnostics.push(diagnostic(
"embedded_worker_execution_rejected",
DiagnosticSeverity::Warning,
"Worker execution rejected the last transient operation; retry or inspect runtime logs".to_string(),
));
}
}
}
if let Some(restore_dry_check) = execution.restore_dry_check.as_ref() {
diagnostics.push(diagnostic(
"embedded_worker_execution_restore_dry_check",
DiagnosticSeverity::Error,
format!(
"Worker restore dry-test {}: {}",
restore_dry_check.code, restore_dry_check.message
),
));
}
diagnostics
}
fn embedded_worker_execution_status_label(
status: EmbeddedWorkerStatus,
execution: &WorkerExecutionStatus,
) -> &'static str {
match status {
EmbeddedWorkerStatus::Stopped => "stopped",
EmbeddedWorkerStatus::Cancelled => "cancelled",
EmbeddedWorkerStatus::Running => match execution.backend {
worker_runtime::execution::WorkerExecutionBackendKind::Stopped => "stopped",
worker_runtime::execution::WorkerExecutionBackendKind::Corrupted => "corrupted",
worker_runtime::execution::WorkerExecutionBackendKind::Alive => {
match execution.run_state {
WorkerExecutionRunState::Idle => "idle",
WorkerExecutionRunState::Busy => "running",
WorkerExecutionRunState::Stopped => "stopped",
WorkerExecutionRunState::Rejected => "rejected",
WorkerExecutionRunState::Errored => "errored",
}
}
},
}
} }
fn default_profile_source_archive_source( fn default_profile_source_archive_source(
@ -4501,7 +4416,7 @@ mod tests {
} }
#[test] #[test]
fn remote_runtime_projection_blocks_stopped_and_corrupted_execution_stop() { fn remote_runtime_projection_uses_canonical_worker_status_for_stop_capability() {
let (base_url, server) = serve_mock_http(vec![ let (base_url, server) = serve_mock_http(vec![
mock_response( mock_response(
"GET", "GET",
@ -4510,30 +4425,10 @@ mod tests {
200, 200,
json!({ json!({
"workers": [ "workers": [
worker_json_with_execution( worker_json_with_status("remote:primary", "1", "stopped"),
"remote:primary", worker_json_with_status("remote:primary", "2", "cancelled"),
"1", worker_json_with_status("remote:primary", "3", "paused"),
"stopped", worker_json_with_status("remote:primary", "4", "idle")
"stopped",
),
worker_json_with_execution(
"remote:primary",
"2",
"corrupted",
"errored",
),
worker_json_with_execution(
"remote:primary",
"3",
"alive",
"rejected",
),
worker_json_with_execution(
"remote:primary",
"4",
"alive",
"errored",
)
] ]
}) })
.to_string(), .to_string(),
@ -4544,12 +4439,7 @@ mod tests {
true, true,
200, 200,
json!({ json!({
"worker": worker_json_with_execution( "worker": worker_json_with_status("remote:primary", "1", "stopped")})
"remote:primary",
"1",
"stopped",
"stopped",
)})
.to_string(), .to_string(),
), ),
]); ]);
@ -4569,17 +4459,14 @@ mod tests {
let workers = registry.list_workers(10); let workers = registry.list_workers(10);
assert_eq!(workers.items.len(), 4); assert_eq!(workers.items.len(), 4);
for worker in &workers.items { assert!(!workers.items[0].capabilities.can_stop);
assert!( assert!(!workers.items[1].capabilities.can_stop);
!worker.capabilities.can_stop, assert!(workers.items[2].capabilities.can_stop);
"{} should not be stoppable", assert!(workers.items[3].capabilities.can_stop);
worker.worker_id
);
}
assert_eq!(workers.items[0].state, "stopped"); assert_eq!(workers.items[0].state, "stopped");
assert_eq!(workers.items[1].state, "corrupted"); assert_eq!(workers.items[1].state, "cancelled");
assert_eq!(workers.items[2].state, "rejected"); assert_eq!(workers.items[2].state, "paused");
assert_eq!(workers.items[3].state, "errored"); assert_eq!(workers.items[3].state, "idle");
let stopped_detail = registry.worker("remote:primary", "1").unwrap(); let stopped_detail = registry.worker("remote:primary", "1").unwrap();
assert!(!stopped_detail.capabilities.can_stop); assert!(!stopped_detail.capabilities.can_stop);
@ -4764,22 +4651,20 @@ mod tests {
} }
fn worker_json(runtime_id: &str, worker_id: &str) -> serde_json::Value { fn worker_json(runtime_id: &str, worker_id: &str) -> serde_json::Value {
worker_json_with_execution(runtime_id, worker_id, "alive", "idle") worker_json_with_status(runtime_id, worker_id, "idle")
} }
fn worker_json_with_execution( fn worker_json_with_status(
runtime_id: &str, runtime_id: &str,
worker_id: &str, worker_id: &str,
backend: &str, status: &str,
run_state: &str,
) -> serde_json::Value { ) -> serde_json::Value {
let worker_id = worker_id.parse::<u64>().unwrap(); let worker_id = worker_id.parse::<u64>().unwrap();
json!({ json!({
"worker_ref": { "runtime_id": runtime_id, "worker_id": worker_id }, "worker_ref": { "runtime_id": runtime_id, "worker_id": worker_id },
"runtime_id": runtime_id, "runtime_id": runtime_id,
"worker_id": worker_id, "worker_id": worker_id,
"status": "running", "status": status,
"execution": { "backend": backend, "run_state": run_state },
"intent": { "kind": "role", "role": "coder", "purpose": "remote test" }, "intent": { "kind": "role", "role": "coder", "purpose": "remote test" },
"profile": { "kind": "builtin", "value": "coder" }, "profile": { "kind": "builtin", "value": "coder" },
"profile_source": { "profile_source": {

20
crates/workspace-server/src/server.rs
View File

@ -7192,15 +7192,6 @@ mod tests {
.cleanup_working_directory(working_directory_id) .cleanup_working_directory(working_directory_id)
} }
fn dry_restore_worker(
&self,
_request: worker_runtime::execution::WorkerExecutionRestoreDryRequest,
) -> worker_runtime::execution::WorkerRestoreDryCheck {
worker_runtime::execution::WorkerRestoreDryCheck::valid(
"deterministic test backend dry restore ok",
)
}
fn spawn_worker( fn spawn_worker(
&self, &self,
request: worker_runtime::execution::WorkerExecutionSpawnRequest, request: worker_runtime::execution::WorkerExecutionSpawnRequest,
@ -9024,8 +9015,7 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn embedded_runtime_fs_store_restores_catalog_config_bundle_and_corrupts_failed_execution() async fn embedded_runtime_fs_store_restores_catalog_and_stops_failed_execution() {
{
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
let config = test_server_config(dir.path().join("workspace")); let config = test_server_config(dir.path().join("workspace"));
let store_root = config.embedded_runtime_store_root.clone(); let store_root = config.embedded_runtime_store_root.clone();
@ -9112,14 +9102,8 @@ mod tests {
.runtime .runtime
.worker("embedded-worker-runtime", &worker_id) .worker("embedded-worker-runtime", &worker_id)
.expect("restored worker"); .expect("restored worker");
assert_eq!(restored_worker.state, "corrupted"); assert_eq!(restored_worker.state, "stopped");
assert!(!restored_worker.capabilities.can_stop); assert!(!restored_worker.capabilities.can_stop);
assert!(
restored_worker
.diagnostics
.iter()
.any(|diagnostic| diagnostic.code == "embedded_worker_execution_corrupted")
);
let bundles = restored let bundles = restored
.runtime .runtime