Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

refactor: make runtime worker state authoritative

Browse Source
This commit is contained in:
Hare
2026-07-28 19:21:09 +09:00
parent 6114cc9018
commit 7a1b5e97c1
11 changed files with 530 additions and 1266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/worker-runtime/src/catalog.rs
+7 -4
View File
@@ -1,4 +1,3 @@
use crate::execution::WorkerExecutionStatus;
use crate::identity::{WorkerId, WorkerRef};
use crate::interaction::WorkerInput;
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef};
@@ -211,14 +210,16 @@ pub struct CreateWorkerRequest {
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkerStatus {
Idle,
Running,
Paused,
Stopped,
Cancelled,
}
impl WorkerStatus {
pub fn is_active(self) -> bool {
matches!(self, Self::Running)
matches!(self, Self::Idle | Self::Running | Self::Paused)
}
}
@@ -228,7 +229,8 @@ pub struct WorkerSummary {
pub worker_ref: WorkerRef,
pub worker_id: WorkerId,
pub status: WorkerStatus,
pub execution: WorkerExecutionStatus,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>,
pub profile: ProfileSelector,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub display_name: Option<String>,
@@ -244,7 +246,8 @@ pub struct WorkerDetail {
pub worker_ref: WorkerRef,
pub worker_id: WorkerId,
pub status: WorkerStatus,
pub execution: WorkerExecutionStatus,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>,
pub profile: ProfileSelector,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub display_name: Option<String>,
crates/worker-runtime/src/execution.rs
+1 -219
View File
@@ -11,44 +11,6 @@ use serde::{Deserialize, Serialize};
use std::fmt;
use std::sync::Arc;
/// Persisted execution lifecycle visible through Worker catalog/detail responses.
///
/// This is intentionally a worker lifecycle projection, not a transport/backend
/// handle state. Runtime restart boundaries invalidate live handles, so a
/// persisted `alive` worker is restored on startup; restore deferral keeps the
/// worker `stopped`, while structural restore failure marks it `corrupted`.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkerExecutionBackendKind {
/// Restoreable persisted state exists, but no live execution handle is attached.
#[default]
#[serde(alias = "unconnected", alias = "stale")]
Stopped,
/// A live execution handle is currently attached. Legacy `connected` maps here.
#[serde(alias = "connected")]
Alive,
/// Persisted execution state is structurally invalid and cannot be restored.
Corrupted,
}
/// Durable, non-authority execution binding projection.
///
/// This records only enough identity to restore through the same backend kind.
/// It is not a live handle and must not contain sockets, paths, credentials, or
/// provider-private authority.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerExecutionBindingIdentity {
pub backend_id: String,
}
impl WorkerExecutionBindingIdentity {
pub fn from_handle(handle: &WorkerExecutionHandle) -> Self {
Self {
backend_id: handle.backend_id.clone(),
}
}
}
/// Current execution-side run state for a Worker.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -155,113 +117,6 @@ impl WorkerExecutionResult {
}
}
/// Execution status surfaced in Worker summary/detail responses.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerExecutionStatus {
pub backend: WorkerExecutionBackendKind,
pub run_state: WorkerExecutionRunState,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub binding: Option<WorkerExecutionBindingIdentity>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub restore_dry_check: Option<WorkerRestoreDryCheck>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum WorkerRestoreDryCheckStatus {
Valid,
Invalid,
Unavailable,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkerRestoreDryCheck {
pub status: WorkerRestoreDryCheckStatus,
pub code: String,
pub message: String,
}
impl WorkerRestoreDryCheck {
pub fn valid(message: impl Into<String>) -> Self {
Self {
status: WorkerRestoreDryCheckStatus::Valid,
code: "restore_dry_check_valid".to_string(),
message: message.into(),
}
}
pub fn invalid(code: impl Into<String>, message: impl Into<String>) -> Self {
Self {
status: WorkerRestoreDryCheckStatus::Invalid,
code: code.into(),
message: message.into(),
}
}
pub fn unavailable(code: impl Into<String>, message: impl Into<String>) -> Self {
Self {
status: WorkerRestoreDryCheckStatus::Unavailable,
code: code.into(),
message: message.into(),
}
}
}
impl WorkerExecutionStatus {
pub fn stopped() -> Self {
Self::default()
}
pub fn alive(run_state: WorkerExecutionRunState) -> Self {
Self {
backend: WorkerExecutionBackendKind::Alive,
run_state,
binding: None,
working_directory: None,
restore_dry_check: None,
}
}
pub fn stopped_from(mut previous: Self) -> Self {
previous.backend = WorkerExecutionBackendKind::Stopped;
previous.run_state = WorkerExecutionRunState::Stopped;
previous.restore_dry_check = None;
previous
}
pub fn corrupted(mut previous: Self) -> Self {
previous.backend = WorkerExecutionBackendKind::Corrupted;
previous.run_state = WorkerExecutionRunState::Errored;
previous.restore_dry_check = None;
previous
}
pub fn with_binding(mut self, binding: WorkerExecutionBindingIdentity) -> Self {
self.binding = Some(binding);
self
}
pub fn with_working_directory(mut self, status: WorkingDirectoryStatus) -> Self {
self.working_directory = Some(status);
self
}
pub fn with_result(mut self, result: WorkerExecutionResult) -> Self {
self.run_state = result.run_state;
self
}
pub fn with_restore_dry_check(
mut self,
restore_dry_check: Option<WorkerRestoreDryCheck>,
) -> Self {
self.restore_dry_check = restore_dry_check;
self
}
}
/// Opaque per-Worker execution handle returned by a backend.
///
/// The handle is a typed token for routing calls back into the same backend. It
@@ -362,23 +217,13 @@ pub struct WorkerExecutionSpawnRequest {
pub config_bundle: Option<ConfigBundle>,
}
/// Request passed to a [`WorkerExecutionBackend`] when validating a restore without side effects.
#[derive(Clone, Debug)]
pub struct WorkerExecutionRestoreDryRequest {
pub worker_ref: WorkerRef,
pub request: crate::catalog::CreateWorkerRequest,
pub previous_execution: WorkerExecutionStatus,
pub working_directory: Option<WorkingDirectoryBinding>,
pub config_bundle: Option<ConfigBundle>,
}
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
#[derive(Clone, Debug)]
pub struct WorkerExecutionRestoreRequest {
pub worker_ref: WorkerRef,
pub request: crate::catalog::CreateWorkerRequest,
pub context: WorkerExecutionContext,
pub previous_execution: WorkerExecutionStatus,
pub previous_working_directory: Option<WorkingDirectoryStatus>,
pub working_directory: Option<WorkingDirectoryBinding>,
pub config_bundle: Option<ConfigBundle>,
}
@@ -414,16 +259,6 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult;
fn dry_restore_worker(
&self,
_request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
WorkerRestoreDryCheck::unavailable(
"restore_dry_check_unsupported",
"execution backend does not support side-effect-free restore validation",
)
}
fn restore_worker(
&self,
_request: WorkerExecutionRestoreRequest,
@@ -536,10 +371,6 @@ impl WorkerExecutionBackendRef {
})
}
pub(crate) fn backend_id(&self) -> &str {
&self.backend_id
}
pub(crate) fn spawn_worker(
&self,
request: WorkerExecutionSpawnRequest,
@@ -547,13 +378,6 @@ impl WorkerExecutionBackendRef {
self.backend.spawn_worker(request)
}
pub(crate) fn dry_restore_worker(
&self,
request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
self.backend.dry_restore_worker(request)
}
pub(crate) fn restore_worker(
&self,
request: WorkerExecutionRestoreRequest,
@@ -635,45 +459,3 @@ impl fmt::Debug for WorkerExecutionBackendRef {
.finish_non_exhaustive()
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn execution_backend_kind_accepts_legacy_values() {
let connected: WorkerExecutionStatus = serde_json::from_value(json!({
"backend": "connected",
"run_state": "idle",
"binding": { "backend_id": "worker-crate" }
}))
.unwrap();
assert_eq!(connected.backend, WorkerExecutionBackendKind::Alive);
let stale: WorkerExecutionStatus = serde_json::from_value(json!({
"backend": "stale",
"run_state": "stopped",
"binding": { "backend_id": "worker-crate" }
}))
.unwrap();
assert_eq!(stale.backend, WorkerExecutionBackendKind::Stopped);
let unconnected: WorkerExecutionStatus = serde_json::from_value(json!({
"backend": "unconnected",
"run_state": "stopped"
}))
.unwrap();
assert_eq!(unconnected.backend, WorkerExecutionBackendKind::Stopped);
}
#[test]
fn execution_status_serializes_without_last_result() {
let status = WorkerExecutionStatus::alive(WorkerExecutionRunState::Idle).with_result(
WorkerExecutionResult::rejected(WorkerExecutionOperation::Input, "transient"),
);
let serialized = serde_json::to_value(status).unwrap();
assert_eq!(serialized["backend"], "alive");
assert!(serialized.get("last_result").is_none());
}
}
crates/worker-runtime/src/fs_store.rs
+29 -113
View File
@@ -1,12 +1,9 @@
use crate::catalog::{CreateWorkerRequest, WorkerStatus};
use crate::catalog::{CreateWorkerRequest, WorkingDirectoryStatus};
use crate::config_bundle::ConfigBundle;
use crate::diagnostics::{DiagnosticSeverity, RuntimeDiagnostic};
use crate::error::RuntimeError;
use crate::execution::WorkerExecutionStatus;
use crate::identity::{WorkerId, WorkerRef};
use crate::management::{RuntimeBackendKind, RuntimeLimits, RuntimeStatus};
#[cfg(feature = "ws-server")]
use crate::observation::WorkerObservationEvent;
use crate::observation::{EventCursor, RuntimeEvent, RuntimeEventBatch};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
@@ -21,8 +18,7 @@ const EVENTS_FILE: &str = "events.jsonl";
const WORKERS_DIR: &str = "workers";
const LEGACY_RUNTIMES_DIR: &str = "runtimes";
const WORKER_FILE: &str = "worker.json";
#[cfg(feature = "ws-server")]
const OBSERVATIONS_FILE: &str = "observations.jsonl";
const LEGACY_OBSERVATIONS_FILE: &str = "observations.jsonl";
static NEXT_TMP_SEQUENCE: AtomicU64 = AtomicU64::new(1);
@@ -153,11 +149,7 @@ impl FsRuntimeStore {
&WorkerSnapshot::from_persisted(worker),
"write worker snapshot",
)?;
#[cfg(feature = "ws-server")]
ensure_file_exists(
&worker_dir.join(OBSERVATIONS_FILE),
"create observations log",
)?;
remove_legacy_observations(&worker_dir);
Ok(())
}
@@ -180,19 +172,6 @@ impl FsRuntimeStore {
append_json_line(&self.events_path(), event, "append event")
}
#[cfg(feature = "ws-server")]
pub(crate) fn append_worker_observation_event(
&self,
event: &WorkerObservationEvent,
) -> Result<(), RuntimeError> {
self.ensure_worker_ref(&event.worker_ref)?;
append_json_line(
&self.observations_path(&event.worker_ref.worker_id),
event,
"append worker observation",
)
}
pub(crate) fn load_runtime_state(&self) -> Result<PersistedRuntimeState, RuntimeError> {
let runtime_path = self.runtime_path();
let mut snapshot: RuntimeSnapshot = read_json(&runtime_path, "read runtime snapshot")?;
@@ -229,9 +208,6 @@ impl FsRuntimeStore {
})?;
worker_dirs.sort_by_key(|entry| entry.path());
#[cfg(feature = "ws-server")]
let mut observation_events = Vec::new();
for entry in worker_dirs {
let path = entry.path();
if !path.is_dir() {
@@ -263,43 +239,7 @@ impl FsRuntimeStore {
);
continue;
}
#[cfg(feature = "ws-server")]
let worker_observations = match read_json_lines::<WorkerObservationEvent>(
&path.join(OBSERVATIONS_FILE),
"read worker observations",
) {
Ok(events) => events,
Err(_error) => {
record_worker_load_diagnostic(
&mut snapshot,
Some(worker_snapshot.worker_ref.clone()),
"ignored worker with unreadable observations while loading runtime store",
);
continue;
}
};
#[cfg(feature = "ws-server")]
let mut observations_valid = true;
#[cfg(feature = "ws-server")]
for event in &worker_observations {
if self.ensure_worker_ref(&event.worker_ref).is_err()
|| event.worker_ref.worker_id != worker_snapshot.worker_id
{
observations_valid = false;
break;
}
}
#[cfg(feature = "ws-server")]
if !observations_valid {
record_worker_load_diagnostic(
&mut snapshot,
Some(worker_snapshot.worker_ref.clone()),
"ignored worker with invalid observations while loading runtime store",
);
continue;
}
#[cfg(feature = "ws-server")]
observation_events.extend(worker_observations);
remove_legacy_observations(&path);
let worker = worker_snapshot.into_persisted();
if workers.insert(worker.worker_id.clone(), worker).is_some() {
record_worker_load_diagnostic(
@@ -310,15 +250,7 @@ impl FsRuntimeStore {
}
}
#[cfg(feature = "ws-server")]
observation_events.sort_by_key(|event| event.sequence);
Ok(snapshot.into_persisted(
events,
workers,
#[cfg(feature = "ws-server")]
observation_events,
))
Ok(snapshot.into_persisted(events, workers))
}
fn ensure_worker_ref(&self, _worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
@@ -336,10 +268,12 @@ impl FsRuntimeStore {
fn worker_dir(&self, worker_id: &WorkerId) -> PathBuf {
self.root.join(WORKERS_DIR).join(worker_id.to_string())
}
}
#[cfg(feature = "ws-server")]
fn observations_path(&self, worker_id: &WorkerId) -> PathBuf {
self.worker_dir(worker_id).join(OBSERVATIONS_FILE)
fn remove_legacy_observations(worker_dir: &Path) {
let path = worker_dir.join(LEGACY_OBSERVATIONS_FILE);
if path.is_file() {
let _ = fs::remove_file(path);
}
}
@@ -360,8 +294,6 @@ pub(crate) struct PersistedRuntimeState {
pub(crate) workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
pub(crate) config_bundles: BTreeMap<String, ConfigBundle>,
pub(crate) events: Vec<RuntimeEvent>,
#[cfg(feature = "ws-server")]
pub(crate) observation_events: Vec<WorkerObservationEvent>,
pub(crate) diagnostics: Vec<RuntimeDiagnostic>,
}
@@ -369,9 +301,8 @@ pub(crate) struct PersistedRuntimeState {
pub(crate) struct PersistedWorkerRecord {
pub(crate) worker_ref: WorkerRef,
pub(crate) worker_id: WorkerId,
pub(crate) status: WorkerStatus,
pub(crate) request: CreateWorkerRequest,
pub(crate) execution: WorkerExecutionStatus,
pub(crate) working_directory: Option<WorkingDirectoryStatus>,
pub(crate) last_event_id: u64,
}
@@ -447,7 +378,6 @@ impl RuntimeSnapshot {
self,
events: Vec<RuntimeEvent>,
workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
#[cfg(feature = "ws-server")] observation_events: Vec<WorkerObservationEvent>,
) -> PersistedRuntimeState {
PersistedRuntimeState {
display_name: self.display_name,
@@ -459,8 +389,6 @@ impl RuntimeSnapshot {
workers,
config_bundles: self.config_bundles,
events,
#[cfg(feature = "ws-server")]
observation_events,
diagnostics: self.diagnostics,
}
}
@@ -471,22 +399,31 @@ struct WorkerSnapshot {
schema_version: u32,
worker_ref: WorkerRef,
worker_id: WorkerId,
status: WorkerStatus,
request: CreateWorkerRequest,
#[serde(default = "WorkerExecutionStatus::stopped")]
execution: WorkerExecutionStatus,
#[serde(default, skip_serializing_if = "Option::is_none")]
working_directory: Option<WorkingDirectoryStatus>,
/// One-way migration input for schema-v1 snapshots. New snapshots never
/// write the removed execution projection.
#[serde(default, rename = "execution", skip_serializing)]
legacy_execution: Option<LegacyWorkerExecutionProjection>,
last_event_id: u64,
}
#[derive(Clone, Debug, Deserialize)]
struct LegacyWorkerExecutionProjection {
#[serde(default)]
working_directory: Option<WorkingDirectoryStatus>,
}
impl WorkerSnapshot {
fn from_persisted(worker: &PersistedWorkerRecord) -> Self {
Self {
schema_version: SCHEMA_VERSION,
worker_ref: worker.worker_ref.clone(),
worker_id: worker.worker_id.clone(),
status: worker.status,
request: worker.request.clone(),
execution: worker.execution.clone(),
working_directory: worker.working_directory.clone(),
legacy_execution: None,
last_event_id: worker.last_event_id,
}
}
@@ -519,9 +456,11 @@ impl WorkerSnapshot {
PersistedWorkerRecord {
worker_ref: self.worker_ref,
worker_id: self.worker_id,
status: self.status,
request: self.request,
execution: self.execution,
working_directory: self.working_directory.or_else(|| {
self.legacy_execution
.and_then(|execution| execution.working_directory)
}),
last_event_id: self.last_event_id,
}
}
@@ -759,29 +698,6 @@ where
})
}
fn ensure_file_exists(path: &Path, operation: &'static str) -> Result<(), RuntimeError> {
let parent = path.parent().ok_or_else(|| RuntimeError::StoreCorrupt {
operation,
path: path.to_path_buf(),
message: "path has no parent directory".to_string(),
})?;
fs::create_dir_all(parent).map_err(|source| RuntimeError::StoreIo {
operation,
path: parent.to_path_buf(),
source,
})?;
OpenOptions::new()
.create(true)
.append(true)
.open(path)
.and_then(|file| file.sync_all())
.map_err(|source| RuntimeError::StoreIo {
operation,
path: path.to_path_buf(),
source,
})
}
fn tmp_path_for(path: &Path) -> PathBuf {
let sequence = NEXT_TMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
let file_name = path
crates/worker-runtime/src/http_server.rs
+62 -4
View File
@@ -179,6 +179,7 @@ pub fn runtime_http_router_with_auth(
get(get_worker).delete(delete_worker),
)
.route("/v1/workers/{worker_id}/input", post(send_worker_input))
.route("/v1/workers/{worker_id}/restore", post(restore_worker))
.route(
"/v1/workers/{worker_id}/completions",
post(worker_completions),
@@ -467,6 +468,18 @@ async fn create_worker(
Ok(Json(RuntimeHttpWorkerResponse { worker }))
}
async fn restore_worker(
State(state): State<RuntimeHttpState>,
Path(worker_id): Path<String>,
) -> RestResult<RuntimeHttpWorkerResponse> {
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
let worker = state
.runtime
.restore_worker(&worker_ref)
.map_err(RuntimeHttpRestError::runtime)?;
Ok(Json(RuntimeHttpWorkerResponse { worker }))
}
#[cfg(feature = "ws-server")]
async fn worker_protocol_ws(
State(state): State<RuntimeHttpState>,
@@ -512,6 +525,22 @@ async fn worker_protocol_ws_session(
}
},
};
// Observation cursors are process-local. After a Runtime restart (or
// bounded backlog expiry), the current Worker snapshot is authoritative
// and replay resumes from the current in-memory tail.
if runtime
.read_worker_observation_events(&worker_ref, cursor)
.is_err()
{
cursor = match runtime.worker_observation_cursor_now(&worker_ref) {
Ok(cursor) => cursor,
Err(error) => {
let event = protocol_error_event(error.to_string());
let _ = send_protocol_event(&mut socket, &event).await;
return;
}
};
}
let mut receiver = match runtime.subscribe_worker_observation() {
Ok(receiver) => receiver,
@@ -806,7 +835,7 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
if path.starts_with("/v1/config-bundles") || path.starts_with("/v1/working-directories") {
return Some("workers:create");
}
if path.ends_with("/input") {
if path.ends_with("/input") || path.ends_with("/restore") {
return Some("workers:input");
}
if path.ends_with("/stop") || path.ends_with("/cancel") {
@@ -948,14 +977,14 @@ pub enum RuntimeHttpServerError {
#[cfg(test)]
mod tests {
use super::*;
use crate::catalog::{ConfigBundleRef, ProfileSelector};
use crate::catalog::{ConfigBundleRef, ProfileSelector, WorkerStatus};
use crate::config_bundle::{
ConfigBundle, ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
};
use crate::execution::{
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionSpawnRequest,
WorkerExecutionSpawnResult,
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
};
use crate::management::RuntimeOptions;
use axum::body::to_bytes;
@@ -1038,6 +1067,17 @@ mod tests {
}
}
fn restore_worker(
&self,
request: WorkerExecutionRestoreRequest,
) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
run_state: WorkerExecutionRunState::Idle,
working_directory: request.previous_working_directory,
}
}
fn dispatch_input(
&self,
_handle: &WorkerExecutionHandle,
@@ -1156,6 +1196,24 @@ mod tests {
let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await;
assert_eq!(stop.ack.worker_ref, created.worker.worker_ref);
let response = empty_request(
app.clone(),
Method::POST,
&format!("/v1/workers/{}/restore", created.worker.worker_id),
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let restored: RuntimeHttpWorkerResponse = read_json(response).await;
assert_eq!(restored.worker.status, WorkerStatus::Idle);
let response = empty_request(
app.clone(),
Method::POST,
&format!("/v1/workers/{}/stop", created.worker.worker_id),
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let response = empty_request(
app.clone(),
Method::POST,
crates/worker-runtime/src/main.rs
+1 -2
View File
@@ -82,8 +82,7 @@ fn build_runtime(config: &ProcessConfig) -> Result<Runtime, ProcessError> {
let fs_paths = config.resolved_fs_paths();
let mut factory = ProfileRuntimeWorkerFactory::new(fs_paths.worker_dir.join("worker-root"))
.with_store_dir(fs_paths.worker_dir.join("sessions"))
.with_worker_metadata_dir(fs_paths.worker_dir.join("metadata"))
.with_runtime_base_dir(fs_paths.worker_dir.join("runtime"));
.with_worker_metadata_dir(fs_paths.worker_dir.join("metadata"));
if let Some(endpoint) = config.backend_resource_endpoint.clone() {
factory = factory.with_resource_client(Arc::new(
worker_runtime::resource::HttpBackendResourceClient::new(
crates/worker-runtime/src/runtime.rs
+258 -534
View File
File diff suppressed because it is too large Load Diff
crates/worker-runtime/src/worker_backend.rs
+54 -212
View File
@@ -10,7 +10,7 @@
use std::collections::HashMap;
use std::future::Future;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, mpsc};
use std::time::Duration;
@@ -20,9 +20,8 @@ use crate::catalog::{
};
use crate::execution::{
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
WorkerExecutionRestoreDryRequest, WorkerExecutionRestoreRequest, WorkerExecutionResult,
WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
WorkerRestoreDryCheck,
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
};
use crate::interaction::{WorkerInput, WorkerInputKind};
use crate::resource::{BackendResourceClient, ProfileSourceArchiveCache};
@@ -33,7 +32,7 @@ use async_trait::async_trait;
use manifest::paths;
use protocol::{Method, Segment, WorkerStatus};
use session_store::FsStore;
use session_store::{CombinedStore, FsWorkerStore, Store, WorkerMetadataStore};
use session_store::{CombinedStore, FsWorkerStore};
use tokio::runtime::Runtime;
#[cfg(feature = "ws-server")]
use tokio::sync::broadcast;
@@ -47,6 +46,42 @@ use worker::{
const DEFAULT_BACKEND_ID: &str = "worker-crate";
const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
static NEXT_RUNTIME_ARTIFACT_ROOT: AtomicU64 = AtomicU64::new(1);
#[derive(Clone)]
enum RuntimeArtifactRoot {
Owned(Arc<OwnedRuntimeArtifactRoot>),
External(PathBuf),
}
impl RuntimeArtifactRoot {
fn owned() -> Self {
let sequence = NEXT_RUNTIME_ARTIFACT_ROOT.fetch_add(1, Ordering::Relaxed);
Self::Owned(Arc::new(OwnedRuntimeArtifactRoot {
path: std::env::temp_dir().join(format!(
"yoi-worker-runtime-artifacts-{}-{sequence}",
std::process::id()
)),
}))
}
fn path(&self) -> &std::path::Path {
match self {
Self::Owned(root) => &root.path,
Self::External(path) => path,
}
}
}
struct OwnedRuntimeArtifactRoot {
path: PathBuf,
}
impl Drop for OwnedRuntimeArtifactRoot {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.path);
}
}
/// Factory seam used by [`WorkerRuntimeExecutionBackend`] to construct a real
/// controller-backed Worker for a Runtime catalog entry.
@@ -61,16 +96,6 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static {
&self,
request: WorkerExecutionRestoreRequest,
) -> Result<WorkerHandle, String>;
async fn dry_restore_controller(
&self,
_request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
WorkerRestoreDryCheck::unavailable(
"restore_dry_check_unsupported",
"runtime worker factory does not support side-effect-free restore validation",
)
}
}
/// Production factory that resolves a normal Worker profile and spawns it under
@@ -81,7 +106,7 @@ pub struct ProfileRuntimeWorkerFactory {
store_dir: Option<PathBuf>,
worker_metadata_dir: Option<PathBuf>,
profile: Option<String>,
runtime_base_dir: Option<PathBuf>,
runtime_base_dir: RuntimeArtifactRoot,
resource_client: Option<Arc<dyn BackendResourceClient>>,
profile_archive_cache: Arc<ProfileSourceArchiveCache>,
}
@@ -94,7 +119,7 @@ impl ProfileRuntimeWorkerFactory {
store_dir: None,
worker_metadata_dir: None,
profile: None,
runtime_base_dir: None,
runtime_base_dir: RuntimeArtifactRoot::owned(),
resource_client: None,
profile_archive_cache: Arc::new(ProfileSourceArchiveCache::default()),
}
@@ -118,7 +143,7 @@ impl ProfileRuntimeWorkerFactory {
}
pub fn with_runtime_base_dir(mut self, runtime_base_dir: impl Into<PathBuf>) -> Self {
self.runtime_base_dir = Some(runtime_base_dir.into());
self.runtime_base_dir = RuntimeArtifactRoot::External(runtime_base_dir.into());
self
}
@@ -146,10 +171,7 @@ impl ProfileRuntimeWorkerFactory {
}
fn runtime_base_dir(&self) -> Result<PathBuf, String> {
self.runtime_base_dir
.clone()
.or_else(|| worker::runtime::dir::default_base().ok())
.ok_or_else(|| "could not resolve worker runtime directory".to_string())
Ok(self.runtime_base_dir.path().to_path_buf())
}
fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String {
@@ -339,52 +361,6 @@ async fn fetch_profile_source_archive_http(
)
}
impl ProfileRuntimeWorkerFactory {
async fn try_dry_restore_controller(
&self,
request: WorkerExecutionRestoreDryRequest,
) -> Result<(), String> {
let WorkerExecutionRestoreDryRequest { worker_ref, .. } = request;
let store_dir = self.store_dir()?;
let session_store = FsStore::new(&store_dir).map_err(|err| {
format!(
"failed to initialize session store at {}: {err}",
store_dir.display()
)
})?;
let worker_metadata_dir = self.worker_metadata_dir(&store_dir);
let worker_metadata_store = FsWorkerStore::new(&worker_metadata_dir).map_err(|err| {
format!(
"failed to initialize worker metadata store at {}: {err}",
worker_metadata_dir.display()
)
})?;
let worker_name = Self::runtime_worker_name_for_ref(&worker_ref);
let active_segment = worker_metadata_store
.read_by_name(&worker_name)
.map_err(|err| format!("failed to read Worker metadata for {worker_name}: {err}"))?
.ok_or_else(|| format!("missing Worker metadata for {worker_name}"))?
.active
.ok_or_else(|| format!("Worker metadata for {worker_name} has no active segment"))?;
let active_segment_id = active_segment
.segment_id
.ok_or_else(|| format!("Worker metadata for {worker_name} has no active segment id"))?;
let raw_entries = session_store
.read_all(active_segment.session_id, active_segment_id)
.map_err(|err| {
format!(
"failed to read active segment {} for session {}: {err}",
active_segment_id, active_segment.session_id
)
})?;
let state = session_store::collect_state(&raw_entries);
if state.entries_count == 0 {
return Err(format!("active segment {active_segment_id} is empty"));
}
Ok(())
}
}
#[async_trait]
impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
async fn spawn_controller(
@@ -462,22 +438,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
.map_err(|err| format!("failed to create Worker from profile: {err}"))?;
let runtime_base = self.runtime_base_dir()?;
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base)
.await
.map_err(|err| format!("failed to spawn Worker controller: {err}"))?;
Ok(handle)
}
async fn dry_restore_controller(
&self,
request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
match self.try_dry_restore_controller(request).await {
Ok(()) => WorkerRestoreDryCheck::valid("restore dry-check succeeded"),
Err(message) => WorkerRestoreDryCheck::invalid("restore_dry_check_failed", message),
}
}
async fn restore_controller(
&self,
request: WorkerExecutionRestoreRequest,
@@ -557,7 +523,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
};
let runtime_base = self.runtime_base_dir()?;
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base)
.await
.map_err(|err| format!("failed to spawn restored Worker controller: {err}"))?;
Ok(handle)
@@ -968,80 +934,11 @@ where
)
}
fn dry_restore_worker(
&self,
mut request: WorkerExecutionRestoreDryRequest,
) -> WorkerRestoreDryCheck {
let working_directory = match request.previous_execution.working_directory.clone() {
Some(status) => {
let Some(materializer) = self.working_directory_materializer.as_ref() else {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
"persisted worker has a working directory binding, but no materializer is configured for this runtime backend",
);
};
let relative_cwd = request
.request
.working_directory
.as_ref()
.and_then(|working_directory| working_directory.relative_cwd.as_deref());
match materializer
.bind_working_directory(&status.summary.working_directory_id, relative_cwd)
{
Ok(binding) => Some(binding),
Err(error) => {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
error.to_string(),
);
}
}
}
None if request.request.working_directory_request.is_some() => {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
"persisted worker requested a working directory, but no persisted working directory binding is available to restore",
);
}
None if request.request.working_directory.is_some() => {
let Some(materializer) = self.working_directory_materializer.as_ref() else {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
"persisted worker has a working directory claim, but no materializer is configured for this runtime backend",
);
};
let working_directory =
request.request.working_directory.as_ref().expect("checked");
match materializer.bind_working_directory(
&working_directory.working_directory_id,
working_directory.relative_cwd.as_deref(),
) {
Ok(binding) => Some(binding),
Err(error) => {
return WorkerRestoreDryCheck::invalid(
"restore_dry_check_failed",
error.to_string(),
);
}
}
}
None => None,
};
request.working_directory = working_directory;
let factory = self.factory.clone();
self.run_on_adapter_runtime(
async move { Ok(factory.dry_restore_controller(request).await) },
)
.unwrap_or_else(|message| {
WorkerRestoreDryCheck::unavailable("restore_dry_check_unavailable", message)
})
}
fn restore_worker(
&self,
mut request: WorkerExecutionRestoreRequest,
) -> WorkerExecutionSpawnResult {
let working_directory = match request.previous_execution.working_directory.clone() {
let working_directory = match request.previous_working_directory.clone() {
Some(status) => {
let Some(materializer) = self.working_directory_materializer.as_ref() else {
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected(
@@ -1341,6 +1238,7 @@ mod tests {
use llm_engine::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent};
use llm_engine::llm_client::{ClientError, LlmClient, Request};
use manifest::{Scope, WorkerManifest};
use session_store::WorkerMetadataStore;
#[derive(Clone)]
struct MockClient {
@@ -1462,9 +1360,10 @@ mod tests {
)
.await
.map_err(|err| err.to_string())?;
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &self.runtime_base)
.await
.map_err(|err| err.to_string())?;
let (handle, _shutdown_rx) =
WorkerController::spawn_runtime_managed(worker, &self.runtime_base)
.await
.map_err(|err| err.to_string())?;
Ok(handle)
}
async fn restore_controller(
@@ -1689,60 +1588,6 @@ mod tests {
.expect("embedded archive should resolve without Backend resource client");
}
#[tokio::test]
async fn dry_restore_validates_saved_worker_state_without_profile_source_resolution() {
let root = tempfile::tempdir().unwrap();
let store_dir = root.path().join("sessions");
let worker_metadata_dir = root.path().join("workers");
let worker_ref = WorkerRef::new(crate::identity::WorkerId::new(1));
let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref);
let session_id = session_store::new_session_id();
let segment_id = session_store::new_segment_id();
FsStore::new(&store_dir)
.unwrap()
.create_segment(
session_id,
segment_id,
&[session_store::LogEntry::Invoke {
ts: 1,
trigger: protocol::InvokeKind::UserSend,
}],
)
.unwrap();
FsWorkerStore::new(&worker_metadata_dir)
.unwrap()
.set_active(
&worker_name,
Some(session_store::WorkerActiveSegmentRef::active_segment(
session_id, segment_id,
)),
None,
)
.unwrap();
let request = create_request("restore");
let result = ProfileRuntimeWorkerFactory::new(root.path())
.with_store_dir(&store_dir)
.with_worker_metadata_dir(&worker_metadata_dir)
.dry_restore_controller(WorkerExecutionRestoreDryRequest {
worker_ref,
request,
previous_execution: crate::execution::WorkerExecutionStatus::alive(
WorkerExecutionRunState::Idle,
),
working_directory: None,
config_bundle: None,
})
.await;
assert_eq!(
result.status,
crate::execution::WorkerRestoreDryCheckStatus::Valid,
"{}",
result.message
);
}
#[tokio::test]
async fn restore_pending_worker_uses_saved_manifest_snapshot() {
let root = tempfile::tempdir().unwrap();
@@ -1793,9 +1638,7 @@ mod tests {
worker_ref: worker_ref.clone(),
request,
context: test_execution_context(worker_ref),
previous_execution: crate::execution::WorkerExecutionStatus::alive(
WorkerExecutionRunState::Idle,
),
previous_working_directory: None,
working_directory: None,
config_bundle: None,
})
@@ -1948,7 +1791,7 @@ mod tests {
);
}
assert!(detail.execution.working_directory.is_some());
assert!(detail.working_directory.is_some());
let cwds = observed_cwds.lock().unwrap();
assert_eq!(cwds.len(), 1);
let cwd = &cwds[0];
@@ -1991,7 +1834,6 @@ mod tests {
request.working_directory_request = Some(working_directory_request(repo.path()));
let detail = runtime.create_worker(request).unwrap();
let workdir_id = detail
.execution
.working_directory
.as_ref()
.unwrap()