refactor: make runtime worker state authoritative
This commit is contained in:
parent
6114cc9018
commit
7a1b5e97c1
|
|
@ -1,4 +1,3 @@
|
|||
use crate::execution::WorkerExecutionStatus;
|
||||
use crate::identity::{WorkerId, WorkerRef};
|
||||
use crate::interaction::WorkerInput;
|
||||
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef};
|
||||
|
|
@ -211,14 +210,16 @@ pub struct CreateWorkerRequest {
|
|||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkerStatus {
|
||||
Idle,
|
||||
Running,
|
||||
Paused,
|
||||
Stopped,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl WorkerStatus {
|
||||
pub fn is_active(self) -> bool {
|
||||
matches!(self, Self::Running)
|
||||
matches!(self, Self::Idle | Self::Running | Self::Paused)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -228,7 +229,8 @@ pub struct WorkerSummary {
|
|||
pub worker_ref: WorkerRef,
|
||||
pub worker_id: WorkerId,
|
||||
pub status: WorkerStatus,
|
||||
pub execution: WorkerExecutionStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub working_directory: Option<WorkingDirectoryStatus>,
|
||||
pub profile: ProfileSelector,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub display_name: Option<String>,
|
||||
|
|
@ -244,7 +246,8 @@ pub struct WorkerDetail {
|
|||
pub worker_ref: WorkerRef,
|
||||
pub worker_id: WorkerId,
|
||||
pub status: WorkerStatus,
|
||||
pub execution: WorkerExecutionStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub working_directory: Option<WorkingDirectoryStatus>,
|
||||
pub profile: ProfileSelector,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub display_name: Option<String>,
|
||||
|
|
|
|||
|
|
@ -11,44 +11,6 @@ use serde::{Deserialize, Serialize};
|
|||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Persisted execution lifecycle visible through Worker catalog/detail responses.
|
||||
///
|
||||
/// This is intentionally a worker lifecycle projection, not a transport/backend
|
||||
/// handle state. Runtime restart boundaries invalidate live handles, so a
|
||||
/// persisted `alive` worker is restored on startup; restore deferral keeps the
|
||||
/// worker `stopped`, while structural restore failure marks it `corrupted`.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkerExecutionBackendKind {
|
||||
/// Restoreable persisted state exists, but no live execution handle is attached.
|
||||
#[default]
|
||||
#[serde(alias = "unconnected", alias = "stale")]
|
||||
Stopped,
|
||||
/// A live execution handle is currently attached. Legacy `connected` maps here.
|
||||
#[serde(alias = "connected")]
|
||||
Alive,
|
||||
/// Persisted execution state is structurally invalid and cannot be restored.
|
||||
Corrupted,
|
||||
}
|
||||
|
||||
/// Durable, non-authority execution binding projection.
|
||||
///
|
||||
/// This records only enough identity to restore through the same backend kind.
|
||||
/// It is not a live handle and must not contain sockets, paths, credentials, or
|
||||
/// provider-private authority.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct WorkerExecutionBindingIdentity {
|
||||
pub backend_id: String,
|
||||
}
|
||||
|
||||
impl WorkerExecutionBindingIdentity {
|
||||
pub fn from_handle(handle: &WorkerExecutionHandle) -> Self {
|
||||
Self {
|
||||
backend_id: handle.backend_id.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Current execution-side run state for a Worker.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
|
|
@ -155,113 +117,6 @@ impl WorkerExecutionResult {
|
|||
}
|
||||
}
|
||||
|
||||
/// Execution status surfaced in Worker summary/detail responses.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct WorkerExecutionStatus {
|
||||
pub backend: WorkerExecutionBackendKind,
|
||||
pub run_state: WorkerExecutionRunState,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub binding: Option<WorkerExecutionBindingIdentity>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub working_directory: Option<WorkingDirectoryStatus>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub restore_dry_check: Option<WorkerRestoreDryCheck>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkerRestoreDryCheckStatus {
|
||||
Valid,
|
||||
Invalid,
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct WorkerRestoreDryCheck {
|
||||
pub status: WorkerRestoreDryCheckStatus,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl WorkerRestoreDryCheck {
|
||||
pub fn valid(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
status: WorkerRestoreDryCheckStatus::Valid,
|
||||
code: "restore_dry_check_valid".to_string(),
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn invalid(code: impl Into<String>, message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
status: WorkerRestoreDryCheckStatus::Invalid,
|
||||
code: code.into(),
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unavailable(code: impl Into<String>, message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
status: WorkerRestoreDryCheckStatus::Unavailable,
|
||||
code: code.into(),
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WorkerExecutionStatus {
|
||||
pub fn stopped() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn alive(run_state: WorkerExecutionRunState) -> Self {
|
||||
Self {
|
||||
backend: WorkerExecutionBackendKind::Alive,
|
||||
run_state,
|
||||
binding: None,
|
||||
working_directory: None,
|
||||
restore_dry_check: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stopped_from(mut previous: Self) -> Self {
|
||||
previous.backend = WorkerExecutionBackendKind::Stopped;
|
||||
previous.run_state = WorkerExecutionRunState::Stopped;
|
||||
previous.restore_dry_check = None;
|
||||
previous
|
||||
}
|
||||
|
||||
pub fn corrupted(mut previous: Self) -> Self {
|
||||
previous.backend = WorkerExecutionBackendKind::Corrupted;
|
||||
previous.run_state = WorkerExecutionRunState::Errored;
|
||||
previous.restore_dry_check = None;
|
||||
previous
|
||||
}
|
||||
|
||||
pub fn with_binding(mut self, binding: WorkerExecutionBindingIdentity) -> Self {
|
||||
self.binding = Some(binding);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_working_directory(mut self, status: WorkingDirectoryStatus) -> Self {
|
||||
self.working_directory = Some(status);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_result(mut self, result: WorkerExecutionResult) -> Self {
|
||||
self.run_state = result.run_state;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_restore_dry_check(
|
||||
mut self,
|
||||
restore_dry_check: Option<WorkerRestoreDryCheck>,
|
||||
) -> Self {
|
||||
self.restore_dry_check = restore_dry_check;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque per-Worker execution handle returned by a backend.
|
||||
///
|
||||
/// The handle is a typed token for routing calls back into the same backend. It
|
||||
|
|
@ -362,23 +217,13 @@ pub struct WorkerExecutionSpawnRequest {
|
|||
pub config_bundle: Option<ConfigBundle>,
|
||||
}
|
||||
|
||||
/// Request passed to a [`WorkerExecutionBackend`] when validating a restore without side effects.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WorkerExecutionRestoreDryRequest {
|
||||
pub worker_ref: WorkerRef,
|
||||
pub request: crate::catalog::CreateWorkerRequest,
|
||||
pub previous_execution: WorkerExecutionStatus,
|
||||
pub working_directory: Option<WorkingDirectoryBinding>,
|
||||
pub config_bundle: Option<ConfigBundle>,
|
||||
}
|
||||
|
||||
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WorkerExecutionRestoreRequest {
|
||||
pub worker_ref: WorkerRef,
|
||||
pub request: crate::catalog::CreateWorkerRequest,
|
||||
pub context: WorkerExecutionContext,
|
||||
pub previous_execution: WorkerExecutionStatus,
|
||||
pub previous_working_directory: Option<WorkingDirectoryStatus>,
|
||||
pub working_directory: Option<WorkingDirectoryBinding>,
|
||||
pub config_bundle: Option<ConfigBundle>,
|
||||
}
|
||||
|
|
@ -414,16 +259,6 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
|||
|
||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult;
|
||||
|
||||
fn dry_restore_worker(
|
||||
&self,
|
||||
_request: WorkerExecutionRestoreDryRequest,
|
||||
) -> WorkerRestoreDryCheck {
|
||||
WorkerRestoreDryCheck::unavailable(
|
||||
"restore_dry_check_unsupported",
|
||||
"execution backend does not support side-effect-free restore validation",
|
||||
)
|
||||
}
|
||||
|
||||
fn restore_worker(
|
||||
&self,
|
||||
_request: WorkerExecutionRestoreRequest,
|
||||
|
|
@ -536,10 +371,6 @@ impl WorkerExecutionBackendRef {
|
|||
})
|
||||
}
|
||||
|
||||
pub(crate) fn backend_id(&self) -> &str {
|
||||
&self.backend_id
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_worker(
|
||||
&self,
|
||||
request: WorkerExecutionSpawnRequest,
|
||||
|
|
@ -547,13 +378,6 @@ impl WorkerExecutionBackendRef {
|
|||
self.backend.spawn_worker(request)
|
||||
}
|
||||
|
||||
pub(crate) fn dry_restore_worker(
|
||||
&self,
|
||||
request: WorkerExecutionRestoreDryRequest,
|
||||
) -> WorkerRestoreDryCheck {
|
||||
self.backend.dry_restore_worker(request)
|
||||
}
|
||||
|
||||
pub(crate) fn restore_worker(
|
||||
&self,
|
||||
request: WorkerExecutionRestoreRequest,
|
||||
|
|
@ -635,45 +459,3 @@ impl fmt::Debug for WorkerExecutionBackendRef {
|
|||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn execution_backend_kind_accepts_legacy_values() {
|
||||
let connected: WorkerExecutionStatus = serde_json::from_value(json!({
|
||||
"backend": "connected",
|
||||
"run_state": "idle",
|
||||
"binding": { "backend_id": "worker-crate" }
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(connected.backend, WorkerExecutionBackendKind::Alive);
|
||||
|
||||
let stale: WorkerExecutionStatus = serde_json::from_value(json!({
|
||||
"backend": "stale",
|
||||
"run_state": "stopped",
|
||||
"binding": { "backend_id": "worker-crate" }
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(stale.backend, WorkerExecutionBackendKind::Stopped);
|
||||
|
||||
let unconnected: WorkerExecutionStatus = serde_json::from_value(json!({
|
||||
"backend": "unconnected",
|
||||
"run_state": "stopped"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(unconnected.backend, WorkerExecutionBackendKind::Stopped);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_status_serializes_without_last_result() {
|
||||
let status = WorkerExecutionStatus::alive(WorkerExecutionRunState::Idle).with_result(
|
||||
WorkerExecutionResult::rejected(WorkerExecutionOperation::Input, "transient"),
|
||||
);
|
||||
let serialized = serde_json::to_value(status).unwrap();
|
||||
assert_eq!(serialized["backend"], "alive");
|
||||
assert!(serialized.get("last_result").is_none());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
use crate::catalog::{CreateWorkerRequest, WorkerStatus};
|
||||
use crate::catalog::{CreateWorkerRequest, WorkingDirectoryStatus};
|
||||
use crate::config_bundle::ConfigBundle;
|
||||
use crate::diagnostics::{DiagnosticSeverity, RuntimeDiagnostic};
|
||||
use crate::error::RuntimeError;
|
||||
use crate::execution::WorkerExecutionStatus;
|
||||
use crate::identity::{WorkerId, WorkerRef};
|
||||
use crate::management::{RuntimeBackendKind, RuntimeLimits, RuntimeStatus};
|
||||
#[cfg(feature = "ws-server")]
|
||||
use crate::observation::WorkerObservationEvent;
|
||||
use crate::observation::{EventCursor, RuntimeEvent, RuntimeEventBatch};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
|
@ -21,8 +18,7 @@ const EVENTS_FILE: &str = "events.jsonl";
|
|||
const WORKERS_DIR: &str = "workers";
|
||||
const LEGACY_RUNTIMES_DIR: &str = "runtimes";
|
||||
const WORKER_FILE: &str = "worker.json";
|
||||
#[cfg(feature = "ws-server")]
|
||||
const OBSERVATIONS_FILE: &str = "observations.jsonl";
|
||||
const LEGACY_OBSERVATIONS_FILE: &str = "observations.jsonl";
|
||||
|
||||
static NEXT_TMP_SEQUENCE: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
|
|
@ -153,11 +149,7 @@ impl FsRuntimeStore {
|
|||
&WorkerSnapshot::from_persisted(worker),
|
||||
"write worker snapshot",
|
||||
)?;
|
||||
#[cfg(feature = "ws-server")]
|
||||
ensure_file_exists(
|
||||
&worker_dir.join(OBSERVATIONS_FILE),
|
||||
"create observations log",
|
||||
)?;
|
||||
remove_legacy_observations(&worker_dir);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -180,19 +172,6 @@ impl FsRuntimeStore {
|
|||
append_json_line(&self.events_path(), event, "append event")
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
pub(crate) fn append_worker_observation_event(
|
||||
&self,
|
||||
event: &WorkerObservationEvent,
|
||||
) -> Result<(), RuntimeError> {
|
||||
self.ensure_worker_ref(&event.worker_ref)?;
|
||||
append_json_line(
|
||||
&self.observations_path(&event.worker_ref.worker_id),
|
||||
event,
|
||||
"append worker observation",
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn load_runtime_state(&self) -> Result<PersistedRuntimeState, RuntimeError> {
|
||||
let runtime_path = self.runtime_path();
|
||||
let mut snapshot: RuntimeSnapshot = read_json(&runtime_path, "read runtime snapshot")?;
|
||||
|
|
@ -229,9 +208,6 @@ impl FsRuntimeStore {
|
|||
})?;
|
||||
worker_dirs.sort_by_key(|entry| entry.path());
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
let mut observation_events = Vec::new();
|
||||
|
||||
for entry in worker_dirs {
|
||||
let path = entry.path();
|
||||
if !path.is_dir() {
|
||||
|
|
@ -263,43 +239,7 @@ impl FsRuntimeStore {
|
|||
);
|
||||
continue;
|
||||
}
|
||||
#[cfg(feature = "ws-server")]
|
||||
let worker_observations = match read_json_lines::<WorkerObservationEvent>(
|
||||
&path.join(OBSERVATIONS_FILE),
|
||||
"read worker observations",
|
||||
) {
|
||||
Ok(events) => events,
|
||||
Err(_error) => {
|
||||
record_worker_load_diagnostic(
|
||||
&mut snapshot,
|
||||
Some(worker_snapshot.worker_ref.clone()),
|
||||
"ignored worker with unreadable observations while loading runtime store",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
#[cfg(feature = "ws-server")]
|
||||
let mut observations_valid = true;
|
||||
#[cfg(feature = "ws-server")]
|
||||
for event in &worker_observations {
|
||||
if self.ensure_worker_ref(&event.worker_ref).is_err()
|
||||
|| event.worker_ref.worker_id != worker_snapshot.worker_id
|
||||
{
|
||||
observations_valid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "ws-server")]
|
||||
if !observations_valid {
|
||||
record_worker_load_diagnostic(
|
||||
&mut snapshot,
|
||||
Some(worker_snapshot.worker_ref.clone()),
|
||||
"ignored worker with invalid observations while loading runtime store",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
#[cfg(feature = "ws-server")]
|
||||
observation_events.extend(worker_observations);
|
||||
remove_legacy_observations(&path);
|
||||
let worker = worker_snapshot.into_persisted();
|
||||
if workers.insert(worker.worker_id.clone(), worker).is_some() {
|
||||
record_worker_load_diagnostic(
|
||||
|
|
@ -310,15 +250,7 @@ impl FsRuntimeStore {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
observation_events.sort_by_key(|event| event.sequence);
|
||||
|
||||
Ok(snapshot.into_persisted(
|
||||
events,
|
||||
workers,
|
||||
#[cfg(feature = "ws-server")]
|
||||
observation_events,
|
||||
))
|
||||
Ok(snapshot.into_persisted(events, workers))
|
||||
}
|
||||
|
||||
fn ensure_worker_ref(&self, _worker_ref: &WorkerRef) -> Result<(), RuntimeError> {
|
||||
|
|
@ -336,10 +268,12 @@ impl FsRuntimeStore {
|
|||
fn worker_dir(&self, worker_id: &WorkerId) -> PathBuf {
|
||||
self.root.join(WORKERS_DIR).join(worker_id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
fn observations_path(&self, worker_id: &WorkerId) -> PathBuf {
|
||||
self.worker_dir(worker_id).join(OBSERVATIONS_FILE)
|
||||
fn remove_legacy_observations(worker_dir: &Path) {
|
||||
let path = worker_dir.join(LEGACY_OBSERVATIONS_FILE);
|
||||
if path.is_file() {
|
||||
let _ = fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -360,8 +294,6 @@ pub(crate) struct PersistedRuntimeState {
|
|||
pub(crate) workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
|
||||
pub(crate) config_bundles: BTreeMap<String, ConfigBundle>,
|
||||
pub(crate) events: Vec<RuntimeEvent>,
|
||||
#[cfg(feature = "ws-server")]
|
||||
pub(crate) observation_events: Vec<WorkerObservationEvent>,
|
||||
pub(crate) diagnostics: Vec<RuntimeDiagnostic>,
|
||||
}
|
||||
|
||||
|
|
@ -369,9 +301,8 @@ pub(crate) struct PersistedRuntimeState {
|
|||
pub(crate) struct PersistedWorkerRecord {
|
||||
pub(crate) worker_ref: WorkerRef,
|
||||
pub(crate) worker_id: WorkerId,
|
||||
pub(crate) status: WorkerStatus,
|
||||
pub(crate) request: CreateWorkerRequest,
|
||||
pub(crate) execution: WorkerExecutionStatus,
|
||||
pub(crate) working_directory: Option<WorkingDirectoryStatus>,
|
||||
pub(crate) last_event_id: u64,
|
||||
}
|
||||
|
||||
|
|
@ -447,7 +378,6 @@ impl RuntimeSnapshot {
|
|||
self,
|
||||
events: Vec<RuntimeEvent>,
|
||||
workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
|
||||
#[cfg(feature = "ws-server")] observation_events: Vec<WorkerObservationEvent>,
|
||||
) -> PersistedRuntimeState {
|
||||
PersistedRuntimeState {
|
||||
display_name: self.display_name,
|
||||
|
|
@ -459,8 +389,6 @@ impl RuntimeSnapshot {
|
|||
workers,
|
||||
config_bundles: self.config_bundles,
|
||||
events,
|
||||
#[cfg(feature = "ws-server")]
|
||||
observation_events,
|
||||
diagnostics: self.diagnostics,
|
||||
}
|
||||
}
|
||||
|
|
@ -471,22 +399,31 @@ struct WorkerSnapshot {
|
|||
schema_version: u32,
|
||||
worker_ref: WorkerRef,
|
||||
worker_id: WorkerId,
|
||||
status: WorkerStatus,
|
||||
request: CreateWorkerRequest,
|
||||
#[serde(default = "WorkerExecutionStatus::stopped")]
|
||||
execution: WorkerExecutionStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
working_directory: Option<WorkingDirectoryStatus>,
|
||||
/// One-way migration input for schema-v1 snapshots. New snapshots never
|
||||
/// write the removed execution projection.
|
||||
#[serde(default, rename = "execution", skip_serializing)]
|
||||
legacy_execution: Option<LegacyWorkerExecutionProjection>,
|
||||
last_event_id: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
struct LegacyWorkerExecutionProjection {
|
||||
#[serde(default)]
|
||||
working_directory: Option<WorkingDirectoryStatus>,
|
||||
}
|
||||
|
||||
impl WorkerSnapshot {
|
||||
fn from_persisted(worker: &PersistedWorkerRecord) -> Self {
|
||||
Self {
|
||||
schema_version: SCHEMA_VERSION,
|
||||
worker_ref: worker.worker_ref.clone(),
|
||||
worker_id: worker.worker_id.clone(),
|
||||
status: worker.status,
|
||||
request: worker.request.clone(),
|
||||
execution: worker.execution.clone(),
|
||||
working_directory: worker.working_directory.clone(),
|
||||
legacy_execution: None,
|
||||
last_event_id: worker.last_event_id,
|
||||
}
|
||||
}
|
||||
|
|
@ -519,9 +456,11 @@ impl WorkerSnapshot {
|
|||
PersistedWorkerRecord {
|
||||
worker_ref: self.worker_ref,
|
||||
worker_id: self.worker_id,
|
||||
status: self.status,
|
||||
request: self.request,
|
||||
execution: self.execution,
|
||||
working_directory: self.working_directory.or_else(|| {
|
||||
self.legacy_execution
|
||||
.and_then(|execution| execution.working_directory)
|
||||
}),
|
||||
last_event_id: self.last_event_id,
|
||||
}
|
||||
}
|
||||
|
|
@ -759,29 +698,6 @@ where
|
|||
})
|
||||
}
|
||||
|
||||
fn ensure_file_exists(path: &Path, operation: &'static str) -> Result<(), RuntimeError> {
|
||||
let parent = path.parent().ok_or_else(|| RuntimeError::StoreCorrupt {
|
||||
operation,
|
||||
path: path.to_path_buf(),
|
||||
message: "path has no parent directory".to_string(),
|
||||
})?;
|
||||
fs::create_dir_all(parent).map_err(|source| RuntimeError::StoreIo {
|
||||
operation,
|
||||
path: parent.to_path_buf(),
|
||||
source,
|
||||
})?;
|
||||
OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(path)
|
||||
.and_then(|file| file.sync_all())
|
||||
.map_err(|source| RuntimeError::StoreIo {
|
||||
operation,
|
||||
path: path.to_path_buf(),
|
||||
source,
|
||||
})
|
||||
}
|
||||
|
||||
fn tmp_path_for(path: &Path) -> PathBuf {
|
||||
let sequence = NEXT_TMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
|
||||
let file_name = path
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ pub fn runtime_http_router_with_auth(
|
|||
get(get_worker).delete(delete_worker),
|
||||
)
|
||||
.route("/v1/workers/{worker_id}/input", post(send_worker_input))
|
||||
.route("/v1/workers/{worker_id}/restore", post(restore_worker))
|
||||
.route(
|
||||
"/v1/workers/{worker_id}/completions",
|
||||
post(worker_completions),
|
||||
|
|
@ -467,6 +468,18 @@ async fn create_worker(
|
|||
Ok(Json(RuntimeHttpWorkerResponse { worker }))
|
||||
}
|
||||
|
||||
async fn restore_worker(
|
||||
State(state): State<RuntimeHttpState>,
|
||||
Path(worker_id): Path<String>,
|
||||
) -> RestResult<RuntimeHttpWorkerResponse> {
|
||||
let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
|
||||
let worker = state
|
||||
.runtime
|
||||
.restore_worker(&worker_ref)
|
||||
.map_err(RuntimeHttpRestError::runtime)?;
|
||||
Ok(Json(RuntimeHttpWorkerResponse { worker }))
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
async fn worker_protocol_ws(
|
||||
State(state): State<RuntimeHttpState>,
|
||||
|
|
@ -512,6 +525,22 @@ async fn worker_protocol_ws_session(
|
|||
}
|
||||
},
|
||||
};
|
||||
// Observation cursors are process-local. After a Runtime restart (or
|
||||
// bounded backlog expiry), the current Worker snapshot is authoritative
|
||||
// and replay resumes from the current in-memory tail.
|
||||
if runtime
|
||||
.read_worker_observation_events(&worker_ref, cursor)
|
||||
.is_err()
|
||||
{
|
||||
cursor = match runtime.worker_observation_cursor_now(&worker_ref) {
|
||||
Ok(cursor) => cursor,
|
||||
Err(error) => {
|
||||
let event = protocol_error_event(error.to_string());
|
||||
let _ = send_protocol_event(&mut socket, &event).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let mut receiver = match runtime.subscribe_worker_observation() {
|
||||
Ok(receiver) => receiver,
|
||||
|
|
@ -806,7 +835,7 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
|||
if path.starts_with("/v1/config-bundles") || path.starts_with("/v1/working-directories") {
|
||||
return Some("workers:create");
|
||||
}
|
||||
if path.ends_with("/input") {
|
||||
if path.ends_with("/input") || path.ends_with("/restore") {
|
||||
return Some("workers:input");
|
||||
}
|
||||
if path.ends_with("/stop") || path.ends_with("/cancel") {
|
||||
|
|
@ -948,14 +977,14 @@ pub enum RuntimeHttpServerError {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::catalog::{ConfigBundleRef, ProfileSelector};
|
||||
use crate::catalog::{ConfigBundleRef, ProfileSelector, WorkerStatus};
|
||||
use crate::config_bundle::{
|
||||
ConfigBundle, ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
|
||||
};
|
||||
use crate::execution::{
|
||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
||||
WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionSpawnRequest,
|
||||
WorkerExecutionSpawnResult,
|
||||
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
|
||||
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||
};
|
||||
use crate::management::RuntimeOptions;
|
||||
use axum::body::to_bytes;
|
||||
|
|
@ -1038,6 +1067,17 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
fn restore_worker(
|
||||
&self,
|
||||
request: WorkerExecutionRestoreRequest,
|
||||
) -> WorkerExecutionSpawnResult {
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||
run_state: WorkerExecutionRunState::Idle,
|
||||
working_directory: request.previous_working_directory,
|
||||
}
|
||||
}
|
||||
|
||||
fn dispatch_input(
|
||||
&self,
|
||||
_handle: &WorkerExecutionHandle,
|
||||
|
|
@ -1156,6 +1196,24 @@ mod tests {
|
|||
let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await;
|
||||
assert_eq!(stop.ack.worker_ref, created.worker.worker_ref);
|
||||
|
||||
let response = empty_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&format!("/v1/workers/{}/restore", created.worker.worker_id),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let restored: RuntimeHttpWorkerResponse = read_json(response).await;
|
||||
assert_eq!(restored.worker.status, WorkerStatus::Idle);
|
||||
|
||||
let response = empty_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
&format!("/v1/workers/{}/stop", created.worker.worker_id),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let response = empty_request(
|
||||
app.clone(),
|
||||
Method::POST,
|
||||
|
|
|
|||
|
|
@ -82,8 +82,7 @@ fn build_runtime(config: &ProcessConfig) -> Result<Runtime, ProcessError> {
|
|||
let fs_paths = config.resolved_fs_paths();
|
||||
let mut factory = ProfileRuntimeWorkerFactory::new(fs_paths.worker_dir.join("worker-root"))
|
||||
.with_store_dir(fs_paths.worker_dir.join("sessions"))
|
||||
.with_worker_metadata_dir(fs_paths.worker_dir.join("metadata"))
|
||||
.with_runtime_base_dir(fs_paths.worker_dir.join("runtime"));
|
||||
.with_worker_metadata_dir(fs_paths.worker_dir.join("metadata"));
|
||||
if let Some(endpoint) = config.backend_resource_endpoint.clone() {
|
||||
factory = factory.with_resource_client(Arc::new(
|
||||
worker_runtime::resource::HttpBackendResourceClient::new(
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -10,7 +10,7 @@
|
|||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, mpsc};
|
||||
use std::time::Duration;
|
||||
|
||||
|
|
@ -20,9 +20,8 @@ use crate::catalog::{
|
|||
};
|
||||
use crate::execution::{
|
||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
||||
WorkerExecutionRestoreDryRequest, WorkerExecutionRestoreRequest, WorkerExecutionResult,
|
||||
WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||
WorkerRestoreDryCheck,
|
||||
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
|
||||
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||
};
|
||||
use crate::interaction::{WorkerInput, WorkerInputKind};
|
||||
use crate::resource::{BackendResourceClient, ProfileSourceArchiveCache};
|
||||
|
|
@ -33,7 +32,7 @@ use async_trait::async_trait;
|
|||
use manifest::paths;
|
||||
use protocol::{Method, Segment, WorkerStatus};
|
||||
use session_store::FsStore;
|
||||
use session_store::{CombinedStore, FsWorkerStore, Store, WorkerMetadataStore};
|
||||
use session_store::{CombinedStore, FsWorkerStore};
|
||||
use tokio::runtime::Runtime;
|
||||
#[cfg(feature = "ws-server")]
|
||||
use tokio::sync::broadcast;
|
||||
|
|
@ -47,6 +46,42 @@ use worker::{
|
|||
|
||||
const DEFAULT_BACKEND_ID: &str = "worker-crate";
|
||||
const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
static NEXT_RUNTIME_ARTIFACT_ROOT: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
#[derive(Clone)]
|
||||
enum RuntimeArtifactRoot {
|
||||
Owned(Arc<OwnedRuntimeArtifactRoot>),
|
||||
External(PathBuf),
|
||||
}
|
||||
|
||||
impl RuntimeArtifactRoot {
|
||||
fn owned() -> Self {
|
||||
let sequence = NEXT_RUNTIME_ARTIFACT_ROOT.fetch_add(1, Ordering::Relaxed);
|
||||
Self::Owned(Arc::new(OwnedRuntimeArtifactRoot {
|
||||
path: std::env::temp_dir().join(format!(
|
||||
"yoi-worker-runtime-artifacts-{}-{sequence}",
|
||||
std::process::id()
|
||||
)),
|
||||
}))
|
||||
}
|
||||
|
||||
fn path(&self) -> &std::path::Path {
|
||||
match self {
|
||||
Self::Owned(root) => &root.path,
|
||||
Self::External(path) => path,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct OwnedRuntimeArtifactRoot {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl Drop for OwnedRuntimeArtifactRoot {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
/// Factory seam used by [`WorkerRuntimeExecutionBackend`] to construct a real
|
||||
/// controller-backed Worker for a Runtime catalog entry.
|
||||
|
|
@ -61,16 +96,6 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static {
|
|||
&self,
|
||||
request: WorkerExecutionRestoreRequest,
|
||||
) -> Result<WorkerHandle, String>;
|
||||
|
||||
async fn dry_restore_controller(
|
||||
&self,
|
||||
_request: WorkerExecutionRestoreDryRequest,
|
||||
) -> WorkerRestoreDryCheck {
|
||||
WorkerRestoreDryCheck::unavailable(
|
||||
"restore_dry_check_unsupported",
|
||||
"runtime worker factory does not support side-effect-free restore validation",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Production factory that resolves a normal Worker profile and spawns it under
|
||||
|
|
@ -81,7 +106,7 @@ pub struct ProfileRuntimeWorkerFactory {
|
|||
store_dir: Option<PathBuf>,
|
||||
worker_metadata_dir: Option<PathBuf>,
|
||||
profile: Option<String>,
|
||||
runtime_base_dir: Option<PathBuf>,
|
||||
runtime_base_dir: RuntimeArtifactRoot,
|
||||
resource_client: Option<Arc<dyn BackendResourceClient>>,
|
||||
profile_archive_cache: Arc<ProfileSourceArchiveCache>,
|
||||
}
|
||||
|
|
@ -94,7 +119,7 @@ impl ProfileRuntimeWorkerFactory {
|
|||
store_dir: None,
|
||||
worker_metadata_dir: None,
|
||||
profile: None,
|
||||
runtime_base_dir: None,
|
||||
runtime_base_dir: RuntimeArtifactRoot::owned(),
|
||||
resource_client: None,
|
||||
profile_archive_cache: Arc::new(ProfileSourceArchiveCache::default()),
|
||||
}
|
||||
|
|
@ -118,7 +143,7 @@ impl ProfileRuntimeWorkerFactory {
|
|||
}
|
||||
|
||||
pub fn with_runtime_base_dir(mut self, runtime_base_dir: impl Into<PathBuf>) -> Self {
|
||||
self.runtime_base_dir = Some(runtime_base_dir.into());
|
||||
self.runtime_base_dir = RuntimeArtifactRoot::External(runtime_base_dir.into());
|
||||
self
|
||||
}
|
||||
|
||||
|
|
@ -146,10 +171,7 @@ impl ProfileRuntimeWorkerFactory {
|
|||
}
|
||||
|
||||
fn runtime_base_dir(&self) -> Result<PathBuf, String> {
|
||||
self.runtime_base_dir
|
||||
.clone()
|
||||
.or_else(|| worker::runtime::dir::default_base().ok())
|
||||
.ok_or_else(|| "could not resolve worker runtime directory".to_string())
|
||||
Ok(self.runtime_base_dir.path().to_path_buf())
|
||||
}
|
||||
|
||||
fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String {
|
||||
|
|
@ -339,52 +361,6 @@ async fn fetch_profile_source_archive_http(
|
|||
)
|
||||
}
|
||||
|
||||
impl ProfileRuntimeWorkerFactory {
|
||||
async fn try_dry_restore_controller(
|
||||
&self,
|
||||
request: WorkerExecutionRestoreDryRequest,
|
||||
) -> Result<(), String> {
|
||||
let WorkerExecutionRestoreDryRequest { worker_ref, .. } = request;
|
||||
let store_dir = self.store_dir()?;
|
||||
let session_store = FsStore::new(&store_dir).map_err(|err| {
|
||||
format!(
|
||||
"failed to initialize session store at {}: {err}",
|
||||
store_dir.display()
|
||||
)
|
||||
})?;
|
||||
let worker_metadata_dir = self.worker_metadata_dir(&store_dir);
|
||||
let worker_metadata_store = FsWorkerStore::new(&worker_metadata_dir).map_err(|err| {
|
||||
format!(
|
||||
"failed to initialize worker metadata store at {}: {err}",
|
||||
worker_metadata_dir.display()
|
||||
)
|
||||
})?;
|
||||
let worker_name = Self::runtime_worker_name_for_ref(&worker_ref);
|
||||
let active_segment = worker_metadata_store
|
||||
.read_by_name(&worker_name)
|
||||
.map_err(|err| format!("failed to read Worker metadata for {worker_name}: {err}"))?
|
||||
.ok_or_else(|| format!("missing Worker metadata for {worker_name}"))?
|
||||
.active
|
||||
.ok_or_else(|| format!("Worker metadata for {worker_name} has no active segment"))?;
|
||||
let active_segment_id = active_segment
|
||||
.segment_id
|
||||
.ok_or_else(|| format!("Worker metadata for {worker_name} has no active segment id"))?;
|
||||
let raw_entries = session_store
|
||||
.read_all(active_segment.session_id, active_segment_id)
|
||||
.map_err(|err| {
|
||||
format!(
|
||||
"failed to read active segment {} for session {}: {err}",
|
||||
active_segment_id, active_segment.session_id
|
||||
)
|
||||
})?;
|
||||
let state = session_store::collect_state(&raw_entries);
|
||||
if state.entries_count == 0 {
|
||||
return Err(format!("active segment {active_segment_id} is empty"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
async fn spawn_controller(
|
||||
|
|
@ -462,22 +438,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
|||
.map_err(|err| format!("failed to create Worker from profile: {err}"))?;
|
||||
|
||||
let runtime_base = self.runtime_base_dir()?;
|
||||
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base)
|
||||
.await
|
||||
.map_err(|err| format!("failed to spawn Worker controller: {err}"))?;
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
async fn dry_restore_controller(
|
||||
&self,
|
||||
request: WorkerExecutionRestoreDryRequest,
|
||||
) -> WorkerRestoreDryCheck {
|
||||
match self.try_dry_restore_controller(request).await {
|
||||
Ok(()) => WorkerRestoreDryCheck::valid("restore dry-check succeeded"),
|
||||
Err(message) => WorkerRestoreDryCheck::invalid("restore_dry_check_failed", message),
|
||||
}
|
||||
}
|
||||
|
||||
async fn restore_controller(
|
||||
&self,
|
||||
request: WorkerExecutionRestoreRequest,
|
||||
|
|
@ -557,7 +523,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
|||
};
|
||||
|
||||
let runtime_base = self.runtime_base_dir()?;
|
||||
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
let (handle, _shutdown_rx) = WorkerController::spawn_runtime_managed(worker, &runtime_base)
|
||||
.await
|
||||
.map_err(|err| format!("failed to spawn restored Worker controller: {err}"))?;
|
||||
Ok(handle)
|
||||
|
|
@ -968,80 +934,11 @@ where
|
|||
)
|
||||
}
|
||||
|
||||
fn dry_restore_worker(
|
||||
&self,
|
||||
mut request: WorkerExecutionRestoreDryRequest,
|
||||
) -> WorkerRestoreDryCheck {
|
||||
let working_directory = match request.previous_execution.working_directory.clone() {
|
||||
Some(status) => {
|
||||
let Some(materializer) = self.working_directory_materializer.as_ref() else {
|
||||
return WorkerRestoreDryCheck::invalid(
|
||||
"restore_dry_check_failed",
|
||||
"persisted worker has a working directory binding, but no materializer is configured for this runtime backend",
|
||||
);
|
||||
};
|
||||
let relative_cwd = request
|
||||
.request
|
||||
.working_directory
|
||||
.as_ref()
|
||||
.and_then(|working_directory| working_directory.relative_cwd.as_deref());
|
||||
match materializer
|
||||
.bind_working_directory(&status.summary.working_directory_id, relative_cwd)
|
||||
{
|
||||
Ok(binding) => Some(binding),
|
||||
Err(error) => {
|
||||
return WorkerRestoreDryCheck::invalid(
|
||||
"restore_dry_check_failed",
|
||||
error.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
None if request.request.working_directory_request.is_some() => {
|
||||
return WorkerRestoreDryCheck::invalid(
|
||||
"restore_dry_check_failed",
|
||||
"persisted worker requested a working directory, but no persisted working directory binding is available to restore",
|
||||
);
|
||||
}
|
||||
None if request.request.working_directory.is_some() => {
|
||||
let Some(materializer) = self.working_directory_materializer.as_ref() else {
|
||||
return WorkerRestoreDryCheck::invalid(
|
||||
"restore_dry_check_failed",
|
||||
"persisted worker has a working directory claim, but no materializer is configured for this runtime backend",
|
||||
);
|
||||
};
|
||||
let working_directory =
|
||||
request.request.working_directory.as_ref().expect("checked");
|
||||
match materializer.bind_working_directory(
|
||||
&working_directory.working_directory_id,
|
||||
working_directory.relative_cwd.as_deref(),
|
||||
) {
|
||||
Ok(binding) => Some(binding),
|
||||
Err(error) => {
|
||||
return WorkerRestoreDryCheck::invalid(
|
||||
"restore_dry_check_failed",
|
||||
error.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
request.working_directory = working_directory;
|
||||
let factory = self.factory.clone();
|
||||
self.run_on_adapter_runtime(
|
||||
async move { Ok(factory.dry_restore_controller(request).await) },
|
||||
)
|
||||
.unwrap_or_else(|message| {
|
||||
WorkerRestoreDryCheck::unavailable("restore_dry_check_unavailable", message)
|
||||
})
|
||||
}
|
||||
|
||||
fn restore_worker(
|
||||
&self,
|
||||
mut request: WorkerExecutionRestoreRequest,
|
||||
) -> WorkerExecutionSpawnResult {
|
||||
let working_directory = match request.previous_execution.working_directory.clone() {
|
||||
let working_directory = match request.previous_working_directory.clone() {
|
||||
Some(status) => {
|
||||
let Some(materializer) = self.working_directory_materializer.as_ref() else {
|
||||
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected(
|
||||
|
|
@ -1341,6 +1238,7 @@ mod tests {
|
|||
use llm_engine::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent};
|
||||
use llm_engine::llm_client::{ClientError, LlmClient, Request};
|
||||
use manifest::{Scope, WorkerManifest};
|
||||
use session_store::WorkerMetadataStore;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MockClient {
|
||||
|
|
@ -1462,9 +1360,10 @@ mod tests {
|
|||
)
|
||||
.await
|
||||
.map_err(|err| err.to_string())?;
|
||||
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &self.runtime_base)
|
||||
.await
|
||||
.map_err(|err| err.to_string())?;
|
||||
let (handle, _shutdown_rx) =
|
||||
WorkerController::spawn_runtime_managed(worker, &self.runtime_base)
|
||||
.await
|
||||
.map_err(|err| err.to_string())?;
|
||||
Ok(handle)
|
||||
}
|
||||
async fn restore_controller(
|
||||
|
|
@ -1689,60 +1588,6 @@ mod tests {
|
|||
.expect("embedded archive should resolve without Backend resource client");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dry_restore_validates_saved_worker_state_without_profile_source_resolution() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let store_dir = root.path().join("sessions");
|
||||
let worker_metadata_dir = root.path().join("workers");
|
||||
let worker_ref = WorkerRef::new(crate::identity::WorkerId::new(1));
|
||||
let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref);
|
||||
let session_id = session_store::new_session_id();
|
||||
let segment_id = session_store::new_segment_id();
|
||||
FsStore::new(&store_dir)
|
||||
.unwrap()
|
||||
.create_segment(
|
||||
session_id,
|
||||
segment_id,
|
||||
&[session_store::LogEntry::Invoke {
|
||||
ts: 1,
|
||||
trigger: protocol::InvokeKind::UserSend,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
FsWorkerStore::new(&worker_metadata_dir)
|
||||
.unwrap()
|
||||
.set_active(
|
||||
&worker_name,
|
||||
Some(session_store::WorkerActiveSegmentRef::active_segment(
|
||||
session_id, segment_id,
|
||||
)),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let request = create_request("restore");
|
||||
let result = ProfileRuntimeWorkerFactory::new(root.path())
|
||||
.with_store_dir(&store_dir)
|
||||
.with_worker_metadata_dir(&worker_metadata_dir)
|
||||
.dry_restore_controller(WorkerExecutionRestoreDryRequest {
|
||||
worker_ref,
|
||||
request,
|
||||
previous_execution: crate::execution::WorkerExecutionStatus::alive(
|
||||
WorkerExecutionRunState::Idle,
|
||||
),
|
||||
working_directory: None,
|
||||
config_bundle: None,
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
result.status,
|
||||
crate::execution::WorkerRestoreDryCheckStatus::Valid,
|
||||
"{}",
|
||||
result.message
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_pending_worker_uses_saved_manifest_snapshot() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
|
|
@ -1793,9 +1638,7 @@ mod tests {
|
|||
worker_ref: worker_ref.clone(),
|
||||
request,
|
||||
context: test_execution_context(worker_ref),
|
||||
previous_execution: crate::execution::WorkerExecutionStatus::alive(
|
||||
WorkerExecutionRunState::Idle,
|
||||
),
|
||||
previous_working_directory: None,
|
||||
working_directory: None,
|
||||
config_bundle: None,
|
||||
})
|
||||
|
|
@ -1948,7 +1791,7 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
assert!(detail.execution.working_directory.is_some());
|
||||
assert!(detail.working_directory.is_some());
|
||||
let cwds = observed_cwds.lock().unwrap();
|
||||
assert_eq!(cwds.len(), 1);
|
||||
let cwd = &cwds[0];
|
||||
|
|
@ -1991,7 +1834,6 @@ mod tests {
|
|||
request.working_directory_request = Some(working_directory_request(repo.path()));
|
||||
let detail = runtime.create_worker(request).unwrap();
|
||||
let workdir_id = detail
|
||||
.execution
|
||||
.working_directory
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
|
|
|
|||
|
|
@ -193,8 +193,36 @@ pub struct WorkerController;
|
|||
|
||||
impl WorkerController {
|
||||
pub async fn spawn<C, St>(
|
||||
worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Self::spawn_inner(worker, runtime_base, false).await
|
||||
}
|
||||
|
||||
/// Spawn a Worker owned by `worker-runtime`.
|
||||
///
|
||||
/// The controller still uses an ephemeral directory for Unix sockets and
|
||||
/// tool spill artifacts, but does not write legacy pid/status/manifest
|
||||
/// liveness projections.
|
||||
pub async fn spawn_runtime_managed<C, St>(
|
||||
worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Self::spawn_inner(worker, runtime_base, true).await
|
||||
}
|
||||
|
||||
async fn spawn_inner<C, St>(
|
||||
mut worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
runtime_managed: bool,
|
||||
) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
|
|
@ -214,8 +242,11 @@ impl WorkerController {
|
|||
// the spawn-tool factories need its socket path, and before the
|
||||
// initial status/history writes consume the greeting we build
|
||||
// after registration is complete.
|
||||
let runtime_dir =
|
||||
Arc::new(RuntimeDir::create(runtime_base, &worker.manifest().worker.name).await?);
|
||||
let runtime_dir = Arc::new(if runtime_managed {
|
||||
RuntimeDir::create_transient(runtime_base, &worker.manifest().worker.name).await?
|
||||
} else {
|
||||
RuntimeDir::create(runtime_base, &worker.manifest().worker.name).await?
|
||||
});
|
||||
|
||||
let spawner_name = worker.manifest().worker.name.clone();
|
||||
let self_parent_socket = worker.callback_socket().cloned();
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ pub struct SpawnedWorkerRecord {
|
|||
/// The directory is removed on drop.
|
||||
pub struct RuntimeDir {
|
||||
path: PathBuf,
|
||||
write_legacy_snapshots: bool,
|
||||
}
|
||||
|
||||
impl RuntimeDir {
|
||||
|
|
@ -52,7 +53,23 @@ impl RuntimeDir {
|
|||
let pid = std::process::id().to_string();
|
||||
fs::write(path.join("pid"), pid.as_bytes()).await?;
|
||||
|
||||
Ok(Self { path })
|
||||
Ok(Self {
|
||||
path,
|
||||
write_legacy_snapshots: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create an ephemeral Runtime-owned artifact directory.
|
||||
///
|
||||
/// Runtime-managed Workers keep their status in the owning Runtime and do
|
||||
/// not materialize legacy pid/status/manifest projections.
|
||||
pub async fn create_transient(base: &Path, worker_name: &str) -> Result<Self, io::Error> {
|
||||
let path = base.join(worker_name);
|
||||
fs::create_dir_all(&path).await?;
|
||||
Ok(Self {
|
||||
path,
|
||||
write_legacy_snapshots: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create in the default base directory resolved via
|
||||
|
|
@ -64,12 +81,18 @@ impl RuntimeDir {
|
|||
|
||||
/// Write status.json atomically.
|
||||
pub async fn write_status(&self, state: &WorkerSharedState) -> Result<(), io::Error> {
|
||||
if !self.write_legacy_snapshots {
|
||||
return Ok(());
|
||||
}
|
||||
let content = state.status_json();
|
||||
atomic_write(&self.path.join("status.json"), content.as_bytes()).await
|
||||
}
|
||||
|
||||
/// Write manifest.toml (typically once at startup).
|
||||
pub async fn write_manifest(&self, toml: &str) -> Result<(), io::Error> {
|
||||
if !self.write_legacy_snapshots {
|
||||
return Ok(());
|
||||
}
|
||||
atomic_write(&self.path.join("manifest.toml"), toml.as_bytes()).await
|
||||
}
|
||||
|
||||
|
|
@ -200,6 +223,23 @@ mod tests {
|
|||
assert_eq!(content, "[engine]\nname = \"test\"");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_directory_does_not_write_liveness_snapshots() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create_transient(tmp.path(), "runtime-worker")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
rt.write_status(&test_state()).await.unwrap();
|
||||
rt.write_manifest("[worker]\nname = \"runtime-worker\"")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!rt.path().join("pid").exists());
|
||||
assert!(!rt.path().join("status.json").exists());
|
||||
assert!(!rt.path().join("manifest.toml").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_spawned_workers_creates_file() {
|
||||
use manifest::{Permission, ScopeRule};
|
||||
|
|
|
|||
|
|
@ -26,9 +26,8 @@ use worker_runtime::config_bundle::{
|
|||
ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
|
||||
};
|
||||
use worker_runtime::error::RuntimeError as EmbeddedRuntimeError;
|
||||
use worker_runtime::execution::{
|
||||
WorkerExecutionBackendKind, WorkerExecutionRunState, WorkerExecutionStatus,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use worker_runtime::execution::WorkerExecutionRunState;
|
||||
use worker_runtime::fs_store::FsRuntimeStoreOptions;
|
||||
use worker_runtime::http_server::{
|
||||
RuntimeHttpConfigBundleAvailabilityResponse, RuntimeHttpConfigBundleSyncRequest,
|
||||
|
|
@ -1332,12 +1331,8 @@ impl EmbeddedWorkerRuntime {
|
|||
Some(EmbeddedWorkerRef::new(EmbeddedWorkerId::parse(worker_id)?))
|
||||
}
|
||||
|
||||
fn can_stop_embedded_worker(
|
||||
&self,
|
||||
status: EmbeddedWorkerStatus,
|
||||
execution: &worker_runtime::execution::WorkerExecutionStatus,
|
||||
) -> bool {
|
||||
runtime_worker_can_stop(self.execution_enabled, status, execution)
|
||||
fn can_stop_embedded_worker(&self, status: EmbeddedWorkerStatus) -> bool {
|
||||
runtime_worker_can_stop(self.execution_enabled, status)
|
||||
}
|
||||
|
||||
fn map_worker_summary(&self, summary: worker_runtime::catalog::WorkerSummary) -> WorkerSummary {
|
||||
|
|
@ -1363,8 +1358,7 @@ impl EmbeddedWorkerRuntime {
|
|||
visibility: "backend_internal".to_string(),
|
||||
identity: "runtime_registry_worker".to_string(),
|
||||
},
|
||||
state: embedded_worker_execution_status_label(summary.status, &summary.execution)
|
||||
.to_string(),
|
||||
state: embedded_worker_status_label(summary.status).to_string(),
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
|
|
@ -1373,15 +1367,11 @@ impl EmbeddedWorkerRuntime {
|
|||
display_hint: "backend-internal worker-runtime Worker".to_string(),
|
||||
},
|
||||
capabilities: WorkerCapabilitySummary {
|
||||
can_stop: self.can_stop_embedded_worker(summary.status, &summary.execution),
|
||||
can_stop: self.can_stop_embedded_worker(summary.status),
|
||||
can_spawn_followup: false,
|
||||
},
|
||||
working_directory: summary
|
||||
.execution
|
||||
.working_directory
|
||||
.clone()
|
||||
.map(|status| status.summary),
|
||||
diagnostics: embedded_worker_projection_diagnostics(&summary.execution),
|
||||
working_directory: summary.working_directory.map(|status| status.summary),
|
||||
diagnostics: embedded_worker_projection_diagnostics(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1408,8 +1398,7 @@ impl EmbeddedWorkerRuntime {
|
|||
visibility: "backend_internal".to_string(),
|
||||
identity: "runtime_registry_worker".to_string(),
|
||||
},
|
||||
state: embedded_worker_execution_status_label(detail.status, &detail.execution)
|
||||
.to_string(),
|
||||
state: embedded_worker_status_label(detail.status).to_string(),
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
|
|
@ -1418,15 +1407,11 @@ impl EmbeddedWorkerRuntime {
|
|||
display_hint: "backend-internal worker-runtime Worker".to_string(),
|
||||
},
|
||||
capabilities: WorkerCapabilitySummary {
|
||||
can_stop: self.can_stop_embedded_worker(detail.status, &detail.execution),
|
||||
can_stop: self.can_stop_embedded_worker(detail.status),
|
||||
can_spawn_followup: false,
|
||||
},
|
||||
working_directory: detail
|
||||
.execution
|
||||
.working_directory
|
||||
.clone()
|
||||
.map(|status| status.summary),
|
||||
diagnostics: embedded_worker_projection_diagnostics(&detail.execution),
|
||||
working_directory: detail.working_directory.map(|status| status.summary),
|
||||
diagnostics: embedded_worker_projection_diagnostics(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2289,8 +2274,7 @@ impl RemoteWorkerRuntime {
|
|||
visibility: "remote_runtime".to_string(),
|
||||
identity: "runtime_registry_worker".to_string(),
|
||||
},
|
||||
state: embedded_worker_execution_status_label(summary.status, &summary.execution)
|
||||
.to_string(),
|
||||
state: embedded_worker_status_label(summary.status).to_string(),
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
|
|
@ -2299,10 +2283,10 @@ impl RemoteWorkerRuntime {
|
|||
display_hint: "Backend-proxied remote worker-runtime Worker".to_string(),
|
||||
},
|
||||
capabilities: WorkerCapabilitySummary {
|
||||
can_stop: runtime_worker_can_stop(true, summary.status, &summary.execution),
|
||||
can_stop: runtime_worker_can_stop(true, summary.status),
|
||||
can_spawn_followup: false,
|
||||
},
|
||||
working_directory: summary.execution.working_directory.clone().map(|status| status.summary),
|
||||
working_directory: summary.working_directory.map(|status| status.summary),
|
||||
diagnostics: vec![diagnostic(
|
||||
"remote_runtime_projection",
|
||||
DiagnosticSeverity::Info,
|
||||
|
|
@ -2334,8 +2318,7 @@ impl RemoteWorkerRuntime {
|
|||
visibility: "remote_runtime".to_string(),
|
||||
identity: "runtime_registry_worker".to_string(),
|
||||
},
|
||||
state: embedded_worker_execution_status_label(detail.status, &detail.execution)
|
||||
.to_string(),
|
||||
state: embedded_worker_status_label(detail.status).to_string(),
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
|
|
@ -2344,10 +2327,10 @@ impl RemoteWorkerRuntime {
|
|||
display_hint: "Backend-proxied remote worker-runtime Worker".to_string(),
|
||||
},
|
||||
capabilities: WorkerCapabilitySummary {
|
||||
can_stop: runtime_worker_can_stop(true, detail.status, &detail.execution),
|
||||
can_stop: runtime_worker_can_stop(true, detail.status),
|
||||
can_spawn_followup: false,
|
||||
},
|
||||
working_directory: detail.execution.working_directory.clone().map(|status| status.summary),
|
||||
working_directory: detail.working_directory.map(|status| status.summary),
|
||||
diagnostics: vec![diagnostic(
|
||||
"remote_runtime_projection",
|
||||
DiagnosticSeverity::Info,
|
||||
|
|
@ -2814,94 +2797,26 @@ fn embedded_runtime_status_label(status: RuntimeStatus) -> &'static str {
|
|||
}
|
||||
}
|
||||
|
||||
fn runtime_worker_can_stop(
|
||||
execution_enabled: bool,
|
||||
status: EmbeddedWorkerStatus,
|
||||
execution: &WorkerExecutionStatus,
|
||||
) -> bool {
|
||||
execution_enabled
|
||||
&& status == EmbeddedWorkerStatus::Running
|
||||
&& execution.backend == worker_runtime::execution::WorkerExecutionBackendKind::Alive
|
||||
&& matches!(
|
||||
execution.run_state,
|
||||
WorkerExecutionRunState::Idle | WorkerExecutionRunState::Busy
|
||||
)
|
||||
fn runtime_worker_can_stop(execution_enabled: bool, status: EmbeddedWorkerStatus) -> bool {
|
||||
execution_enabled && status.is_active()
|
||||
}
|
||||
|
||||
fn embedded_worker_status_label(status: EmbeddedWorkerStatus) -> &'static str {
|
||||
match status {
|
||||
EmbeddedWorkerStatus::Idle => "idle",
|
||||
EmbeddedWorkerStatus::Running => "running",
|
||||
EmbeddedWorkerStatus::Paused => "paused",
|
||||
EmbeddedWorkerStatus::Stopped => "stopped",
|
||||
EmbeddedWorkerStatus::Cancelled => "cancelled",
|
||||
}
|
||||
}
|
||||
|
||||
fn embedded_worker_projection_diagnostics(
|
||||
execution: &WorkerExecutionStatus,
|
||||
) -> Vec<RuntimeDiagnostic> {
|
||||
let mut diagnostics = vec![diagnostic(
|
||||
fn embedded_worker_projection_diagnostics() -> Vec<RuntimeDiagnostic> {
|
||||
vec![diagnostic(
|
||||
"embedded_runtime_projection",
|
||||
DiagnosticSeverity::Info,
|
||||
"Worker identity is projected only as runtime_id plus worker_id; embedded runtime internals remain backend-private".to_string(),
|
||||
)];
|
||||
|
||||
match execution.backend {
|
||||
WorkerExecutionBackendKind::Stopped => diagnostics.push(diagnostic(
|
||||
"embedded_worker_execution_stopped",
|
||||
DiagnosticSeverity::Info,
|
||||
"Worker execution is stopped; the runtime will restore it before dispatching input or protocol methods".to_string(),
|
||||
)),
|
||||
WorkerExecutionBackendKind::Corrupted => diagnostics.push(diagnostic(
|
||||
"embedded_worker_execution_corrupted",
|
||||
DiagnosticSeverity::Error,
|
||||
"Worker execution state is corrupted and cannot be restored without repair".to_string(),
|
||||
)),
|
||||
WorkerExecutionBackendKind::Alive => {
|
||||
if execution.run_state == WorkerExecutionRunState::Rejected {
|
||||
diagnostics.push(diagnostic(
|
||||
"embedded_worker_execution_rejected",
|
||||
DiagnosticSeverity::Warning,
|
||||
"Worker execution rejected the last transient operation; retry or inspect runtime logs".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(restore_dry_check) = execution.restore_dry_check.as_ref() {
|
||||
diagnostics.push(diagnostic(
|
||||
"embedded_worker_execution_restore_dry_check",
|
||||
DiagnosticSeverity::Error,
|
||||
format!(
|
||||
"Worker restore dry-test {}: {}",
|
||||
restore_dry_check.code, restore_dry_check.message
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
diagnostics
|
||||
}
|
||||
|
||||
fn embedded_worker_execution_status_label(
|
||||
status: EmbeddedWorkerStatus,
|
||||
execution: &WorkerExecutionStatus,
|
||||
) -> &'static str {
|
||||
match status {
|
||||
EmbeddedWorkerStatus::Stopped => "stopped",
|
||||
EmbeddedWorkerStatus::Cancelled => "cancelled",
|
||||
EmbeddedWorkerStatus::Running => match execution.backend {
|
||||
worker_runtime::execution::WorkerExecutionBackendKind::Stopped => "stopped",
|
||||
worker_runtime::execution::WorkerExecutionBackendKind::Corrupted => "corrupted",
|
||||
worker_runtime::execution::WorkerExecutionBackendKind::Alive => {
|
||||
match execution.run_state {
|
||||
WorkerExecutionRunState::Idle => "idle",
|
||||
WorkerExecutionRunState::Busy => "running",
|
||||
WorkerExecutionRunState::Stopped => "stopped",
|
||||
WorkerExecutionRunState::Rejected => "rejected",
|
||||
WorkerExecutionRunState::Errored => "errored",
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
)]
|
||||
}
|
||||
|
||||
fn default_profile_source_archive_source(
|
||||
|
|
@ -4501,7 +4416,7 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn remote_runtime_projection_blocks_stopped_and_corrupted_execution_stop() {
|
||||
fn remote_runtime_projection_uses_canonical_worker_status_for_stop_capability() {
|
||||
let (base_url, server) = serve_mock_http(vec![
|
||||
mock_response(
|
||||
"GET",
|
||||
|
|
@ -4510,30 +4425,10 @@ mod tests {
|
|||
200,
|
||||
json!({
|
||||
"workers": [
|
||||
worker_json_with_execution(
|
||||
"remote:primary",
|
||||
"1",
|
||||
"stopped",
|
||||
"stopped",
|
||||
),
|
||||
worker_json_with_execution(
|
||||
"remote:primary",
|
||||
"2",
|
||||
"corrupted",
|
||||
"errored",
|
||||
),
|
||||
worker_json_with_execution(
|
||||
"remote:primary",
|
||||
"3",
|
||||
"alive",
|
||||
"rejected",
|
||||
),
|
||||
worker_json_with_execution(
|
||||
"remote:primary",
|
||||
"4",
|
||||
"alive",
|
||||
"errored",
|
||||
)
|
||||
worker_json_with_status("remote:primary", "1", "stopped"),
|
||||
worker_json_with_status("remote:primary", "2", "cancelled"),
|
||||
worker_json_with_status("remote:primary", "3", "paused"),
|
||||
worker_json_with_status("remote:primary", "4", "idle")
|
||||
]
|
||||
})
|
||||
.to_string(),
|
||||
|
|
@ -4544,12 +4439,7 @@ mod tests {
|
|||
true,
|
||||
200,
|
||||
json!({
|
||||
"worker": worker_json_with_execution(
|
||||
"remote:primary",
|
||||
"1",
|
||||
"stopped",
|
||||
"stopped",
|
||||
)})
|
||||
"worker": worker_json_with_status("remote:primary", "1", "stopped")})
|
||||
.to_string(),
|
||||
),
|
||||
]);
|
||||
|
|
@ -4569,17 +4459,14 @@ mod tests {
|
|||
|
||||
let workers = registry.list_workers(10);
|
||||
assert_eq!(workers.items.len(), 4);
|
||||
for worker in &workers.items {
|
||||
assert!(
|
||||
!worker.capabilities.can_stop,
|
||||
"{} should not be stoppable",
|
||||
worker.worker_id
|
||||
);
|
||||
}
|
||||
assert!(!workers.items[0].capabilities.can_stop);
|
||||
assert!(!workers.items[1].capabilities.can_stop);
|
||||
assert!(workers.items[2].capabilities.can_stop);
|
||||
assert!(workers.items[3].capabilities.can_stop);
|
||||
assert_eq!(workers.items[0].state, "stopped");
|
||||
assert_eq!(workers.items[1].state, "corrupted");
|
||||
assert_eq!(workers.items[2].state, "rejected");
|
||||
assert_eq!(workers.items[3].state, "errored");
|
||||
assert_eq!(workers.items[1].state, "cancelled");
|
||||
assert_eq!(workers.items[2].state, "paused");
|
||||
assert_eq!(workers.items[3].state, "idle");
|
||||
|
||||
let stopped_detail = registry.worker("remote:primary", "1").unwrap();
|
||||
assert!(!stopped_detail.capabilities.can_stop);
|
||||
|
|
@ -4764,22 +4651,20 @@ mod tests {
|
|||
}
|
||||
|
||||
fn worker_json(runtime_id: &str, worker_id: &str) -> serde_json::Value {
|
||||
worker_json_with_execution(runtime_id, worker_id, "alive", "idle")
|
||||
worker_json_with_status(runtime_id, worker_id, "idle")
|
||||
}
|
||||
|
||||
fn worker_json_with_execution(
|
||||
fn worker_json_with_status(
|
||||
runtime_id: &str,
|
||||
worker_id: &str,
|
||||
backend: &str,
|
||||
run_state: &str,
|
||||
status: &str,
|
||||
) -> serde_json::Value {
|
||||
let worker_id = worker_id.parse::<u64>().unwrap();
|
||||
json!({
|
||||
"worker_ref": { "runtime_id": runtime_id, "worker_id": worker_id },
|
||||
"runtime_id": runtime_id,
|
||||
"worker_id": worker_id,
|
||||
"status": "running",
|
||||
"execution": { "backend": backend, "run_state": run_state },
|
||||
"status": status,
|
||||
"intent": { "kind": "role", "role": "coder", "purpose": "remote test" },
|
||||
"profile": { "kind": "builtin", "value": "coder" },
|
||||
"profile_source": {
|
||||
|
|
|
|||
|
|
@ -7192,15 +7192,6 @@ mod tests {
|
|||
.cleanup_working_directory(working_directory_id)
|
||||
}
|
||||
|
||||
fn dry_restore_worker(
|
||||
&self,
|
||||
_request: worker_runtime::execution::WorkerExecutionRestoreDryRequest,
|
||||
) -> worker_runtime::execution::WorkerRestoreDryCheck {
|
||||
worker_runtime::execution::WorkerRestoreDryCheck::valid(
|
||||
"deterministic test backend dry restore ok",
|
||||
)
|
||||
}
|
||||
|
||||
fn spawn_worker(
|
||||
&self,
|
||||
request: worker_runtime::execution::WorkerExecutionSpawnRequest,
|
||||
|
|
@ -9024,8 +9015,7 @@ mod tests {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn embedded_runtime_fs_store_restores_catalog_config_bundle_and_corrupts_failed_execution()
|
||||
{
|
||||
async fn embedded_runtime_fs_store_restores_catalog_and_stops_failed_execution() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let config = test_server_config(dir.path().join("workspace"));
|
||||
let store_root = config.embedded_runtime_store_root.clone();
|
||||
|
|
@ -9112,14 +9102,8 @@ mod tests {
|
|||
.runtime
|
||||
.worker("embedded-worker-runtime", &worker_id)
|
||||
.expect("restored worker");
|
||||
assert_eq!(restored_worker.state, "corrupted");
|
||||
assert_eq!(restored_worker.state, "stopped");
|
||||
assert!(!restored_worker.capabilities.can_stop);
|
||||
assert!(
|
||||
restored_worker
|
||||
.diagnostics
|
||||
.iter()
|
||||
.any(|diagnostic| diagnostic.code == "embedded_worker_execution_corrupted")
|
||||
);
|
||||
|
||||
let bundles = restored
|
||||
.runtime
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user