Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

refactor: make worker state snapshots authoritative

Browse Source
This commit is contained in:
Hare
2026-09-17 02:02:51 +09:00
parent d6bef5d1c7
commit 10ebac142e
31 changed files with 473 additions and 1132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/worker-runtime/src/execution.rs
-4
View File
@@ -243,8 +243,6 @@ impl fmt::Debug for WorkerExecutionContext {
#[derive(Clone, Debug)]
pub struct WorkerExecutionSpawnRequest {
pub worker_ref: WorkerRef,
/// Monotonic execution generation reserved durably before launch.
pub run_generation: u64,
pub request: crate::catalog::CreateWorkerRequest,
pub workspace_scope: Option<crate::runtime::RuntimeWorkspaceScope>,
pub context: WorkerExecutionContext,
@@ -256,8 +254,6 @@ pub struct WorkerExecutionSpawnRequest {
#[derive(Clone, Debug)]
pub struct WorkerExecutionRestoreRequest {
pub worker_ref: WorkerRef,
/// Monotonic execution generation reserved durably before restore.
pub run_generation: u64,
pub request: crate::catalog::CreateWorkerRequest,
pub workspace_scope: Option<crate::runtime::RuntimeWorkspaceScope>,
pub context: WorkerExecutionContext,
crates/worker-runtime/src/fs_store.rs
+112 -337
View File
@@ -17,10 +17,8 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
const SCHEMA_VERSION: u32 = 6;
const PREVIOUS_SCHEMA_VERSION: u32 = 5;
const EXECUTION_SCHEMA_VERSION: u32 = 4;
const PRE_EXECUTION_SCHEMA_VERSION: u32 = 3;
const SCHEMA_VERSION: u32 = 7;
const PREVIOUS_SCHEMA_VERSION: u32 = 6;
const RUNTIME_FILE: &str = "runtime.json";
const WORKERS_DIR: &str = "workers";
const WORKER_FILE: &str = "worker.json";
@@ -371,13 +369,12 @@ pub(crate) struct PersistedRuntimeState {
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct PersistedWorkerExecutionBinding {
pub(crate) run_generation: u64,
}
#[serde(deny_unknown_fields)]
pub(crate) struct PersistedWorkerExecutionBinding {}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct PersistedWorkerExecution {
pub(crate) last_run_generation: u64,
pub(crate) binding: Option<PersistedWorkerExecutionBinding>,
pub(crate) restore_intent: WorkerRestoreIntent,
}
@@ -465,8 +462,8 @@ fn plan_runtime_store_migration(
format!("Runtime store schema version {schema_version} is out of range"),
)
})?;
let staging = migration_sibling(root, "schema-v6-staging")?;
let backup = migration_sibling(root, "pre-schema-v6-backup")?;
let staging = migration_sibling(root, "schema-v7-staging")?;
let backup = migration_sibling(root, "pre-schema-v7-backup")?;
if staging.exists() || backup.exists() {
return Err(runtime_store_corrupt(
root,
@@ -492,14 +489,11 @@ fn plan_runtime_store_migration(
};
return Ok((plan, Vec::new()));
}
if !matches!(
current_schema_version,
PRE_EXECUTION_SCHEMA_VERSION | EXECUTION_SCHEMA_VERSION | PREVIOUS_SCHEMA_VERSION
) {
if current_schema_version != PREVIOUS_SCHEMA_VERSION {
return Err(runtime_store_corrupt(
&runtime_path,
format!(
"unsupported Runtime store schema version {schema_version}; expected {PRE_EXECUTION_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
"unsupported Runtime store schema version {schema_version}; expected {PREVIOUS_SCHEMA_VERSION} or {SCHEMA_VERSION}"
),
));
}
@@ -661,118 +655,19 @@ struct DiagnosticWorkerRefMigrationCounts {
cleared: usize,
}
fn migrate_v1_worker_document(
mut snapshot: serde_json::Value,
mapping: &LegacyWorkerIdentityMapping,
snapshot_path: &Path,
) -> Result<serde_json::Value, RuntimeError> {
let worker_id_text = mapping.worker_id.to_string();
let snapshot_object = snapshot.as_object_mut().ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker snapshot must be an object".to_string(),
)
})?;
snapshot_object.insert(
"schema_version".to_string(),
serde_json::Value::from(SCHEMA_VERSION),
);
snapshot_object.insert(
"worker_id".to_string(),
serde_json::Value::String(worker_id_text.clone()),
);
snapshot_object
.get_mut("worker_ref")
.and_then(serde_json::Value::as_object_mut)
.ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker snapshot worker_ref must be an object".to_string(),
)
})?
.insert(
"worker_id".to_string(),
serde_json::Value::String(worker_id_text.clone()),
);
let request = snapshot_object
.get_mut("request")
.and_then(serde_json::Value::as_object_mut)
.ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker snapshot request must be an object".to_string(),
)
})?;
let fingerprint = request
.remove("idempotency_fingerprint")
.and_then(|value| value.as_str().map(ToOwned::to_owned))
.unwrap_or_else(|| {
format!(
"legacy:{}:{}:{}",
mapping.workspace_id, mapping.runtime_id, mapping.legacy_worker_id
)
});
request.remove("idempotency_key");
request.insert(
"worker_id".to_string(),
serde_json::Value::String(worker_id_text),
);
request.insert(
"create_fingerprint".to_string(),
serde_json::Value::String(fingerprint),
);
Ok(snapshot)
}
fn max_persisted_run_generation(snapshot_path: &Path) -> Result<u64, RuntimeError> {
let worker_dir = snapshot_path.parent().ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker snapshot path is missing its aggregate directory".to_string(),
)
})?;
let runs_dir = worker_dir.join("runs");
if !runs_dir
.try_exists()
.map_err(|source| runtime_io_error("inspect Worker runs", &runs_dir, source))?
{
return Ok(0);
}
let entries = fs::read_dir(&runs_dir)
.map_err(|source| runtime_io_error("read Worker runs", &runs_dir, source))?;
let mut max_generation = 0;
for entry in entries {
let entry =
entry.map_err(|source| runtime_io_error("read Worker runs", &runs_dir, source))?;
let Some(generation) = entry
.file_name()
.to_str()
.and_then(|name| name.parse::<u64>().ok())
else {
continue;
};
max_generation = max_generation.max(generation);
}
Ok(max_generation)
}
fn migrate_worker_document(
mut document: serde_json::Value,
source_schema_version: u32,
mapping: Option<&LegacyWorkerIdentityMapping>,
_mapping: Option<&LegacyWorkerIdentityMapping>,
snapshot_path: &Path,
) -> Result<serde_json::Value, RuntimeError> {
if source_schema_version == 1 {
document = migrate_v1_worker_document(
document,
mapping.ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"schema-v1 Worker migration is missing its identity mapping".to_string(),
)
})?,
if source_schema_version != PREVIOUS_SCHEMA_VERSION {
return Err(runtime_store_corrupt(
snapshot_path,
)?;
format!(
"unsupported Worker snapshot schema {source_schema_version}; expected {PREVIOUS_SCHEMA_VERSION}"
),
));
}
let object = document.as_object_mut().ok_or_else(|| {
runtime_store_corrupt(
@@ -780,117 +675,80 @@ fn migrate_worker_document(
"Worker snapshot must be an object".to_string(),
)
})?;
let declared_run_generation = object
.remove("run_generation")
.map(|value| {
value.as_u64().ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker snapshot run_generation must be an unsigned integer".to_string(),
)
})
})
.transpose()?;
let legacy_execution = object.remove("execution");
let execution = legacy_execution
.as_ref()
.and_then(serde_json::Value::as_object);
let persisted_last_run_generation = execution
.and_then(|execution| execution.get("last_run_generation"))
.map(|value| {
value.as_u64().ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker execution last_run_generation must be an unsigned integer".to_string(),
)
})
})
.transpose()?;
let binding_run_generation = execution
.and_then(|execution| execution.get("binding"))
.and_then(serde_json::Value::as_object)
.and_then(|binding| binding.get("run_generation"))
.map(|value| {
value.as_u64().ok_or_else(|| {
let execution = object
.get_mut("execution")
.and_then(serde_json::Value::as_object_mut)
.ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker snapshot execution must be an object".to_string(),
)
})?;
let last_run_generation = execution
.remove("last_run_generation")
.and_then(|value| value.as_u64())
.ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker execution last_run_generation must be an unsigned integer".to_string(),
)
})?;
let binding = execution.get_mut("binding").ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker execution is missing binding".to_string(),
)
})?;
if let Some(binding_object) = binding.as_object_mut() {
let binding_run_generation = binding_object
.remove("run_generation")
.and_then(|value| value.as_u64())
.ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"Worker execution binding run_generation must be an unsigned integer"
.to_string(),
)
})
})
.transpose()?;
let run_generation = declared_run_generation
.into_iter()
.chain(persisted_last_run_generation)
.chain(binding_run_generation)
.chain(std::iter::once(max_persisted_run_generation(
snapshot_path,
)?))
.max()
.unwrap_or(0);
if !object.contains_key("working_directory") {
if let Some(working_directory) = legacy_execution
.as_ref()
.and_then(serde_json::Value::as_object)
.and_then(|execution| execution.get("working_directory"))
.cloned()
{
object.insert("working_directory".to_string(), working_directory);
}
}
let legacy_materialization = object
.get("working_directory")
.and_then(|working_directory| working_directory.get("summary"))
.and_then(|summary| summary.get("materializer_kind"))
.and_then(serde_json::Value::as_str)
.is_some_and(|kind| matches!(kind, "runtime_git_cache" | "local_git_worktree"));
if legacy_materialization {
object.insert("working_directory".to_string(), serde_json::Value::Null);
}
if let Some(profile_source) = object
.get_mut("request")
.and_then(serde_json::Value::as_object_mut)
.and_then(|request| request.get_mut("profile_source"))
.and_then(serde_json::Value::as_object_mut)
&& profile_source
.get("kind")
.and_then(serde_json::Value::as_str)
== Some("http")
{
let archive = profile_source
.get_mut("location")
.and_then(serde_json::Value::as_object_mut)
.and_then(|location| location.remove("archive"))
.ok_or_else(|| {
runtime_store_corrupt(
snapshot_path,
"legacy HTTP profile source is missing its archive".to_string(),
)
})?;
profile_source.clear();
profile_source.insert(
"kind".to_string(),
serde_json::Value::String("workspace_config".to_string()),
);
profile_source.insert("archive".to_string(), archive);
if binding_run_generation != last_run_generation {
return Err(runtime_store_corrupt(
snapshot_path,
format!(
"execution binding run_generation {binding_run_generation} does not match last_run_generation {last_run_generation}"
),
));
}
if !binding_object.is_empty() {
return Err(runtime_store_corrupt(
snapshot_path,
"Worker execution binding contains unsupported fields".to_string(),
));
}
} else if !binding.is_null() {
return Err(runtime_store_corrupt(
snapshot_path,
"Worker execution binding must be an object or null".to_string(),
));
}
if !execution.contains_key("restore_intent") {
return Err(runtime_store_corrupt(
snapshot_path,
"Worker execution is missing restore_intent".to_string(),
));
}
if execution
.keys()
.any(|key| key != "binding" && key != "restore_intent")
{
return Err(runtime_store_corrupt(
snapshot_path,
"Worker execution contains unsupported fields".to_string(),
));
}
object.insert(
"schema_version".to_string(),
serde_json::Value::from(SCHEMA_VERSION),
);
object.insert(
"status".to_string(),
serde_json::Value::String("stopped".to_string()),
);
object.insert(
"execution".to_string(),
serde_json::json!({
"last_run_generation": run_generation,
"binding": null,
"restore_intent": "explicit",
}),
);
Ok(document)
}
@@ -1259,8 +1117,8 @@ fn migrate_runtime_store(
if !plan.migration_required {
return Ok(plan);
}
let staging = migration_sibling(root, "schema-v6-staging")?;
let backup = migration_sibling(root, "pre-schema-v6-backup")?;
let staging = migration_sibling(root, "schema-v7-staging")?;
let backup = migration_sibling(root, "pre-schema-v7-backup")?;
if staging.exists() || backup.exists() {
return Err(runtime_store_corrupt(
root,
@@ -1534,52 +1392,18 @@ impl WorkerSnapshot {
),
});
}
if let Some(binding) = self.execution.binding.as_ref()
&& binding.run_generation != self.execution.last_run_generation
{
return Err(RuntimeError::StoreCorrupt {
operation: "read worker snapshot",
path: path.to_path_buf(),
message: format!(
"execution binding run_generation {} does not match last_run_generation {}",
binding.run_generation, self.execution.last_run_generation
),
});
}
match (self.status, self.execution.restore_intent) {
(status, WorkerRestoreIntent::Automatic) if status.is_active() => {
let Some(binding) = self.execution.binding.as_ref() else {
if self.execution.binding.is_none() {
return Err(RuntimeError::StoreCorrupt {
operation: "read worker snapshot",
path: path.to_path_buf(),
message: "automatic restore intent requires an execution binding"
.to_string(),
});
};
if binding.run_generation == 0 {
return Err(RuntimeError::StoreCorrupt {
operation: "read worker snapshot",
path: path.to_path_buf(),
message: "execution binding run_generation must be greater than zero"
.to_string(),
});
}
}
(WorkerStatus::Stopped, WorkerRestoreIntent::Explicit) => {
if self
.execution
.binding
.as_ref()
.is_some_and(|binding| binding.run_generation == 0)
{
return Err(RuntimeError::StoreCorrupt {
operation: "read worker snapshot",
path: path.to_path_buf(),
message: "execution binding run_generation must be greater than zero"
.to_string(),
});
}
}
(WorkerStatus::Stopped, WorkerRestoreIntent::Explicit) => {}
_ => {
return Err(RuntimeError::StoreCorrupt {
operation: "read worker snapshot",
@@ -1837,96 +1661,47 @@ mod tests {
}
#[test]
fn schema_v5_worker_migration_recovers_last_generation_from_run_aggregates() {
let root = tempfile::tempdir().unwrap();
let worker_dir = root.path().join("worker-a");
fs::create_dir_all(worker_dir.join("runs/1")).unwrap();
fs::create_dir_all(worker_dir.join("runs/7")).unwrap();
fs::create_dir_all(worker_dir.join("runs/incomplete")).unwrap();
let path = worker_dir.join(WORKER_FILE);
let source = serde_json::json!({
"schema_version": 5,
"execution": {
"binding": null,
"restore_intent": "explicit"
}
});
let migrated =
migrate_worker_document(source, PREVIOUS_SCHEMA_VERSION, None, &path).unwrap();
assert_eq!(
migrated["execution"]["last_run_generation"],
serde_json::json!(7)
);
assert_eq!(migrated["execution"]["binding"], serde_json::Value::Null);
}
#[test]
fn schema_v4_worker_migration_discards_unsupported_linked_worktree_binding() {
let source = serde_json::json!({
"schema_version": 4,
"request": {
"profile_source": {
"kind": "http",
"location": {
"url": "https://workspace.example.test/archive",
"etag": "profile-source:test",
"archive": {
"id": "profiles-v1",
"digest": "sha256:test",
"size_bytes": 1,
"source_graph": {
"source_count": 1,
"total_source_bytes": 1,
"entrypoints": {},
"import_count": 0
}
}
}
}
},
"working_directory": {
"summary": {
"materializer_kind": "runtime_git_cache"
}
}
});
fn schema_v6_worker_migration_removes_generation_and_preserves_active_restore() {
let path = Path::new("worker.json");
let source = serde_json::json!({
"schema_version": PREVIOUS_SCHEMA_VERSION,
"status": "running",
"execution": {
"last_run_generation": 7,
"binding": { "run_generation": 7 },
"restore_intent": "automatic"
}
});
let migrated =
migrate_worker_document(source, EXECUTION_SCHEMA_VERSION, None, path).unwrap();
migrate_worker_document(source, PREVIOUS_SCHEMA_VERSION, None, path).unwrap();
assert_eq!(migrated["schema_version"], SCHEMA_VERSION);
assert_eq!(migrated["status"], "stopped");
assert_eq!(migrated["working_directory"], serde_json::Value::Null);
assert_eq!(
migrated["request"]["profile_source"]["kind"],
"workspace_config"
);
assert_eq!(
migrated["request"]["profile_source"]["archive"]["id"],
"profiles-v1"
);
assert_eq!(migrated["execution"]["restore_intent"], "explicit");
assert_eq!(migrated["status"], "running");
assert_eq!(migrated["execution"]["binding"], serde_json::json!({}));
assert_eq!(migrated["execution"]["restore_intent"], "automatic");
assert!(migrated["execution"].get("last_run_generation").is_none());
}
#[test]
fn schema_v4_worker_migration_preserves_runtime_clone_observation() {
fn schema_v6_worker_migration_rejects_mismatched_generation_state() {
let path = Path::new("worker.json");
let source = serde_json::json!({
"schema_version": 4,
"working_directory": {
"summary": {
"materializer_kind": "runtime_git_clone"
}
"schema_version": PREVIOUS_SCHEMA_VERSION,
"execution": {
"last_run_generation": 7,
"binding": { "run_generation": 6 },
"restore_intent": "automatic"
}
});
let expected = source["working_directory"].clone();
let path = Path::new("worker.json");
let migrated =
migrate_worker_document(source, EXECUTION_SCHEMA_VERSION, None, path).unwrap();
let error =
migrate_worker_document(source, PREVIOUS_SCHEMA_VERSION, None, path).unwrap_err();
assert_eq!(migrated["working_directory"], expected);
assert!(
error
.to_string()
.contains("does not match last_run_generation")
);
}
}
crates/worker-runtime/src/http_server.rs
+1 -5
View File
@@ -3126,7 +3126,6 @@ mod tests {
WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
worker_state: protocol::WorkerStateSnapshot {
execution_generation: request.run_generation,
..protocol::WorkerStatus::Idle.into()
},
working_directory: request
@@ -3143,7 +3142,6 @@ mod tests {
WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
worker_state: protocol::WorkerStateSnapshot {
execution_generation: request.run_generation,
..protocol::WorkerStatus::Idle.into()
},
working_directory: request.previous_working_directory,
@@ -3561,7 +3559,6 @@ mod ws_tests {
WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
worker_state: protocol::WorkerStateSnapshot {
execution_generation: request.run_generation,
..protocol::WorkerStatus::Idle.into()
},
working_directory: request
@@ -3604,7 +3601,7 @@ mod ws_tests {
context_window: 0,
context_tokens: 0,
},
state: protocol::WorkerStateSnapshot::initial(1),
state: protocol::WorkerStateSnapshot::initial(),
in_flight: protocol::InFlightSnapshot {
blocks: Vec::new(),
commands: Vec::new(),
@@ -3833,7 +3830,6 @@ mod ws_tests {
.unwrap()
.worker_state
.expect("connected test Worker must expose its initial state");
snapshot.revision += 1;
snapshot.state = protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
));
crates/worker-runtime/src/main.rs
+4 -4
View File
@@ -1273,14 +1273,14 @@ mod tests {
}
#[test]
fn migration_dry_run_accepts_supported_schema_v3_without_workers_field() {
fn migration_dry_run_accepts_previous_schema_without_workers_field() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
std::fs::create_dir_all(root.join("workers")).unwrap();
std::fs::write(
root.join("runtime.json"),
serde_json::to_vec_pretty(&serde_json::json!({
"schema_version": 3,
"schema_version": 6,
"display_name": "local",
"backend": "fs_store",
"status": "running",
@@ -1312,14 +1312,14 @@ mod tests {
}
#[test]
fn migration_dry_run_rejects_schema_v3_document_that_cannot_decode_as_v6() {
fn migration_dry_run_rejects_previous_schema_that_cannot_decode_as_current() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
std::fs::create_dir_all(root.join("workers")).unwrap();
std::fs::write(
root.join("runtime.json"),
serde_json::to_vec_pretty(&serde_json::json!({
"schema_version": 3,
"schema_version": 6,
"display_name": "local",
"backend": "fs_store",
"status": 3,
crates/worker-runtime/src/retention.rs
+31 -93
View File
@@ -39,7 +39,6 @@ pub struct WorkerRetentionInventory {
pub workspace_id: String,
pub runtime_id: String,
pub worker_id: WorkerId,
pub run_generation: u64,
pub session_id: Option<String>,
pub segment_ids: Vec<String>,
pub session_bytes: u64,
@@ -118,7 +117,6 @@ pub struct WorkerRetentionExecutionRequest {
pub source_runtime_id: String,
pub worker_id: WorkerId,
pub expected_worker_revision: String,
pub expected_run_generation: u64,
pub source_created_at: String,
pub removed_at: String,
pub effective_profile: Option<String>,
@@ -171,7 +169,6 @@ pub(crate) trait WorkerRetentionProvider: Send + Sync {
workspace_id: &str,
runtime_id: &str,
worker_id: WorkerId,
run_generation: u64,
) -> Result<WorkerRetentionInventory, RuntimeError>;
fn execute(
@@ -283,7 +280,7 @@ impl FsWorkerRetentionProvider {
continue;
};
let worker_dir = self.worker_dir(worker_id);
let snapshot: WorkerGenerationSnapshot = match read_json(
let snapshot: WorkerAggregateSnapshot = match read_json(
&worker_dir.join("worker.json"),
"scan Worker retention inventory",
) {
@@ -303,12 +300,7 @@ impl FsWorkerRetentionProvider {
));
continue;
}
match self.inventory(
workspace_id,
runtime_id,
worker_id,
snapshot.run_generation(),
) {
match self.inventory(workspace_id, runtime_id, worker_id) {
Ok(item) => workers.push(item),
Err(_) => diagnostics.push(runtime_aggregate_diagnostic(
&bounded_id,
@@ -380,26 +372,18 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider {
workspace_id: &str,
runtime_id: &str,
worker_id: WorkerId,
run_generation: u64,
) -> Result<WorkerRetentionInventory, RuntimeError> {
let worker_dir = self.worker_dir(worker_id);
if !worker_dir.is_dir() {
return Err(RuntimeError::WorkerNotFound { worker_id });
}
let worker: WorkerGenerationSnapshot = read_json(
let worker: WorkerAggregateSnapshot = read_json(
&worker_dir.join("worker.json"),
"inventory Worker retention",
)?;
if worker.workspace_id.as_deref() != Some(workspace_id) {
return Err(RuntimeError::WorkerNotFound { worker_id });
}
let current_run_generation = worker.run_generation();
if current_run_generation != run_generation {
return Err(RuntimeError::InvalidRequest(format!(
"Worker retention inventory expected generation {run_generation}, current generation is {}",
current_run_generation
)));
}
let session_dir = worker_dir.join("session");
let (session_id, segment_ids, session_bytes) = if session_dir.is_dir() {
let manifest: CanonicalSessionManifest = read_json(
@@ -437,7 +421,6 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider {
workspace_id: workspace_id.to_string(),
runtime_id: runtime_id.to_string(),
worker_id,
run_generation,
session_id,
segment_ids,
session_bytes,
@@ -497,21 +480,13 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider {
worker_id: request.worker_id,
});
}
let snapshot: WorkerGenerationSnapshot =
let snapshot: WorkerAggregateSnapshot =
read_json(&worker_dir.join("worker.json"), "execute Worker retention")?;
if snapshot.workspace_id.as_deref() != Some(request.workspace_id.as_str()) {
return Err(RuntimeError::WorkerNotFound {
worker_id: request.worker_id,
});
}
let run_generation = snapshot.run_generation();
if run_generation != request.expected_run_generation {
return Err(RuntimeError::InvalidRequest(format!(
"Worker retention plan expected generation {}, current generation is {}",
request.expected_run_generation, run_generation
)));
}
let archive = match request.session_disposition {
SessionDisposition::Archive => {
Some(commit_session_archive(self, request, &worker_dir)?)
@@ -572,30 +547,9 @@ impl WorkerRetentionProvider for FsWorkerRetentionProvider {
}
#[derive(Deserialize)]
struct WorkerGenerationSnapshot {
struct WorkerAggregateSnapshot {
#[serde(default)]
workspace_id: Option<String>,
execution: WorkerGenerationExecution,
}
#[derive(Deserialize)]
struct WorkerGenerationExecution {
binding: Option<WorkerGenerationBinding>,
}
#[derive(Deserialize)]
struct WorkerGenerationBinding {
run_generation: u64,
}
impl WorkerGenerationSnapshot {
fn run_generation(&self) -> u64 {
self.execution
.binding
.as_ref()
.map(|binding| binding.run_generation)
.unwrap_or(0)
}
}
#[derive(Deserialize)]
@@ -1286,13 +1240,12 @@ mod tests {
fs::write(path, serde_json::to_vec_pretty(value).unwrap()).unwrap();
}
fn source(root: &Path, worker_id: WorkerId, generation: u64) {
fn source(root: &Path, worker_id: WorkerId) {
let worker = root.join("workers").join(worker_id.to_string());
write_json(
&worker.join("worker.json"),
&serde_json::json!({
"workspace_id": "workspace-a",
"execution": {"binding": {"run_generation": generation}}
"workspace_id": "workspace-a"
}),
);
write_json(
@@ -1301,22 +1254,17 @@ mod tests {
);
fs::create_dir_all(worker.join("session/segments")).unwrap();
fs::write(worker.join("session/segments/segment-a.jsonl"), b"one\n").unwrap();
fs::create_dir_all(worker.join(format!("runs/{generation}"))).unwrap();
fs::create_dir_all(worker.join("runs/attempt-a")).unwrap();
fs::write(
worker.join(format!("runs/{generation}/worker.out.log")),
worker.join("runs/attempt-a/worker.out.log"),
b"diagnostic\n",
)
.unwrap();
fs::write(
worker.join(format!("runs/{generation}/worker.sock")),
b"not retained",
)
.unwrap();
fs::write(worker.join("runs/attempt-a/worker.sock"), b"not retained").unwrap();
}
fn request(
worker_id: WorkerId,
generation: u64,
disposition: SessionDisposition,
) -> WorkerRetentionExecutionRequest {
WorkerRetentionExecutionRequest {
@@ -1328,7 +1276,6 @@ mod tests {
workspace_id: "workspace-a".to_string(),
source_runtime_id: "runtime-a".to_string(),
worker_id,
expected_run_generation: generation,
source_created_at: "2026-01-01T00:00:00Z".to_string(),
removed_at: "2026-01-02T00:00:00Z".to_string(),
effective_profile: Some("builtin:coder".to_string()),
@@ -1344,9 +1291,9 @@ mod tests {
fn archive_is_verified_before_source_removal_and_retry_converges() {
let temp = tempfile::tempdir().unwrap();
let worker_id = WorkerId::from_legacy_u64(7);
source(temp.path(), worker_id, 4);
source(temp.path(), worker_id);
let provider = FsWorkerRetentionProvider::new(temp.path());
let request = request(worker_id, 4, SessionDisposition::Archive);
let request = request(worker_id, SessionDisposition::Archive);
let first = provider.execute(&request).unwrap();
assert!(first.source_removed);
@@ -1375,7 +1322,7 @@ mod tests {
fn archive_failure_keeps_live_source_for_retry() {
let temp = tempfile::tempdir().unwrap();
let worker_id = WorkerId::from_legacy_u64(8);
source(temp.path(), worker_id, 2);
source(temp.path(), worker_id);
let collision = temp.path().join("archives/workers/archive-a");
fs::create_dir_all(&collision).unwrap();
fs::write(collision.join("manifest.json"), b"not-json").unwrap();
@@ -1383,7 +1330,7 @@ mod tests {
assert!(
provider
.execute(&request(worker_id, 2, SessionDisposition::Archive))
.execute(&request(worker_id, SessionDisposition::Archive))
.is_err()
);
assert!(
@@ -1403,13 +1350,13 @@ mod tests {
fn target_inventory_and_execute_reject_cross_workspace_aggregate() {
let temp = tempfile::tempdir().unwrap();
let worker_id = WorkerId::from_legacy_u64(16);
source(temp.path(), worker_id, 3);
source(temp.path(), worker_id);
let provider = FsWorkerRetentionProvider::new(temp.path());
assert!(matches!(
provider.inventory("other-workspace", "runtime-a", worker_id, 3),
provider.inventory("other-workspace", "runtime-a", worker_id),
Err(RuntimeError::WorkerNotFound { .. })
));
let mut request = request(worker_id, 3, SessionDisposition::Purge);
let mut request = request(worker_id, SessionDisposition::Purge);
request.workspace_id = "other-workspace".to_string();
assert!(matches!(
provider.execute(&request),
@@ -1434,20 +1381,12 @@ mod tests {
}
#[test]
fn purge_removes_aggregate_and_rejects_stale_generation() {
fn purge_removes_worker_aggregate() {
let temp = tempfile::tempdir().unwrap();
let provider = FsWorkerRetentionProvider::new(temp.path());
let worker_id = WorkerId::from_legacy_u64(9);
source(temp.path(), worker_id, 5);
let stale = request(worker_id, 4, SessionDisposition::Purge);
assert!(provider.execute(&stale).is_err());
assert!(
temp.path()
.join(format!("workers/{worker_id}/session"))
.is_dir()
);
let mut current = request(worker_id, 5, SessionDisposition::Purge);
source(temp.path(), worker_id);
let mut current = request(worker_id, SessionDisposition::Purge);
current.operation_id = "operation-current".to_string();
current.input_fingerprint = "fingerprint-current".to_string();
let result = provider.execute(&current).unwrap();
@@ -1464,9 +1403,9 @@ mod tests {
fn pending_receipt_recovers_delete_to_receipt_crash_window() {
let temp = tempfile::tempdir().unwrap();
let worker_id = WorkerId::from_legacy_u64(11);
source(temp.path(), worker_id, 1);
source(temp.path(), worker_id);
let provider = FsWorkerRetentionProvider::new(temp.path());
let request = request(worker_id, 1, SessionDisposition::Archive);
let request = request(worker_id, SessionDisposition::Archive);
let completed = provider.execute(&request).unwrap();
let receipt_path = temp.path().join("retention/operations/operation-a.json");
let mut receipt: RetentionOperationReceipt =
@@ -1482,9 +1421,9 @@ mod tests {
#[test]
fn provider_snapshot_scans_aggregate_storage_independent_of_runtime_catalog() {
let temp = tempfile::tempdir().unwrap();
source(temp.path(), WorkerId::from_legacy_u64(13), 2);
source(temp.path(), WorkerId::from_legacy_u64(13));
let other_worker = WorkerId::from_legacy_u64(14);
source(temp.path(), other_worker, 1);
source(temp.path(), other_worker);
write_json(
&temp
.path()
@@ -1492,8 +1431,7 @@ mod tests {
.join(other_worker.to_string())
.join("worker.json"),
&serde_json::json!({
"workspace_id": "other-workspace",
"execution": {"binding": {"run_generation": 1}}
"workspace_id": "other-workspace"
}),
);
fs::create_dir_all(temp.path().join("workers/not-a-worker")).unwrap();
@@ -1532,9 +1470,9 @@ mod tests {
fn diagnostics_retry_rejects_corrupt_existing_archive_before_source_delete() {
let temp = tempfile::tempdir().unwrap();
let worker_id = WorkerId::from_legacy_u64(12);
source(temp.path(), worker_id, 1);
source(temp.path(), worker_id);
let provider = FsWorkerRetentionProvider::new(temp.path());
let mut request = request(worker_id, 1, SessionDisposition::Archive);
let mut request = request(worker_id, SessionDisposition::Archive);
request.diagnostics_disposition = DiagnosticsDisposition::Retain;
provider.execute(&request).unwrap();
@@ -1543,10 +1481,10 @@ mod tests {
serde_json::from_slice(&fs::read(&receipt_path).unwrap()).unwrap();
receipt.result.source_removed = false;
fs::write(&receipt_path, serde_json::to_vec_pretty(&receipt).unwrap()).unwrap();
source(temp.path(), worker_id, 1);
source(temp.path(), worker_id);
fs::write(
temp.path()
.join("archives/diagnostics/operation-a/runs/1/worker.out.log"),
.join("archives/diagnostics/operation-a/runs/attempt-a/worker.out.log"),
b"corrupt\n",
)
.unwrap();
@@ -1564,9 +1502,9 @@ mod tests {
fn concurrent_retry_produces_one_archive() {
let temp = tempfile::tempdir().unwrap();
let worker_id = WorkerId::from_legacy_u64(10);
source(temp.path(), worker_id, 1);
source(temp.path(), worker_id);
let provider = Arc::new(FsWorkerRetentionProvider::new(temp.path()));
let request = Arc::new(request(worker_id, 1, SessionDisposition::Archive));
let request = Arc::new(request(worker_id, SessionDisposition::Archive));
let barrier = Arc::new(Barrier::new(3));
let handles = (0..2)
.map(|_| {
crates/worker-runtime/src/runtime.rs
+56 -151
View File
@@ -987,7 +987,6 @@ impl Runtime {
worker_state: None,
workspace_id: scope.map(|scope| scope.workspace_id.clone()),
request: durable_request,
run_generation: 1,
execution_bound: true,
restore_intent: WorkerRestoreIntent::Explicit,
working_directory: None,
@@ -999,7 +998,6 @@ impl Runtime {
state.persist_worker(&worker_ref.worker_id)?;
let spawn_request = WorkerExecutionSpawnRequest {
worker_ref: worker_ref.clone(),
run_generation: 1,
request,
workspace_scope: scope.cloned(),
context: self.execution_context(worker_ref.clone()),
@@ -1361,7 +1359,7 @@ impl Runtime {
let (backend, request) = {
let mut state = self.lock()?;
state.ensure_running()?;
let (worker_request, previous_working_directory, run_generation) = {
let (worker_request, previous_working_directory) = {
let worker = state.worker(worker_ref)?;
if worker.execution_handle.is_some() {
if worker.status.is_active() {
@@ -1387,11 +1385,7 @@ impl Runtime {
}
_ => {}
}
(
worker.request.clone(),
worker.working_directory.clone(),
worker.run_generation.saturating_add(1).max(1),
)
(worker.request.clone(), worker.working_directory.clone())
};
let backend = state.execution_backend.clone().ok_or_else(|| {
RuntimeError::WorkerExecutionUnavailable {
@@ -1401,7 +1395,6 @@ impl Runtime {
})?;
{
let worker = state.worker_mut(worker_ref)?;
worker.run_generation = run_generation;
worker.execution_bound = true;
}
state.persist_worker(&worker_ref.worker_id)?;
@@ -1413,7 +1406,6 @@ impl Runtime {
});
let request = WorkerExecutionRestoreRequest {
worker_ref: worker_ref.clone(),
run_generation,
request: worker_request,
workspace_scope,
context: self.execution_context(worker_ref.clone()),
@@ -1910,12 +1902,7 @@ impl Runtime {
};
let mut state = self.lock()?;
let worker = state.worker_mut(worker_ref)?;
let applied = worker
.apply_worker_state(&snapshot)
.is_ok_and(|result| matches!(result, protocol::WorkerStateSnapshotApply::Applied));
if !applied {
return Ok(());
}
worker.apply_worker_state(&snapshot);
state.publish_worker_upsert(worker_ref.worker_id)?;
state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?;
@@ -2517,7 +2504,6 @@ impl Runtime {
workspace_id,
runtime_id,
worker.worker_id,
worker.run_generation,
)
}
@@ -2589,12 +2575,6 @@ impl Runtime {
"Worker retention requires a stopped Worker".to_string(),
));
}
if worker.run_generation != request.expected_run_generation {
return Err(RuntimeError::InvalidRequest(format!(
"Worker retention plan expected generation {}, current generation is {}",
request.expected_run_generation, worker.run_generation
)));
}
let result = provider.execute(request)?;
state.workers.remove(&request.worker_id);
state.persist_runtime_snapshot()?;
@@ -2758,7 +2738,6 @@ impl RuntimeState {
let diagnostics = persisted.diagnostics;
let next_diagnostic_id = persisted.next_diagnostic_id;
for (worker_id, worker) in persisted.workers {
let run_generation = worker.execution.last_run_generation;
workers.insert(
worker_id,
WorkerRecord {
@@ -2768,7 +2747,6 @@ impl RuntimeState {
worker_state: None,
workspace_id: worker.workspace_id,
request: worker.request,
run_generation,
execution_bound: worker.execution.binding.is_some(),
restore_intent: worker.execution.restore_intent,
working_directory: worker.working_directory,
@@ -3563,14 +3541,8 @@ impl RuntimeState {
} => snapshot,
_ => return false,
};
match worker.apply_worker_state(incoming) {
Ok(protocol::WorkerStateSnapshotApply::Applied) => true,
Ok(
protocol::WorkerStateSnapshotApply::Duplicate
| protocol::WorkerStateSnapshotApply::Stale,
)
| Err(_) => false,
}
worker.apply_worker_state(incoming);
true
}
}
@@ -3614,7 +3586,6 @@ struct WorkerRecord {
worker_state: Option<protocol::WorkerStateSnapshot>,
workspace_id: Option<String>,
request: CreateWorkerRequest,
run_generation: u64,
execution_bound: bool,
restore_intent: WorkerRestoreIntent,
working_directory: Option<CatalogWorkingDirectoryStatus>,
@@ -3623,17 +3594,8 @@ struct WorkerRecord {
}
impl WorkerRecord {
fn apply_worker_state(
&mut self,
incoming: &protocol::WorkerStateSnapshot,
) -> Result<protocol::WorkerStateSnapshotApply, protocol::WorkerStateSnapshotConflict> {
match self.worker_state.as_mut() {
Some(current) => protocol::apply_worker_state_snapshot(current, incoming),
None => {
self.worker_state = Some(incoming.clone());
Ok(protocol::WorkerStateSnapshotApply::Applied)
}
}
fn apply_worker_state(&mut self, incoming: &protocol::WorkerStateSnapshot) {
self.worker_state = Some(incoming.clone());
}
fn belongs_to_workspace(&self, workspace_id: &str) -> bool {
@@ -3678,12 +3640,9 @@ impl WorkerRecord {
request: self.request.clone(),
status: self.status,
execution: PersistedWorkerExecution {
last_run_generation: self.run_generation,
binding: self
.execution_bound
.then_some(PersistedWorkerExecutionBinding {
run_generation: self.run_generation,
}),
.then_some(PersistedWorkerExecutionBinding {}),
restore_intent: self.restore_intent,
},
workspace_id: self.workspace_id.clone(),
@@ -4028,11 +3987,7 @@ mod tests {
}
fn test_command() -> protocol::WorkerCommandEnvelope {
protocol::WorkerCommandEnvelope {
command_id: 1,
expected_execution_generation: 1,
expected_worker_state_revision: 0,
}
protocol::WorkerCommandEnvelope { command_id: 1 }
}
#[test]
@@ -4891,7 +4846,6 @@ mod tests {
restore_result: Mutex<Option<WorkerExecutionSpawnResult>>,
restore_gate: Mutex<Option<Arc<RestoreGate>>>,
restore_count: Mutex<u64>,
run_generations: Mutex<Vec<u64>>,
config_bundles: Mutex<Vec<Option<ConfigBundle>>>,
workspace_config_fetches: Mutex<Vec<WorkspaceConfigFetchRequest>>,
workspace_config_results: Mutex<Vec<WorkspaceConfigFetchResult>>,
@@ -4989,10 +4943,6 @@ mod tests {
}
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
self.run_generations
.lock()
.unwrap()
.push(request.run_generation);
self.config_bundles
.lock()
.unwrap()
@@ -5004,7 +4954,6 @@ mod tests {
WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
worker_state: protocol::WorkerStateSnapshot {
execution_generation: request.run_generation,
..protocol::WorkerStatus::Idle.into()
},
working_directory: request
@@ -5023,10 +4972,6 @@ mod tests {
if let Some(gate) = restore_gate {
gate.enter_and_wait();
}
self.run_generations
.lock()
.unwrap()
.push(request.run_generation);
self.config_bundles
.lock()
.unwrap()
@@ -5041,7 +4986,6 @@ mod tests {
WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
worker_state: protocol::WorkerStateSnapshot {
execution_generation: request.run_generation,
..protocol::WorkerStatus::Idle.into()
},
working_directory: request
@@ -5962,11 +5906,8 @@ mod tests {
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert_eq!(restored.status, WorkerStatus::Idle);
assert_eq!(
restored
.worker_state
.as_ref()
.map(|state| state.execution_generation),
Some(2)
restored.worker_state.as_ref().map(|state| &state.state),
Some(&protocol::WorkerState::Idle)
);
}
@@ -6010,16 +5951,7 @@ mod tests {
);
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert_eq!(restored[0].worker_ref, restored[1].worker_ref);
assert_eq!(
restored[0]
.worker_state
.as_ref()
.map(|state| state.execution_generation),
restored[1]
.worker_state
.as_ref()
.map(|state| state.execution_generation)
);
assert_eq!(restored[0].worker_state, restored[1].worker_state);
assert!(runtime.worker_operations.lock().unwrap().is_empty());
}
@@ -6087,69 +6019,55 @@ mod tests {
let worker_state = restored
.worker_state
.expect("restored Worker must expose its initial state");
assert_eq!(worker_state.execution_generation, 2);
assert_eq!(worker_state.last_command_id, 0);
assert_eq!(worker_state.state, protocol::WorkerState::Idle);
}
#[test]
fn runtime_applies_only_newer_worker_state_snapshots() {
fn runtime_replaces_worker_state_with_each_full_snapshot() {
let (runtime, _) = runtime_and_backend();
let detail = runtime
.create_worker(task_request("state ordering"))
.create_worker(task_request("state replacement"))
.unwrap();
let running = protocol::WorkerStateSnapshot {
execution_generation: 7,
revision: 3,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 2,
};
assert!({
{
let mut state = runtime.lock().unwrap();
state.project_protocol_event_to_worker_state(
assert!(state.project_protocol_event_to_worker_state(
&detail.worker_ref,
&protocol::Event::WorkerState {
snapshot: running.clone(),
},
)
});
));
}
assert_eq!(
runtime
.worker_detail(&detail.worker_ref)
.unwrap()
.worker_state,
Some(running.clone())
Some(running)
);
assert!({
let fresh = protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
last_command_id: 0,
};
{
let mut state = runtime.lock().unwrap();
!state.project_protocol_event_to_worker_state(
assert!(state.project_protocol_event_to_worker_state(
&detail.worker_ref,
&protocol::Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
revision: 2,
state: protocol::WorkerState::Idle,
..running.clone()
},
snapshot: fresh.clone(),
},
)
});
assert!({
let mut state = runtime.lock().unwrap();
!state.project_protocol_event_to_worker_state(
&detail.worker_ref,
&protocol::Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
..running.clone()
},
},
)
});
));
}
let after = runtime.worker_detail(&detail.worker_ref).unwrap();
assert_eq!(after.status, WorkerStatus::Idle);
assert_eq!(after.worker_state, Some(running));
assert_eq!(after.worker_state, Some(fresh));
}
#[test]
@@ -6369,7 +6287,6 @@ mod tests {
WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
worker_state: protocol::WorkerStateSnapshot {
execution_generation: request.run_generation,
..protocol::WorkerStatus::Idle.into()
},
working_directory: request
@@ -6468,15 +6385,14 @@ mod tests {
.unwrap();
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]);
let restored = runtime.worker_detail(&detail.worker_ref).unwrap();
assert_eq!(restored.status, WorkerStatus::Idle);
assert_eq!(
restored
.worker_state
.as_ref()
.map(|snapshot| (snapshot.execution_generation, &snapshot.state)),
Some((2, &protocol::WorkerState::Idle))
.map(|snapshot| &snapshot.state),
Some(&protocol::WorkerState::Idle)
);
}
@@ -6828,7 +6744,7 @@ mod tests {
assert!(
error
.to_string()
.contains("unsupported Runtime store schema version 2; expected 3, 4, 5, or 6")
.contains("unsupported Runtime store schema version 2; expected 6 or 7")
);
let _ = std::fs::remove_dir_all(root);
@@ -6870,15 +6786,16 @@ mod tests {
let worker_snapshot: serde_json::Value =
serde_json::from_slice(&std::fs::read(worker_store_dir.join("worker.json")).unwrap())
.unwrap();
assert_eq!(worker_snapshot["schema_version"], serde_json::json!(6));
assert_eq!(worker_snapshot["schema_version"], serde_json::json!(7));
assert_eq!(worker_snapshot["status"], serde_json::json!("stopped"));
assert_eq!(
worker_snapshot["execution"]["last_run_generation"],
serde_json::json!(1)
assert!(
worker_snapshot["execution"]
.get("last_run_generation")
.is_none()
);
assert_eq!(
worker_snapshot["execution"]["binding"]["run_generation"],
serde_json::json!(1)
worker_snapshot["execution"]["binding"],
serde_json::json!({})
);
assert_eq!(
worker_snapshot["execution"]["restore_intent"],
@@ -7256,8 +7173,8 @@ mod tests {
#[cfg(feature = "fs-store")]
#[test]
fn fs_store_migrates_schema_v3_workers_to_stopped_explicit_restore() {
let root = fs_store_root("schema-v3-restore-intent");
fn fs_store_migrates_schema_v6_workers_without_losing_automatic_restore() {
let root = fs_store_root("schema-v6-no-generation");
let options = crate::fs_store::FsRuntimeStoreOptions {
root: root.clone(),
runtime_id: "test-runtime".to_string(),
@@ -7270,7 +7187,7 @@ mod tests {
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let worker = runtime
.create_worker(task_request("schema v3 worker"))
.create_worker(task_request("schema v6 worker"))
.unwrap();
drop(runtime);
@@ -7281,7 +7198,7 @@ mod tests {
.join("worker.json");
let mut runtime_json: serde_json::Value =
serde_json::from_slice(&std::fs::read(&runtime_path).unwrap()).unwrap();
runtime_json["schema_version"] = serde_json::json!(3);
runtime_json["schema_version"] = serde_json::json!(6);
std::fs::write(
&runtime_path,
serde_json::to_vec_pretty(&runtime_json).unwrap(),
@@ -7289,10 +7206,9 @@ mod tests {
.unwrap();
let mut worker_json: serde_json::Value =
serde_json::from_slice(&std::fs::read(&worker_path).unwrap()).unwrap();
worker_json["schema_version"] = serde_json::json!(3);
worker_json.as_object_mut().unwrap().remove("status");
worker_json.as_object_mut().unwrap().remove("execution");
worker_json["run_generation"] = serde_json::json!(7);
worker_json["schema_version"] = serde_json::json!(6);
worker_json["execution"]["last_run_generation"] = serde_json::json!(1);
worker_json["execution"]["binding"] = serde_json::json!({"run_generation": 1});
std::fs::write(
&worker_path,
serde_json::to_vec_pretty(&worker_json).unwrap(),
@@ -7302,35 +7218,24 @@ mod tests {
let backend = Arc::new(TestExecutionBackend::default());
let migrated =
Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert_eq!(
migrated.worker_detail(&worker.worker_ref).unwrap().status,
WorkerStatus::Stopped
WorkerStatus::Idle
);
let migrated_json: serde_json::Value =
serde_json::from_slice(&std::fs::read(&worker_path).unwrap()).unwrap();
assert_eq!(migrated_json["schema_version"], serde_json::json!(6));
assert_eq!(migrated_json["status"], serde_json::json!("stopped"));
assert_eq!(
migrated_json["execution"]["last_run_generation"],
serde_json::json!(7)
);
assert_eq!(
migrated_json["execution"]["binding"],
serde_json::Value::Null
assert_eq!(migrated_json["schema_version"], serde_json::json!(7));
assert_eq!(migrated_json["execution"]["binding"], serde_json::json!({}));
assert!(
migrated_json["execution"]
.get("last_run_generation")
.is_none()
);
assert_eq!(
migrated_json["execution"]["restore_intent"],
serde_json::json!("explicit")
serde_json::json!("automatic")
);
assert!(matches!(
migrated.send_input(&worker.worker_ref, WorkerInput::notify("do not restore")),
Err(RuntimeError::WorkerExecutionUnavailable { .. })
));
migrated.restore_worker(&worker.worker_ref).unwrap();
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert_eq!(backend.run_generations.lock().unwrap().as_slice(), &[8]);
let _ = std::fs::remove_dir_all(root);
}
crates/worker-runtime/src/worker_backend.rs
+51 -59
View File
@@ -57,7 +57,7 @@ fn next_internal_command(
})
.unwrap_or(floor)
.max(floor);
Ok(WorkerCommandEnvelope::for_snapshot(command_id, &snapshot))
Ok(WorkerCommandEnvelope::new(command_id))
}
use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore};
#[cfg(test)]
@@ -403,6 +403,13 @@ impl ProfileRuntimeWorkerFactory {
})
}
fn worker_run_dir(&self, worker_ref: &WorkerRef) -> Result<PathBuf, String> {
Ok(self
.worker_aggregate_dir(worker_ref)?
.join("runs")
.join(uuid::Uuid::now_v7().to_string()))
}
fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String {
format!("worker-runtime-{}", worker_ref.worker_id)
}
@@ -898,9 +905,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
)?;
let store = CombinedStore::new(session_store, worker_metadata_store);
let run_dir = worker_aggregate_dir
.join("runs")
.join(request.run_generation.to_string());
let run_dir = self.worker_run_dir(&request.worker_ref)?;
let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id);
let mut prepared = WorkerBootstrap::new(
manifest,
@@ -1152,9 +1157,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
}
let workspace_client = worker.workspace_client_handle();
let run_dir = worker_aggregate_dir
.join("runs")
.join(request.run_generation.to_string());
let run_dir = self.worker_run_dir(&request.worker_ref)?;
let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id);
let started = PreparedWorker::new(
worker,
@@ -1745,25 +1748,17 @@ fn apply_protocol_worker_state(
current: &Arc<RwLock<protocol::WorkerStateSnapshot>>,
event: &mut Event,
) -> Result<bool, String> {
let (incoming, replace_stale) = match event {
Event::WorkerState { snapshot } => (snapshot, false),
Event::Snapshot { state, .. } => (state, true),
Event::CommandAcknowledged { acknowledgement } => (&mut acknowledgement.state, true),
let incoming = match event {
Event::WorkerState { snapshot } => snapshot,
Event::Snapshot { state, .. } => state,
Event::CommandAcknowledged { acknowledgement } => &mut acknowledgement.state,
_ => return Ok(true),
};
let mut current = current
.write()
.map_err(|_| "worker state projection lock is poisoned".to_string())?;
match protocol::apply_worker_state_snapshot(&mut current, incoming) {
Ok(protocol::WorkerStateSnapshotApply::Applied)
| Ok(protocol::WorkerStateSnapshotApply::Duplicate) => Ok(true),
Ok(protocol::WorkerStateSnapshotApply::Stale) if replace_stale => {
*incoming = current.clone();
Ok(true)
}
Ok(protocol::WorkerStateSnapshotApply::Stale) => Ok(false),
Err(error) => Err(error.to_string()),
}
*current = incoming.clone();
Ok(true)
}
impl<F> WorkerExecutionBackend for WorkerRuntimeExecutionBackend<F>
@@ -2572,37 +2567,33 @@ mod tests {
.read()
.unwrap()
.clone();
WorkerCommandEnvelope::for_snapshot(state.last_command_id.saturating_add(1), &state)
WorkerCommandEnvelope::new(state.last_command_id.saturating_add(1))
}
#[test]
fn protocol_bridge_applies_state_and_acknowledgement_monotonically() {
fn protocol_bridge_replaces_every_full_state_snapshot() {
let running = protocol::WorkerStateSnapshot {
execution_generation: 4,
revision: 3,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 2,
};
let current = Arc::new(RwLock::new(running.clone()));
let mut stale = Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
revision: 2,
state: protocol::WorkerState::Idle,
..running.clone()
},
let current = Arc::new(RwLock::new(running));
let idle = protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
last_command_id: 0,
};
assert!(!apply_protocol_worker_state(&current, &mut stale).unwrap());
assert_eq!(*current.read().unwrap(), running);
let mut replacement = Event::WorkerState {
snapshot: idle.clone(),
};
assert!(apply_protocol_worker_state(&current, &mut replacement).unwrap());
assert_eq!(*current.read().unwrap(), idle);
let paused = protocol::WorkerStateSnapshot {
revision: 4,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Paused,
)),
last_command_id: 3,
..running.clone()
};
let mut acknowledgement = Event::CommandAcknowledged {
acknowledgement: protocol::WorkerCommandAcknowledgement {
@@ -2614,15 +2605,6 @@ mod tests {
};
assert!(apply_protocol_worker_state(&current, &mut acknowledgement).unwrap());
assert_eq!(*current.read().unwrap(), paused);
let mut conflict = Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
..paused.clone()
},
};
assert!(apply_protocol_worker_state(&current, &mut conflict).is_err());
assert_eq!(*current.read().unwrap(), paused);
}
#[test]
@@ -3037,7 +3019,6 @@ mod tests {
) -> Result<RuntimeWorkerController, String> {
let request = WorkerExecutionSpawnRequest {
worker_ref: request.worker_ref,
run_generation: request.run_generation,
request: request.request,
workspace_scope: request.workspace_scope,
context: request.context,
@@ -3388,7 +3369,6 @@ mod tests {
crate::identity::WorkerRef::new(crate::identity::WorkerId::from_legacy_u64(1));
let request = WorkerExecutionSpawnRequest {
worker_ref: worker_ref.clone(),
run_generation: 1,
request: create_request("1"),
workspace_scope: None,
context: test_execution_context(worker_ref),
@@ -3518,7 +3498,6 @@ mod tests {
.with_remote_worker_mutation_identity(identity)
.restore_controller(WorkerExecutionRestoreRequest {
worker_ref: worker_ref.clone(),
run_generation: 1,
request,
workspace_scope: Some(crate::runtime::RuntimeWorkspaceScope::new(
"workspace-restore",
@@ -3583,11 +3562,13 @@ mod tests {
)
.unwrap();
let run_dir = runtime_store_dir
let runs_dir = runtime_store_dir
.join("workers")
.join(worker_ref.worker_id.to_string())
.join("runs/2");
let socket_path = run_dir.join("worker.sock");
.join("runs");
let socket_path = runs_dir
.join(uuid::Uuid::nil().to_string())
.join("worker.sock");
assert!(
socket_path.as_os_str().as_encoded_bytes().len() > 107,
"test path must exceed Linux sockaddr_un.sun_path capacity: {}",
@@ -3599,7 +3580,6 @@ mod tests {
.with_controller_transport(WorkerControllerTransport::InProcess)
.restore_controller(WorkerExecutionRestoreRequest {
worker_ref: worker_ref.clone(),
run_generation: 2,
request: create_request("embedded restore"),
workspace_scope: None,
context: test_execution_context(worker_ref),
@@ -3614,7 +3594,12 @@ mod tests {
controller.handle.shared_state.catalog_status(),
WorkerStatus::Idle
);
assert!(!socket_path.exists());
let run_dir = std::fs::read_dir(&runs_dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.next()
.expect("restore must create one run artifact directory");
assert!(!run_dir.join("worker.sock").exists());
assert!(run_dir.join("worker.out.log").is_file());
assert!(run_dir.join("worker.err.log").is_file());
let worker_state = Arc::new(RwLock::new(controller.handle.shared_state.snapshot()));
@@ -3726,10 +3711,16 @@ mod tests {
let mut request = create_request("embedded singleton");
request.profile = ProfileSelector::Builtin("default".to_string());
let worker = runtime.create_worker(request).unwrap();
let first_run_socket = runtime_store_dir
let runs_dir = runtime_store_dir
.join("workers")
.join(worker.worker_id.to_string())
.join("runs/1/worker.sock");
.join("runs");
let first_run = std::fs::read_dir(&runs_dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.next()
.expect("fresh launch must create one run artifact directory");
let first_run_socket = first_run.join("worker.sock");
assert!(
first_run_socket.as_os_str().as_encoded_bytes().len() > 107,
"test path must exceed Linux sockaddr_un.sun_path capacity: {}",
@@ -3766,10 +3757,11 @@ mod tests {
diagnostic.code == "worker_execution_restore_failed"
&& diagnostic.worker_ref.as_ref() == Some(&worker.worker_ref)
}));
let restored_run = runtime_store_dir
.join("workers")
.join(worker.worker_id.to_string())
.join("runs/2");
let restored_run = std::fs::read_dir(&runs_dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.find(|path| path != &first_run)
.expect("restore must create a distinct run artifact directory");
assert!(!restored_run.join("worker.sock").exists());
assert!(restored_run.join("worker.out.log").is_file());
assert!(restored_run.join("worker.err.log").is_file());