runtime: add revisioned worker retention authority

This commit is contained in:
2026-08-11 23:57:37 +09:00
parent 09fd17e38b
commit da8313fa1a
7 changed files with 2599 additions and 12 deletions
+2
View File
@@ -22,6 +22,8 @@ pub mod management;
pub mod observation;
pub mod profile_archive;
pub mod resource;
#[cfg(feature = "fs-store")]
pub mod retention;
mod runtime;
pub mod worker_backend;
pub mod working_directory;
File diff suppressed because it is too large Load Diff
+135
View File
@@ -26,6 +26,11 @@ use crate::management::{
};
#[cfg(feature = "ws-server")]
use crate::observation::{WorkerObservationCursor, WorkerObservationEvent};
#[cfg(feature = "fs-store")]
use crate::retention::{
FsWorkerRetentionProvider, WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult,
WorkerRetentionInventory, WorkerRetentionProvider,
};
use protocol::subscription::{
EventSubscriptionSelector, SubscriptionEventPayload, SubscriptionSnapshot,
SubscriptionValidationError, SubscriptionWorkdirId, SubscriptionWorker, SubscriptionWorkerId,
@@ -1648,6 +1653,118 @@ impl Runtime {
Ok(())
}
/// Bind the Backend registry identity once. Retention evidence fails closed
/// until the Runtime host supplies this trusted configuration.
pub fn bind_runtime_identity(&self, runtime_id: &str) -> Result<(), RuntimeError> {
if runtime_id.trim().is_empty() || runtime_id.len() > 160 {
return Err(RuntimeError::InvalidRequest(
"Runtime identity must be non-empty and bounded".to_string(),
));
}
let mut state = self.lock()?;
match state.runtime_identity.as_deref() {
Some(current) if current == runtime_id => Ok(()),
Some(_) => Err(RuntimeError::InvalidRequest(
"Runtime identity is already bound".to_string(),
)),
None => {
state.runtime_identity = Some(runtime_id.to_string());
Ok(())
}
}
}
/// Read canonical aggregate facts needed by a Backend removal plan.
#[cfg(feature = "fs-store")]
pub fn worker_retention_inventory(
&self,
workspace_id: &str,
worker_ref: &WorkerRef,
) -> Result<WorkerRetentionInventory, RuntimeError> {
let state = self.lock()?;
let runtime_id = state.runtime_identity.as_deref().ok_or_else(|| {
RuntimeError::InvalidRequest(
"Runtime identity is not bound for Worker retention".to_string(),
)
})?;
let worker = state.worker(worker_ref)?;
if worker.workspace_id.as_deref() != Some(workspace_id) {
return Err(RuntimeError::WorkerNotFound {
worker_id: worker_ref.worker_id,
});
}
let store = state.fs_store().ok_or_else(|| {
RuntimeError::InvalidRequest(
"Worker retention archive authority requires an fs-backed Runtime".to_string(),
)
})?;
FsWorkerRetentionProvider::new(store.runtime_dir()).inventory(
workspace_id,
runtime_id,
worker.worker_id,
worker.run_generation,
)
}
/// Execute a Backend-resolved retention plan. Only stopped Workers are
/// eligible. Provider receipt lookup happens before live lookup so exact
/// retries converge after aggregate removal.
#[cfg(feature = "fs-store")]
pub fn execute_worker_retention(
&self,
request: &WorkerRetentionExecutionRequest,
) -> Result<WorkerRetentionExecutionResult, RuntimeError> {
let mut state = self.lock()?;
let runtime_id = state.runtime_identity.clone().ok_or_else(|| {
RuntimeError::InvalidRequest(
"Runtime identity is not bound for Worker retention".to_string(),
)
})?;
if request.source_runtime_id != runtime_id {
return Err(RuntimeError::InvalidRequest(
"Worker retention Runtime identity mismatch".to_string(),
));
}
let store = state.fs_store().ok_or_else(|| {
RuntimeError::InvalidRequest(
"Worker retention execution requires an fs-backed Runtime".to_string(),
)
})?;
let provider = FsWorkerRetentionProvider::new(store.runtime_dir());
if let Some(completed) =
provider.completed(&request.operation_id, &request.input_fingerprint)?
{
state.workers.remove(&request.worker_id);
state.persist_runtime_snapshot()?;
return Ok(completed);
}
let Some(worker) = state.workers.get(&request.worker_id) else {
// Recover a pending receipt after a crash between aggregate removal
// and final receipt/Runtime catalog commit.
return provider.recover_after_source_removal(request);
};
if worker.workspace_id.as_deref() != Some(request.workspace_id.as_str()) {
return Err(RuntimeError::WorkerNotFound {
worker_id: request.worker_id,
});
}
if worker.status != WorkerStatus::Stopped {
return Err(RuntimeError::InvalidRequest(
"Worker retention requires a stopped Worker".to_string(),
));
}
if worker.run_generation != request.expected_run_generation {
return Err(RuntimeError::InvalidRequest(format!(
"Worker retention plan expected generation {}, current generation is {}",
request.expected_run_generation, worker.run_generation
)));
}
let result = provider.execute(request)?;
state.workers.remove(&request.worker_id);
state.persist_runtime_snapshot()?;
Ok(result)
}
fn lock(&self) -> Result<MutexGuard<'_, RuntimeState>, RuntimeError> {
self.inner.lock().map_err(|_| RuntimeError::StatePoisoned)
}
@@ -1673,6 +1790,9 @@ struct SubscriptionSink {
struct RuntimeState {
display_name: Option<String>,
backend: RuntimeBackendKind,
/// Backend-bound stable identity used for cross-boundary retention evidence.
/// It is configured once by the Runtime host and never model input.
runtime_identity: Option<String>,
#[cfg_attr(not(feature = "fs-store"), allow(dead_code))]
persistence: RuntimePersistence,
status: RuntimeStatus,
@@ -1701,6 +1821,7 @@ impl RuntimeState {
Self {
display_name,
backend: RuntimeBackendKind::Memory,
runtime_identity: None,
persistence: RuntimePersistence::Memory,
status: RuntimeStatus::Running,
execution_backend: None,
@@ -1729,6 +1850,7 @@ impl RuntimeState {
Self {
display_name,
backend: RuntimeBackendKind::FsStore,
runtime_identity: None,
persistence: RuntimePersistence::Fs(store),
status: RuntimeStatus::Running,
execution_backend: None,
@@ -1779,6 +1901,7 @@ impl RuntimeState {
Ok(Self {
display_name: persisted.display_name,
backend: RuntimeBackendKind::FsStore,
runtime_identity: None,
persistence: RuntimePersistence::Fs(store),
status: persisted.status,
execution_backend: None,
@@ -2550,6 +2673,18 @@ mod tests {
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
#[test]
fn runtime_identity_binding_is_immutable_and_host_owned() {
let runtime = Runtime::new_memory();
runtime.bind_runtime_identity("runtime-a").unwrap();
runtime.bind_runtime_identity("runtime-a").unwrap();
assert!(runtime.bind_runtime_identity("runtime-b").is_err());
assert_eq!(
runtime.lock().unwrap().runtime_identity.as_deref(),
Some("runtime-a")
);
}
#[test]
fn typed_segments_allow_empty_flat_content() {
let input = WorkerInput {