fix: fail closed on unknown standalone leases

This commit is contained in:
2026-08-30 19:44:15 +09:00
parent 133feb8c76
commit 31e18205f0
3 changed files with 168 additions and 16 deletions
+5
View File
@@ -41,6 +41,8 @@ pub enum StandaloneStartupError {
StateStore,
#[error("the standalone session is already active")]
SessionActive,
#[error("the standalone session lease cannot be observed safely; recovery is rejected")]
LeaseLivenessUnknown,
#[error("the standalone session working directory is unavailable or changed")]
WorkingDirectoryUnavailable,
#[error("the resolved Worker configuration or persisted history is invalid")]
@@ -380,6 +382,9 @@ async fn stop_started_worker(started: BootstrappedWorker) {
fn classify_store_startup_error(error: StandaloneStoreError) -> StandaloneStartupError {
match error {
StandaloneStoreError::SessionLeased(_) => StandaloneStartupError::SessionActive,
StandaloneStoreError::LeaseLivenessUnknown(_) => {
StandaloneStartupError::LeaseLivenessUnknown
}
StandaloneStoreError::CwdUnavailable(_)
| StandaloneStoreError::CwdNotDirectory
| StandaloneStoreError::CwdIdentityMismatch => {
+120 -16
View File
@@ -304,8 +304,14 @@ impl StandaloneSessionStore {
}
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
let existing = read_lease(&path, id)?;
if existing.is_live() {
return Err(StandaloneStoreError::SessionLeased(id));
match existing.liveness() {
LeaseLiveness::Live => {
return Err(StandaloneStoreError::SessionLeased(id));
}
LeaseLiveness::Unknown => {
return Err(StandaloneStoreError::LeaseLivenessUnknown(id));
}
LeaseLiveness::Stale => {}
}
if policy == StaleLeasePolicy::Reject {
return Err(StandaloneStoreError::StaleLease(id));
@@ -363,10 +369,10 @@ impl StandaloneSessionStore {
let lease_path = session_dir.join(LEASE_FILE);
if lease_path.exists() {
let lease = read_lease(&lease_path, id)?;
return Err(if lease.is_live() {
StandaloneStoreError::SessionLeased(id)
} else {
StandaloneStoreError::StaleLease(id)
return Err(match lease.liveness() {
LeaseLiveness::Live => StandaloneStoreError::SessionLeased(id),
LeaseLiveness::Stale => StandaloneStoreError::StaleLease(id),
LeaseLiveness::Unknown => StandaloneStoreError::LeaseLivenessUnknown(id),
});
}
fs::remove_dir_all(self.session_dir(id)).map_err(StandaloneStoreError::Io)?;
@@ -572,15 +578,49 @@ impl LeaseRecord {
Ok(Self {
lease_id: Uuid::now_v7(),
pid: std::process::id(),
process_start_marker: process_start_marker(std::process::id()),
process_start_marker: match observe_process(std::process::id()) {
ProcessObservation::Running { start_marker } => Some(start_marker),
ProcessObservation::Missing | ProcessObservation::Unobservable => None,
},
acquired_at_unix_ms: now_unix_ms()?,
})
}
fn is_live(&self) -> bool {
process_start_marker(self.pid)
.zip(self.process_start_marker)
.is_some_and(|(current, recorded)| current == recorded)
fn liveness(&self) -> LeaseLiveness {
classify_lease_liveness(self.process_start_marker, observe_process(self.pid))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LeaseLiveness {
Live,
Stale,
Unknown,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ProcessObservation {
Running { start_marker: u64 },
Missing,
Unobservable,
}
fn classify_lease_liveness(
recorded_start_marker: Option<u64>,
observation: ProcessObservation,
) -> LeaseLiveness {
match (recorded_start_marker, observation) {
(Some(recorded), ProcessObservation::Running { start_marker })
if recorded == start_marker =>
{
LeaseLiveness::Live
}
(Some(_), ProcessObservation::Running { .. }) | (_, ProcessObservation::Missing) => {
LeaseLiveness::Stale
}
(None, ProcessObservation::Running { .. }) | (_, ProcessObservation::Unobservable) => {
LeaseLiveness::Unknown
}
}
}
@@ -591,15 +631,44 @@ fn read_lease(path: &Path, id: StandaloneSessionId) -> Result<LeaseRecord, Stand
}
#[cfg(target_os = "linux")]
fn process_start_marker(pid: u32) -> Option<u64> {
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
let tail = stat.rsplit_once(") ")?.1;
fn observe_process(pid: u32) -> ProcessObservation {
let stat = match fs::read_to_string(format!("/proc/{pid}/stat")) {
Ok(stat) => stat,
Err(error) if error.kind() == io::ErrorKind::NotFound => {
return if pid != std::process::id() && linux_proc_is_observable() {
ProcessObservation::Missing
} else {
ProcessObservation::Unobservable
};
}
Err(_) => return ProcessObservation::Unobservable,
};
parse_linux_process_start_marker(&stat)
.map(|start_marker| ProcessObservation::Running { start_marker })
.unwrap_or(ProcessObservation::Unobservable)
}
#[cfg(target_os = "linux")]
fn linux_proc_is_observable() -> bool {
fs::read_to_string("/proc/self/stat")
.ok()
.and_then(|stat| parse_linux_process_start_marker(&stat))
.is_some()
}
#[cfg(target_os = "linux")]
fn parse_linux_process_start_marker(stat: &str) -> Option<u64> {
let (_, tail) = stat.rsplit_once(") ")?;
tail.split_whitespace().nth(19)?.parse().ok()
}
#[cfg(not(target_os = "linux"))]
fn process_start_marker(pid: u32) -> Option<u64> {
(pid == std::process::id()).then_some(0)
fn observe_process(pid: u32) -> ProcessObservation {
if pid == std::process::id() {
ProcessObservation::Running { start_marker: 0 }
} else {
ProcessObservation::Unobservable
}
}
fn now_unix_ms() -> Result<u64, StandaloneStoreError> {
@@ -651,6 +720,8 @@ pub enum StandaloneStoreError {
},
#[error("standalone session {0} is already active")]
SessionLeased(StandaloneSessionId),
#[error("standalone session {0} lease liveness cannot be proven; recovery is rejected")]
LeaseLivenessUnknown(StandaloneSessionId),
#[error("standalone session {0} has a stale lease; explicit recovery is required")]
StaleLease(StandaloneSessionId),
#[error("standalone session lease ownership changed")]
@@ -672,3 +743,36 @@ pub enum StandaloneStoreError {
#[error("standalone state I/O failed")]
Io(#[source] io::Error),
}
#[cfg(test)]
mod tests {
use super::{LeaseLiveness, ProcessObservation, classify_lease_liveness};
#[test]
fn lease_liveness_requires_positive_live_or_stale_evidence() {
assert_eq!(
classify_lease_liveness(Some(41), ProcessObservation::Running { start_marker: 41 }),
LeaseLiveness::Live
);
assert_eq!(
classify_lease_liveness(Some(41), ProcessObservation::Running { start_marker: 42 }),
LeaseLiveness::Stale
);
assert_eq!(
classify_lease_liveness(Some(41), ProcessObservation::Missing),
LeaseLiveness::Stale
);
assert_eq!(
classify_lease_liveness(None, ProcessObservation::Running { start_marker: 41 }),
LeaseLiveness::Unknown
);
assert_eq!(
classify_lease_liveness(Some(41), ProcessObservation::Unobservable),
LeaseLiveness::Unknown
);
assert_eq!(
classify_lease_liveness(None, ProcessObservation::Unobservable),
LeaseLiveness::Unknown
);
}
}
+43
View File
@@ -426,6 +426,49 @@ async fn standalone_restore_recovers_only_a_proven_stale_lease() -> TestResult {
Ok(())
}
#[tokio::test]
async fn standalone_restore_rejects_lease_with_missing_start_marker() -> TestResult {
let temp = tempfile::tempdir()?;
let state_dir = temp.path().join("state");
let launch = StandaloneLaunchConfig::new(
temp.path(),
&state_dir,
manifest::ProfileSelector::Default,
"standalone-unknown-lease-test",
)
.resolve()?;
let host =
StandaloneHost::start_with_model_client(launch, ScriptedClient::new(Vec::new())).await?;
let session_id = host.session_id();
host.shutdown().await?;
let session_dir = state_dir.join(session_id.to_string());
std::fs::write(
session_dir.join("lease.json"),
serde_json::to_vec(&serde_json::json!({
"lease_id": uuid::Uuid::now_v7(),
"pid": std::process::id(),
"acquired_at_unix_ms": 1
}))?,
)?;
let store = StandaloneSessionStore::open(&state_dir)?;
assert!(matches!(
store.acquire_lease(session_id, StaleLeasePolicy::Recover),
Err(StandaloneStoreError::LeaseLivenessUnknown(id)) if id == session_id
));
let restore = StandaloneHost::restore_with_model_client(
state_dir,
session_id,
ScriptedClient::new(Vec::new()),
)
.await;
assert!(matches!(
restore,
Err(StandaloneStartupError::LeaseLivenessUnknown)
));
Ok(())
}
#[tokio::test]
async fn standalone_metadata_fails_closed_on_incomplete_or_newer_records() -> TestResult {
let temp = tempfile::tempdir()?;