fix: fail closed on unknown standalone leases
This commit is contained in:
@@ -41,6 +41,8 @@ pub enum StandaloneStartupError {
|
|||||||
StateStore,
|
StateStore,
|
||||||
#[error("the standalone session is already active")]
|
#[error("the standalone session is already active")]
|
||||||
SessionActive,
|
SessionActive,
|
||||||
|
#[error("the standalone session lease cannot be observed safely; recovery is rejected")]
|
||||||
|
LeaseLivenessUnknown,
|
||||||
#[error("the standalone session working directory is unavailable or changed")]
|
#[error("the standalone session working directory is unavailable or changed")]
|
||||||
WorkingDirectoryUnavailable,
|
WorkingDirectoryUnavailable,
|
||||||
#[error("the resolved Worker configuration or persisted history is invalid")]
|
#[error("the resolved Worker configuration or persisted history is invalid")]
|
||||||
@@ -380,6 +382,9 @@ async fn stop_started_worker(started: BootstrappedWorker) {
|
|||||||
fn classify_store_startup_error(error: StandaloneStoreError) -> StandaloneStartupError {
|
fn classify_store_startup_error(error: StandaloneStoreError) -> StandaloneStartupError {
|
||||||
match error {
|
match error {
|
||||||
StandaloneStoreError::SessionLeased(_) => StandaloneStartupError::SessionActive,
|
StandaloneStoreError::SessionLeased(_) => StandaloneStartupError::SessionActive,
|
||||||
|
StandaloneStoreError::LeaseLivenessUnknown(_) => {
|
||||||
|
StandaloneStartupError::LeaseLivenessUnknown
|
||||||
|
}
|
||||||
StandaloneStoreError::CwdUnavailable(_)
|
StandaloneStoreError::CwdUnavailable(_)
|
||||||
| StandaloneStoreError::CwdNotDirectory
|
| StandaloneStoreError::CwdNotDirectory
|
||||||
| StandaloneStoreError::CwdIdentityMismatch => {
|
| StandaloneStoreError::CwdIdentityMismatch => {
|
||||||
|
|||||||
+120
-16
@@ -304,8 +304,14 @@ impl StandaloneSessionStore {
|
|||||||
}
|
}
|
||||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
|
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
|
||||||
let existing = read_lease(&path, id)?;
|
let existing = read_lease(&path, id)?;
|
||||||
if existing.is_live() {
|
match existing.liveness() {
|
||||||
return Err(StandaloneStoreError::SessionLeased(id));
|
LeaseLiveness::Live => {
|
||||||
|
return Err(StandaloneStoreError::SessionLeased(id));
|
||||||
|
}
|
||||||
|
LeaseLiveness::Unknown => {
|
||||||
|
return Err(StandaloneStoreError::LeaseLivenessUnknown(id));
|
||||||
|
}
|
||||||
|
LeaseLiveness::Stale => {}
|
||||||
}
|
}
|
||||||
if policy == StaleLeasePolicy::Reject {
|
if policy == StaleLeasePolicy::Reject {
|
||||||
return Err(StandaloneStoreError::StaleLease(id));
|
return Err(StandaloneStoreError::StaleLease(id));
|
||||||
@@ -363,10 +369,10 @@ impl StandaloneSessionStore {
|
|||||||
let lease_path = session_dir.join(LEASE_FILE);
|
let lease_path = session_dir.join(LEASE_FILE);
|
||||||
if lease_path.exists() {
|
if lease_path.exists() {
|
||||||
let lease = read_lease(&lease_path, id)?;
|
let lease = read_lease(&lease_path, id)?;
|
||||||
return Err(if lease.is_live() {
|
return Err(match lease.liveness() {
|
||||||
StandaloneStoreError::SessionLeased(id)
|
LeaseLiveness::Live => StandaloneStoreError::SessionLeased(id),
|
||||||
} else {
|
LeaseLiveness::Stale => StandaloneStoreError::StaleLease(id),
|
||||||
StandaloneStoreError::StaleLease(id)
|
LeaseLiveness::Unknown => StandaloneStoreError::LeaseLivenessUnknown(id),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
fs::remove_dir_all(self.session_dir(id)).map_err(StandaloneStoreError::Io)?;
|
fs::remove_dir_all(self.session_dir(id)).map_err(StandaloneStoreError::Io)?;
|
||||||
@@ -572,15 +578,49 @@ impl LeaseRecord {
|
|||||||
Ok(Self {
|
Ok(Self {
|
||||||
lease_id: Uuid::now_v7(),
|
lease_id: Uuid::now_v7(),
|
||||||
pid: std::process::id(),
|
pid: std::process::id(),
|
||||||
process_start_marker: process_start_marker(std::process::id()),
|
process_start_marker: match observe_process(std::process::id()) {
|
||||||
|
ProcessObservation::Running { start_marker } => Some(start_marker),
|
||||||
|
ProcessObservation::Missing | ProcessObservation::Unobservable => None,
|
||||||
|
},
|
||||||
acquired_at_unix_ms: now_unix_ms()?,
|
acquired_at_unix_ms: now_unix_ms()?,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_live(&self) -> bool {
|
fn liveness(&self) -> LeaseLiveness {
|
||||||
process_start_marker(self.pid)
|
classify_lease_liveness(self.process_start_marker, observe_process(self.pid))
|
||||||
.zip(self.process_start_marker)
|
}
|
||||||
.is_some_and(|(current, recorded)| current == recorded)
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum LeaseLiveness {
|
||||||
|
Live,
|
||||||
|
Stale,
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum ProcessObservation {
|
||||||
|
Running { start_marker: u64 },
|
||||||
|
Missing,
|
||||||
|
Unobservable,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classify_lease_liveness(
|
||||||
|
recorded_start_marker: Option<u64>,
|
||||||
|
observation: ProcessObservation,
|
||||||
|
) -> LeaseLiveness {
|
||||||
|
match (recorded_start_marker, observation) {
|
||||||
|
(Some(recorded), ProcessObservation::Running { start_marker })
|
||||||
|
if recorded == start_marker =>
|
||||||
|
{
|
||||||
|
LeaseLiveness::Live
|
||||||
|
}
|
||||||
|
(Some(_), ProcessObservation::Running { .. }) | (_, ProcessObservation::Missing) => {
|
||||||
|
LeaseLiveness::Stale
|
||||||
|
}
|
||||||
|
(None, ProcessObservation::Running { .. }) | (_, ProcessObservation::Unobservable) => {
|
||||||
|
LeaseLiveness::Unknown
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -591,15 +631,44 @@ fn read_lease(path: &Path, id: StandaloneSessionId) -> Result<LeaseRecord, Stand
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
fn process_start_marker(pid: u32) -> Option<u64> {
|
fn observe_process(pid: u32) -> ProcessObservation {
|
||||||
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
|
let stat = match fs::read_to_string(format!("/proc/{pid}/stat")) {
|
||||||
let tail = stat.rsplit_once(") ")?.1;
|
Ok(stat) => stat,
|
||||||
|
Err(error) if error.kind() == io::ErrorKind::NotFound => {
|
||||||
|
return if pid != std::process::id() && linux_proc_is_observable() {
|
||||||
|
ProcessObservation::Missing
|
||||||
|
} else {
|
||||||
|
ProcessObservation::Unobservable
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Err(_) => return ProcessObservation::Unobservable,
|
||||||
|
};
|
||||||
|
parse_linux_process_start_marker(&stat)
|
||||||
|
.map(|start_marker| ProcessObservation::Running { start_marker })
|
||||||
|
.unwrap_or(ProcessObservation::Unobservable)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn linux_proc_is_observable() -> bool {
|
||||||
|
fs::read_to_string("/proc/self/stat")
|
||||||
|
.ok()
|
||||||
|
.and_then(|stat| parse_linux_process_start_marker(&stat))
|
||||||
|
.is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn parse_linux_process_start_marker(stat: &str) -> Option<u64> {
|
||||||
|
let (_, tail) = stat.rsplit_once(") ")?;
|
||||||
tail.split_whitespace().nth(19)?.parse().ok()
|
tail.split_whitespace().nth(19)?.parse().ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(target_os = "linux"))]
|
#[cfg(not(target_os = "linux"))]
|
||||||
fn process_start_marker(pid: u32) -> Option<u64> {
|
fn observe_process(pid: u32) -> ProcessObservation {
|
||||||
(pid == std::process::id()).then_some(0)
|
if pid == std::process::id() {
|
||||||
|
ProcessObservation::Running { start_marker: 0 }
|
||||||
|
} else {
|
||||||
|
ProcessObservation::Unobservable
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn now_unix_ms() -> Result<u64, StandaloneStoreError> {
|
fn now_unix_ms() -> Result<u64, StandaloneStoreError> {
|
||||||
@@ -651,6 +720,8 @@ pub enum StandaloneStoreError {
|
|||||||
},
|
},
|
||||||
#[error("standalone session {0} is already active")]
|
#[error("standalone session {0} is already active")]
|
||||||
SessionLeased(StandaloneSessionId),
|
SessionLeased(StandaloneSessionId),
|
||||||
|
#[error("standalone session {0} lease liveness cannot be proven; recovery is rejected")]
|
||||||
|
LeaseLivenessUnknown(StandaloneSessionId),
|
||||||
#[error("standalone session {0} has a stale lease; explicit recovery is required")]
|
#[error("standalone session {0} has a stale lease; explicit recovery is required")]
|
||||||
StaleLease(StandaloneSessionId),
|
StaleLease(StandaloneSessionId),
|
||||||
#[error("standalone session lease ownership changed")]
|
#[error("standalone session lease ownership changed")]
|
||||||
@@ -672,3 +743,36 @@ pub enum StandaloneStoreError {
|
|||||||
#[error("standalone state I/O failed")]
|
#[error("standalone state I/O failed")]
|
||||||
Io(#[source] io::Error),
|
Io(#[source] io::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{LeaseLiveness, ProcessObservation, classify_lease_liveness};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn lease_liveness_requires_positive_live_or_stale_evidence() {
|
||||||
|
assert_eq!(
|
||||||
|
classify_lease_liveness(Some(41), ProcessObservation::Running { start_marker: 41 }),
|
||||||
|
LeaseLiveness::Live
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_lease_liveness(Some(41), ProcessObservation::Running { start_marker: 42 }),
|
||||||
|
LeaseLiveness::Stale
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_lease_liveness(Some(41), ProcessObservation::Missing),
|
||||||
|
LeaseLiveness::Stale
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_lease_liveness(None, ProcessObservation::Running { start_marker: 41 }),
|
||||||
|
LeaseLiveness::Unknown
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_lease_liveness(Some(41), ProcessObservation::Unobservable),
|
||||||
|
LeaseLiveness::Unknown
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_lease_liveness(None, ProcessObservation::Unobservable),
|
||||||
|
LeaseLiveness::Unknown
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -426,6 +426,49 @@ async fn standalone_restore_recovers_only_a_proven_stale_lease() -> TestResult {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn standalone_restore_rejects_lease_with_missing_start_marker() -> TestResult {
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
let state_dir = temp.path().join("state");
|
||||||
|
let launch = StandaloneLaunchConfig::new(
|
||||||
|
temp.path(),
|
||||||
|
&state_dir,
|
||||||
|
manifest::ProfileSelector::Default,
|
||||||
|
"standalone-unknown-lease-test",
|
||||||
|
)
|
||||||
|
.resolve()?;
|
||||||
|
let host =
|
||||||
|
StandaloneHost::start_with_model_client(launch, ScriptedClient::new(Vec::new())).await?;
|
||||||
|
let session_id = host.session_id();
|
||||||
|
host.shutdown().await?;
|
||||||
|
let session_dir = state_dir.join(session_id.to_string());
|
||||||
|
std::fs::write(
|
||||||
|
session_dir.join("lease.json"),
|
||||||
|
serde_json::to_vec(&serde_json::json!({
|
||||||
|
"lease_id": uuid::Uuid::now_v7(),
|
||||||
|
"pid": std::process::id(),
|
||||||
|
"acquired_at_unix_ms": 1
|
||||||
|
}))?,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let store = StandaloneSessionStore::open(&state_dir)?;
|
||||||
|
assert!(matches!(
|
||||||
|
store.acquire_lease(session_id, StaleLeasePolicy::Recover),
|
||||||
|
Err(StandaloneStoreError::LeaseLivenessUnknown(id)) if id == session_id
|
||||||
|
));
|
||||||
|
let restore = StandaloneHost::restore_with_model_client(
|
||||||
|
state_dir,
|
||||||
|
session_id,
|
||||||
|
ScriptedClient::new(Vec::new()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(matches!(
|
||||||
|
restore,
|
||||||
|
Err(StandaloneStartupError::LeaseLivenessUnknown)
|
||||||
|
));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn standalone_metadata_fails_closed_on_incomplete_or_newer_records() -> TestResult {
|
async fn standalone_metadata_fails_closed_on_incomplete_or_newer_records() -> TestResult {
|
||||||
let temp = tempfile::tempdir()?;
|
let temp = tempfile::tempdir()?;
|
||||||
|
|||||||
Reference in New Issue
Block a user