diff --git a/crates/job/src/lib.rs b/crates/job/src/lib.rs index e889a413b..a675726d5 100644 --- a/crates/job/src/lib.rs +++ b/crates/job/src/lib.rs @@ -191,7 +191,6 @@ pub enum JobState { Pending, Completed, Failed, - Unknown, Cancelled, } @@ -203,7 +202,6 @@ pub enum JobAttemptState { Dispatched, Completed, Failed, - Unknown, Cancelled, } diff --git a/crates/job/tests/contract.rs b/crates/job/tests/contract.rs index 6c445a395..c83a3dd4d 100644 --- a/crates/job/tests/contract.rs +++ b/crates/job/tests/contract.rs @@ -39,7 +39,6 @@ fn shared_lifecycle_enums_round_trip_all_host_states() { (JobState::Pending, "pending"), (JobState::Completed, "completed"), (JobState::Failed, "failed"), - (JobState::Unknown, "unknown"), (JobState::Cancelled, "cancelled"), ] { assert_eq!(serde_json::to_value(state).unwrap(), json!(name)); @@ -54,7 +53,6 @@ fn shared_lifecycle_enums_round_trip_all_host_states() { (JobAttemptState::Dispatched, "dispatched"), (JobAttemptState::Completed, "completed"), (JobAttemptState::Failed, "failed"), - (JobAttemptState::Unknown, "unknown"), (JobAttemptState::Cancelled, "cancelled"), ] { assert_eq!(serde_json::to_value(state).unwrap(), json!(name)); @@ -63,6 +61,9 @@ fn shared_lifecycle_enums_round_trip_all_host_states() { state ); } + // Previous uncertain lifecycle values belong only to versioned migration input. + assert!(serde_json::from_value::(json!("unknown")).is_err()); + assert!(serde_json::from_value::(json!("unknown")).is_err()); assert!(serde_json::from_value::(json!("running")).is_err()); assert!(serde_json::from_value::(json!("idle")).is_err()); } @@ -111,13 +112,6 @@ fn outcome_is_a_neutral_read_snapshot_with_optional_result_and_failure() { Some("execution"), Some("known failure"), ), - ( - JobState::Unknown, - JobAttemptState::Unknown, - None, - Some("interrupted"), - Some("requires reconciliation"), - ), ( JobState::Cancelled, JobAttemptState::Cancelled, diff --git a/crates/standalone/src/host.rs b/crates/standalone/src/host.rs index d875aa390..b4f63236a 100644 --- a/crates/standalone/src/host.rs +++ b/crates/standalone/src/host.rs @@ -53,6 +53,7 @@ pub struct StandaloneHost { record: StandaloneWorkerRecord, lease: Option, jobs: crate::jobs::StandaloneJobs, + result_recovery: Option>>, subject: Option, subject_release: Option, } @@ -101,7 +102,7 @@ pub enum StandaloneStartupError { InitialContextPending { worker_id: WorkerId }, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum StandaloneShutdownError { #[error("standalone Job execution cleanup or terminal persistence could not be confirmed")] JobCleanup, @@ -109,6 +110,10 @@ pub enum StandaloneShutdownError { DeadlineExceeded, #[error("the standalone Worker shutdown confirmation was lost")] ConfirmationLost, + #[error("the standalone Worker Feature cleanup failed")] + FeatureCleanup { + failures: Vec, + }, #[error("the standalone Worker final state could not be committed")] StateStore, } @@ -178,7 +183,11 @@ impl StandaloneHost { .as_ref() .map(|s| s.jobs_path.clone()) .unwrap_or_else(|| store.jobs_path(worker_id)); - let jobs = match crate::jobs::StandaloneJobs::open(&jobs_path, launch.cwd.clone()) { + let jobs = match crate::jobs::StandaloneJobs::open_for_host( + &jobs_path, + launch.cwd.clone(), + allocation.lease(), + ) { Ok(jobs) => jobs, Err(_) => { cancel_unpublished_start(&store, &mut subject, allocation)?; @@ -402,8 +411,12 @@ impl StandaloneHost { .as_ref() .map(|s| s.jobs_path.clone()) .unwrap_or_else(|| store.jobs_path(worker_id)); - let jobs = crate::jobs::StandaloneJobs::open(&jobs_path, record.cwd.canonical_path.clone()) - .map_err(|_| StandaloneStartupError::StateStore)?; + let jobs = crate::jobs::StandaloneJobs::open_for_host( + &jobs_path, + record.cwd.canonical_path.clone(), + &lease, + ) + .map_err(|_| StandaloneStartupError::StateStore)?; if let Some(client) = &model_client { jobs.bind_model_client(client.clone_boxed()); } @@ -482,6 +495,13 @@ impl StandaloneHost { if let Some(connection) = &subject { connection.host.recover_jobs(); } + let reserved_result_jobs = jobs.reserved_result_job_ids(); + let result_jobs = jobs.clone(); + let result_recovery = tokio::spawn(async move { + result_jobs + .resume_reserved_result_jobs(reserved_result_jobs?) + .await + }); Self { handle: started.handle, shutdown: Some(started.shutdown), @@ -491,6 +511,7 @@ impl StandaloneHost { record, lease: Some(lease), jobs, + result_recovery: Some(result_recovery), subject, subject_release: None, } @@ -594,6 +615,7 @@ impl StandaloneHost { } pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> { + self.stop_result_recovery().await; let jobs_result = self.jobs.shutdown().await; let command = protocol::WorkerCommandEnvelope::new(u64::MAX); let _ = self.handle.send(Method::Shutdown { command }).await; @@ -601,16 +623,9 @@ impl StandaloneHost { self.retain_lease(); return Err(StandaloneShutdownError::ConfirmationLost); }; - match tokio::time::timeout(self.shutdown_timeout, shutdown).await { - Ok(Ok(())) => {} - Ok(Err(_)) => { - self.retain_lease(); - return Err(StandaloneShutdownError::ConfirmationLost); - } - Err(_) => { - self.retain_lease(); - return Err(StandaloneShutdownError::DeadlineExceeded); - } + if let Err(error) = confirm_controller_shutdown(shutdown, self.shutdown_timeout).await { + self.retain_lease(); + return Err(error); } if jobs_result.is_err() { self.retain_lease(); @@ -650,6 +665,16 @@ impl StandaloneHost { Ok(()) } + async fn stop_result_recovery(&mut self) { + self.jobs.close(); + if let Some(task) = self.result_recovery.take() { + // This joins only the dispatcher. Every admitted Worker/Host effect + // remains owned by jobs.shutdown; cancelling a waiter proves no cleanup. + task.abort(); + let _ = task.await; + } + } + fn retain_lease(&mut self) { if let Some(subject) = &mut self.subject { subject.retain(); @@ -660,9 +685,29 @@ impl StandaloneHost { } } +async fn confirm_controller_shutdown( + shutdown: worker::controller::ShutdownReceiver, + timeout: Duration, +) -> Result<(), StandaloneShutdownError> { + let report = tokio::time::timeout(timeout, shutdown) + .await + .map_err(|_| StandaloneShutdownError::DeadlineExceeded)? + .map_err(|_| StandaloneShutdownError::ConfirmationLost)?; + if report.feature_failures.is_empty() { + Ok(()) + } else { + Err(StandaloneShutdownError::FeatureCleanup { + failures: report.feature_failures, + }) + } +} + impl Drop for StandaloneHost { fn drop(&mut self) { self.jobs.close(); + if let Some(task) = self.result_recovery.take() { + task.abort(); + } // A dropped Host cannot synchronously confirm either controller or Job // cleanup. Keep its lease until explicit shutdown or process recovery, // preventing another Host from reopening a still-live execution store. @@ -931,6 +976,45 @@ mod local_input_tests { use super::*; use protocol::{AuthenticatedInputSource, Method}; + // Production report consumer over a real Controller completion channel. The report sender + // is replaced here; actual stopping-hook production is covered by Worker Job tests. + #[tokio::test] + async fn controller_report_preserves_cleanup_failure_instead_of_accepting_scope_return() { + let failure = worker::hook::HookError::new( + worker::hook::HookErrorCategory::Dependency, + "retained Host cleanup", + ); + let (sender, receiver) = tokio::sync::oneshot::channel(); + sender + .send(worker::controller::ControllerShutdownReport { + feature_failures: vec![failure.clone()], + }) + .unwrap(); + assert_eq!( + confirm_controller_shutdown(receiver, Duration::from_secs(1)).await, + Err(StandaloneShutdownError::FeatureCleanup { + failures: vec![failure] + }) + ); + + let (sender, receiver) = tokio::sync::oneshot::channel(); + sender + .send(worker::controller::ControllerShutdownReport::default()) + .unwrap(); + assert_eq!( + confirm_controller_shutdown(receiver, Duration::from_secs(1)).await, + Ok(()) + ); + + let (sender, receiver) = + tokio::sync::oneshot::channel::(); + drop(sender); + assert_eq!( + confirm_controller_shutdown(receiver, Duration::from_secs(1)).await, + Err(StandaloneShutdownError::ConfirmationLost) + ); + } + #[test] fn local_transport_stamps_operator_without_backend_account() { let source = AuthenticatedInputSource::Account { diff --git a/crates/standalone/src/job_store.rs b/crates/standalone/src/job_store.rs index dd69c1568..30da5b2fe 100644 --- a/crates/standalone/src/job_store.rs +++ b/crates/standalone/src/job_store.rs @@ -6,9 +6,10 @@ //! Job state is not stored in Memory or interactive Sessions. //! //! Identical intent under the same ID returns the current snapshot; changes conflict. -//! Only known failures can create new attempts through explicit retry. Reopening preserves -//! Reserved and makes unaccepted Dispatched attempts Unknown. Unknown does not prove no effects; -//! cancellation does not erase that attempt's uncertainty or serialization fence. +//! Version cutover fails only old unfinished work and permanently forbids old Job continuation. +//! Ordinary reopen preserves execution bindings: neither a terminal outcome nor process absence +//! releases serialization/capacity. Retry requires independent owned shutdown evidence; granted +//! Jobs also require authoritative domain reconciliation of the original input and operations. //! //! Results commit before releasing the Internal Worker. Replaying the same result is idempotent; //! different results and stale attempts are rejected. Later cancellation/execution failure does @@ -25,7 +26,10 @@ use thiserror::Error; #[path = "job_store/legacy_migrations.rs"] mod legacy_migrations; -const SCHEMA_VERSION: i64 = 5; +#[path = "job_store/recovery.rs"] +mod recovery; + +const SCHEMA_VERSION: i64 = 6; /// Availability of the immutable input body, independent of execution outcome. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -72,7 +76,6 @@ pub enum JobState { Pending, Completed, Failed, - Unknown, Cancelled, } @@ -83,7 +86,6 @@ pub enum JobAttemptState { Dispatched, Completed, Failed, - Unknown, Cancelled, } @@ -98,7 +100,6 @@ impl JobAttemptState { Self::Dispatched => "dispatched", Self::Completed => "completed", Self::Failed => "failed", - Self::Unknown => "unknown", Self::Cancelled => "cancelled", } } @@ -117,6 +118,40 @@ pub struct JobSnapshot { pub attempt: JobAttempt, pub result: Option, pub acknowledged: bool, + /// A version cutover is final even when input/checkpoint remains incomplete. + pub version_stopped: bool, + /// All historical execution owners, independent of Job outcome/checkpoint. + pub executions: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct JobExecutionOwnership { + pub attempt_id: String, + pub owner_id: String, + pub run_id: String, + pub writes_revoked: bool, + pub cleanup_receipt: Option, +} + +impl JobSnapshot { + /// Terminal failure/completion without the attempt's admission binding is not + /// an empty set of successfully joined executions. Keep that missing evidence + /// distinct from an unstarted reservation and from checkpoint coverage. + pub fn executions_released(&self) -> bool { + let needs_binding = matches!( + self.attempt.state, + JobAttemptState::Dispatched | JobAttemptState::Completed | JobAttemptState::Failed + ); + (!needs_binding + || self + .executions + .iter() + .any(|execution| execution.attempt_id == self.attempt.attempt_id)) + && self + .executions + .iter() + .all(|e| e.writes_revoked && e.cleanup_receipt.is_some()) + } } impl JobSnapshot { @@ -147,7 +182,6 @@ impl JobSnapshot { JobState::Pending => job::JobState::Pending, JobState::Completed => job::JobState::Completed, JobState::Failed => job::JobState::Failed, - JobState::Unknown => job::JobState::Unknown, JobState::Cancelled => job::JobState::Cancelled, }, attempt_state: match self.attempt.state { @@ -155,14 +189,12 @@ impl JobSnapshot { JobAttemptState::Dispatched => job::JobAttemptState::Dispatched, JobAttemptState::Completed => job::JobAttemptState::Completed, JobAttemptState::Failed => job::JobAttemptState::Failed, - JobAttemptState::Unknown => job::JobAttemptState::Unknown, JobAttemptState::Cancelled => job::JobAttemptState::Cancelled, }, result: self.result.clone(), failure_category: self.attempt.failure.as_ref().map(|_| { match self.attempt.state { JobAttemptState::Cancelled => "cancelled", - JobAttemptState::Unknown => "unknown_outcome", _ => "execution_failed", } .to_string() @@ -216,6 +248,7 @@ pub enum JobStoreError { pub(crate) struct JobStore { conn: Connection, + owner_id: String, } impl JobStore { @@ -239,61 +272,12 @@ impl JobStore { } let mut conn = Connection::open(path)?; feature_storage::configure_connection(&conn)?; - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - let version: i64 = tx.pragma_query_value(None, "user_version", |row| row.get(0))?; - match version { - 0 => { - tx.execute_batch(SCHEMA)?; - tx.execute_batch(GRANT_SCHEMA)?; - tx.execute_batch(RECONCILIATION_SCHEMA)?; - tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; - } - 1 | 2 => { - if version == 1 { - tx.execute_batch(GRANT_SCHEMA)?; - } - legacy_migrations::migrate_legacy_jobs(&tx)?; - tx.execute_batch(RECONCILIATION_SCHEMA)?; - tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; - } - 3 => { - tx.execute_batch(RECONCILIATION_SCHEMA)?; - tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; - } - 4 | SCHEMA_VERSION => {} - other => return Err(JobStoreError::UnsupportedSchema(other)), - } - if version < 5 { - tx.execute_batch(INPUT_RETENTION_SCHEMA)?; - let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?; - if has_legacy_archive { - tx.execute_batch("DROP TRIGGER legacy_job_archive_immutable; - CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive - BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END; - CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive - WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available') - OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE - BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;")?; - } - tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; - } - // An accepted result is the only evidence of success. Reserved attempts - // were never started and remain pending; cancelled attempts never resume. - tx.execute_batch( - "UPDATE job_attempts SET state = 'completed', failure = NULL - WHERE state = 'dispatched' AND result_json IS NOT NULL; - UPDATE job_attempts SET state = 'unknown', - failure = 'Host interrupted before an accepted result' - WHERE state = 'dispatched' AND result_json IS NULL; - UPDATE job_intents SET state = ( - SELECT state FROM job_attempts - WHERE job_id = job_intents.job_id AND number = current_attempt) - WHERE state = 'pending' AND EXISTS ( - SELECT 1 FROM job_attempts WHERE job_id = job_intents.job_id - AND number = current_attempt AND state IN ('completed', 'unknown'));", - )?; - tx.commit()?; - Ok(Self { conn }) + recovery::initialize(&mut conn)?; + // Opening never infers shutdown from a missing task or a process exit. + Ok(Self { + conn, + owner_id: uuid::Uuid::now_v7().to_string(), + }) } pub(crate) fn reserve(&mut self, request: JobRequest) -> Result { @@ -351,9 +335,9 @@ impl JobStore { Ok(snapshot) } - /// Domain Host attests that this fenced attempt's only effects are reconciled in its - /// authoritative durable consumer. This releases serialization, not the original Unknown - /// history, and cannot authorize retry or change its result. The service checks cleanup. + /// Domain Host attests that this fenced attempt's effects are reconciled in its + /// authoritative durable consumer. Preserve the original attempt/cause; this receipt + /// neither proves execution shutdown nor permits an old-version Job to continue. pub(crate) fn reconcile_granted_attempt( &mut self, job_id: &str, @@ -374,11 +358,16 @@ impl JobStore { let snapshot = require_snapshot(&tx, job_id)?; check_attempt(&snapshot, attempt_id)?; if snapshot.domain_grant.is_none() - || snapshot.attempt.state != JobAttemptState::Unknown - || !matches!(snapshot.state, JobState::Unknown | JobState::Cancelled) + || snapshot.attempt.state != JobAttemptState::Failed + || !matches!(snapshot.state, JobState::Failed | JobState::Cancelled) { return Err(JobStoreError::InvalidTransition( - "domain reconciliation requires a fenced unknown granted attempt", + "domain reconciliation requires a failed granted attempt", + )); + } + if !snapshot.executions_released() { + return Err(JobStoreError::InvalidTransition( + "execution cleanup is not proven", )); } let old: Option = tx.query_row("SELECT domain_receipt FROM job_domain_reconciliations WHERE job_id=?1 AND attempt_id=?2", @@ -394,14 +383,24 @@ impl JobStore { Ok(()) } - /// Only definitive failure can create a new immutable attempt. Unknown - /// outcomes require caller reconciliation, never an automatic replay. + /// Only result-only work with independent shutdown proof can use generic retry. + /// Side-effecting domain work must reconcile its authoritative operation receipts. pub(crate) fn retry(&mut self, job_id: &str) -> Result { let tx = self .conn .transaction_with_behavior(TransactionBehavior::Immediate)?; let previous = require_snapshot(&tx, job_id)?; previous.load_executable_request()?; + if previous.version_stopped || !previous.executions_released() { + return Err(JobStoreError::InvalidTransition( + "version stop or unreleased execution forbids retry", + )); + } + if previous.domain_grant.is_some() { + return Err(JobStoreError::InvalidTransition( + "domain Jobs require same-Job reconciliation, not generic retry", + )); + } if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed { return Err(JobStoreError::InvalidTransition( "retry requires known failure", @@ -463,7 +462,8 @@ impl JobStore { return Ok(snapshot); } if reason == JobInputRetirementReason::Completed - && (snapshot.state != JobState::Completed + && (!snapshot.executions_released() + || snapshot.state != JobState::Completed || snapshot.attempt.state != JobAttemptState::Completed || snapshot.result.is_none()) { @@ -497,6 +497,164 @@ impl JobStore { Ok(snapshot) } + /// Settlement belongs to the current attempt and the exact Host connection + /// that dispatched it. Reopen/map absence cannot transfer that authority. + pub(crate) fn check_execution_binding( + &self, + job_id: &str, + attempt_id: &str, + input_digest: &str, + execution: &JobExecutionOwnership, + ) -> Result<(), JobStoreError> { + let saved = self.get(job_id)?; + check_attempt(&saved, attempt_id)?; + if saved.input_digest != input_digest { + return Err(JobStoreError::DigestMismatch); + } + if saved.version_stopped + || execution.attempt_id != attempt_id + || execution.owner_id != self.owner_id + || !saved.executions.iter().any(|bound| { + bound.attempt_id == attempt_id + && bound.owner_id == execution.owner_id + && bound.run_id == execution.run_id + }) + { + return Err(JobStoreError::AttemptMismatch); + } + Ok(()) + } + + /// Only a bound runner proof can discharge ownership. Terminal state, process + /// absence, or a successful result do not call this method implicitly. + pub(crate) fn confirm_execution_cleanup( + &mut self, + job_id: &str, + execution: &JobExecutionOwnership, + receipt: &str, + ) -> Result<(), JobStoreError> { + if execution.owner_id != self.owner_id + || receipt.is_empty() + || receipt.len() > 256 + || receipt.chars().any(char::is_control) + { + return Err(JobStoreError::AttemptMismatch); + } + let changed = self.conn.execute("UPDATE job_execution_ownership SET writes_revoked=1,cleanup_receipt=?5 WHERE job_id=?1 AND attempt_id=?2 AND owner_id=?3 AND run_id=?4", + params![job_id,execution.attempt_id,execution.owner_id,execution.run_id,receipt])?; + if changed != 1 { + return Err(JobStoreError::AttemptMismatch); + } + Ok(()) + } + + pub(crate) fn execution_obligations( + &self, + ) -> Result, JobStoreError> { + // A failed/dispatched row without its admission binding cannot disappear + // from the shutdown barrier merely because there is no ownership row. + let missing: Option = self.conn.query_row( + "SELECT a.attempt_id FROM job_intents i JOIN job_attempts a + ON a.job_id=i.job_id AND a.number=i.current_attempt + WHERE a.state IN ('dispatched','failed','completed') + AND NOT EXISTS(SELECT 1 FROM job_execution_ownership e WHERE e.attempt_id=a.attempt_id) + AND NOT EXISTS(SELECT 1 FROM job_version_stops s WHERE s.job_id=i.job_id) + LIMIT 1", [], |row|row.get(0)).optional()?; + if let Some(attempt) = missing { + return Err(JobStoreError::Corrupt(format!( + "execution admission/cleanup evidence missing for attempt {attempt}" + ))); + } + let mut stmt = self.conn.prepare("SELECT attempt_id,owner_id,run_id,writes_revoked,cleanup_receipt FROM job_execution_ownership WHERE cleanup_receipt IS NULL OR writes_revoked=0 ORDER BY rowid")?; + Ok(stmt + .query_map([], |r| { + Ok(JobExecutionOwnership { + attempt_id: r.get(0)?, + owner_id: r.get(1)?, + run_id: r.get(2)?, + writes_revoked: r.get(3)?, + cleanup_receipt: r.get(4)?, + }) + })? + .collect::, _>>()?) + } + + /// Same immutable Journal input, bounded new attempt, never a replacement Job. + /// The caller has reconciled/closed the old generation's saved operations. + pub(crate) fn resume_granted( + &mut self, + job_id: &str, + attempt_id: &str, + ) -> Result { + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let saved = require_snapshot(&tx, job_id)?; + check_attempt(&saved, attempt_id)?; + saved.load_executable_request()?; + let reconciled: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM job_domain_reconciliations WHERE job_id=?1 AND attempt_id=?2)", params![job_id,attempt_id], |row|row.get(0))?; + if saved.version_stopped + || saved.domain_grant.is_none() + || !saved.executions_released() + || !reconciled + || saved.state != JobState::Failed + || saved.attempt.state != JobAttemptState::Failed + || saved.attempt.number >= saved.request.limits.max_attempts + { + return Err(JobStoreError::InvalidTransition( + "same-Job recovery requires reconciled execution and available attempt budget", + )); + } + let next = saved.attempt.number + 1; + insert_attempt(&tx, &saved.request, next)?; + tx.execute( + "UPDATE job_intents SET state='pending',current_attempt=?2 WHERE job_id=?1", + params![job_id, next], + )?; + let snapshot = require_snapshot(&tx, job_id)?; + tx.commit()?; + Ok(snapshot) + } + + /// A domain's saved complete checkpoint may settle the current interrupted + /// attempt without invoking the model or claiming that execution has ended. + pub(crate) fn accept_reconciled( + &mut self, + submission: &JobResultSubmission, + ) -> Result { + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let saved = require_snapshot(&tx, &submission.job_id)?; + check_attempt(&saved, &submission.attempt_id)?; + if saved.version_stopped + || saved.domain_grant.is_none() + || saved.state == JobState::Cancelled + { + return Err(JobStoreError::InvalidTransition( + "domain completion is unavailable for this Job", + )); + } + if saved.input_digest != submission.input_digest { + return Err(JobStoreError::DigestMismatch); + } + if saved.state == JobState::Completed { + drop(tx); + return self.accept(submission); + } + saved.load_executable_request()?; + let (digest, encoded) = + job::result_digest(&submission.result, saved.request.limits.max_result_bytes)?; + tx.execute("UPDATE job_attempts SET state='completed',result_json=?3,result_digest=?4 WHERE job_id=?1 AND attempt_id=?2",params![submission.job_id,submission.attempt_id,encoded,digest])?; + tx.execute( + "UPDATE job_intents SET state='completed' WHERE job_id=?1", + [&submission.job_id], + )?; + let snapshot = require_snapshot(&tx, &submission.job_id)?; + tx.commit()?; + Ok(snapshot) + } + pub(crate) fn get(&self, job_id: &str) -> Result { require_snapshot(&self.conn, job_id) } @@ -511,8 +669,8 @@ impl JobStore { .collect() } - pub(crate) fn unknown_granted(&self) -> Result, JobStoreError> { - let mut statement = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_attempts a ON a.job_id=i.job_id AND a.number=i.current_attempt JOIN job_domain_grants g ON g.job_id=i.job_id WHERE a.state='unknown' AND NOT EXISTS (SELECT 1 FROM job_domain_reconciliations r WHERE r.job_id=a.job_id AND r.attempt_id=a.attempt_id) ORDER BY i.rowid")?; + pub(crate) fn unreleased_granted(&self) -> Result, JobStoreError> { + let mut statement = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_attempts a ON a.job_id=i.job_id AND a.number=i.current_attempt JOIN job_domain_grants g ON g.job_id=i.job_id WHERE (EXISTS(SELECT 1 FROM job_execution_ownership e WHERE e.job_id=i.job_id AND e.cleanup_receipt IS NULL) OR (a.state='failed' AND NOT EXISTS (SELECT 1 FROM job_domain_reconciliations r WHERE r.job_id=a.job_id AND r.attempt_id=a.attempt_id))) ORDER BY i.rowid")?; let ids = statement .query_map([], |row| row.get::<_, String>(0))? .collect::>>()?; @@ -526,7 +684,7 @@ impl JobStore { let mut statement = self.conn.prepare( "SELECT i.job_id FROM job_intents i JOIN job_attempts a ON a.job_id = i.job_id AND a.number = i.current_attempt - WHERE i.state = 'pending' AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid", + WHERE i.state = 'pending' AND NOT EXISTS (SELECT 1 FROM job_version_stops s WHERE s.job_id=i.job_id) AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid", )?; let ids = statement .query_map([], |row| row.get::<_, String>(0))? @@ -550,7 +708,9 @@ impl JobStore { let snapshot = require_snapshot(&tx, job_id)?; check_attempt(&snapshot, attempt_id)?; snapshot.load_executable_request()?; - if snapshot.state != JobState::Pending + if snapshot.version_stopped + || !snapshot.executions_released() + || snapshot.state != JobState::Pending || snapshot.attempt.state != JobAttemptState::Reserved { return Err(JobStoreError::InvalidTransition( @@ -559,9 +719,9 @@ impl JobStore { } check_serialization(&tx, snapshot.request.serialization_key.as_deref(), job_id)?; let (active, active_limit): (i64, Option) = tx.query_row( - "SELECT COUNT(*), MIN(i.max_concurrent) FROM job_attempts a JOIN job_intents i - ON i.job_id = a.job_id AND i.current_attempt = a.number - WHERE a.state = 'dispatched'", + "SELECT COUNT(*), MIN(i.max_concurrent) FROM job_execution_ownership e JOIN job_intents i + ON i.job_id = e.job_id + WHERE e.writes_revoked=0 OR e.cleanup_receipt IS NULL", [], |row| Ok((row.get(0)?, row.get(1)?)), )?; @@ -577,6 +737,8 @@ impl JobStore { "UPDATE job_attempts SET state = 'dispatched' WHERE job_id = ?1 AND attempt_id = ?2", params![job_id, attempt_id], )?; + tx.execute("INSERT INTO job_execution_ownership(job_id,attempt_id,owner_id,run_id) VALUES(?1,?2,?3,?4)", + params![job_id,attempt_id,self.owner_id,worker::job::execution_run_id(&snapshot.request, attempt_id)?.to_string()])?; let snapshot = require_snapshot(&tx, job_id)?; tx.commit()?; Ok(snapshot) @@ -613,9 +775,14 @@ impl JobStore { snapshot.load_executable_request()?; if snapshot.state != JobState::Pending || snapshot.attempt.state != JobAttemptState::Dispatched + || !snapshot.executions.iter().any(|execution| { + execution.attempt_id == bound.attempt_id + && execution.owner_id == self.owner_id + && !execution.writes_revoked + }) { return Err(JobStoreError::InvalidTransition( - "result requires the active dispatched attempt", + "result requires the active dispatched attempt owned by this Host", )); } tx.execute( @@ -670,20 +837,17 @@ impl JobStore { Ok(snapshot) } - /// Cancellation is durable. Cancelling unknown intent must not erase the - /// attempt's uncertain execution evidence or release its serialization fence. - /// A previously accepted result wins and remains success. + /// Cancellation preserves execution ownership and the serialization fence until + /// independent cleanup proof exists. A previously accepted result remains success. pub(crate) fn cancel(&mut self, job_id: &str) -> Result { let tx = self .conn .transaction_with_behavior(TransactionBehavior::Immediate)?; let snapshot = require_snapshot(&tx, job_id)?; - if snapshot.state == JobState::Unknown { - tx.execute( - "UPDATE job_intents SET state = 'cancelled' WHERE job_id = ?1", - [job_id], - )?; - } else if !matches!(snapshot.state, JobState::Completed | JobState::Cancelled) { + if !matches!( + snapshot.state, + JobState::Completed | JobState::Cancelled | JobState::Failed + ) { set_terminal( &tx, &snapshot, @@ -732,9 +896,11 @@ fn check_serialization( let conflicting: bool = conn.query_row( "SELECT EXISTS(SELECT 1 FROM job_intents i WHERE serialization_key = ?1 AND job_id <> ?2 AND (state = 'pending' OR EXISTS - (SELECT 1 FROM job_attempts a WHERE a.job_id = i.job_id AND a.state = 'unknown' - AND NOT EXISTS (SELECT 1 FROM job_domain_reconciliations r - WHERE r.job_id=a.job_id AND r.attempt_id=a.attempt_id))))", + (SELECT 1 FROM job_execution_ownership e WHERE e.job_id=i.job_id + AND (e.writes_revoked=0 OR e.cleanup_receipt IS NULL)) + OR (state='failed' AND EXISTS(SELECT 1 FROM job_domain_grants g WHERE g.job_id=i.job_id) + AND NOT EXISTS(SELECT 1 FROM job_domain_reconciliations r JOIN job_attempts a ON a.attempt_id=r.attempt_id + WHERE r.job_id=i.job_id AND a.number=i.current_attempt))))", params![key, job_id], |row| row.get(0), )?; @@ -832,7 +998,6 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, "pending" => JobState::Pending, "completed" => JobState::Completed, "failed" => JobState::Failed, - "unknown" => JobState::Unknown, "cancelled" => JobState::Cancelled, _ => { return Err(JobStoreError::Corrupt(format!( @@ -845,7 +1010,6 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, "dispatched" => JobAttemptState::Dispatched, "completed" => JobAttemptState::Completed, "failed" => JobAttemptState::Failed, - "unknown" => JobAttemptState::Unknown, "cancelled" => JobAttemptState::Cancelled, _ => { return Err(JobStoreError::Corrupt(format!( @@ -875,6 +1039,11 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, .transpose()?, state, acknowledged, + version_stopped: conn.query_row("SELECT EXISTS(SELECT 1 FROM job_version_stops WHERE job_id=?1)", [job_id], |r|r.get(0))?, + executions: { + let mut stmt = conn.prepare("SELECT attempt_id,owner_id,run_id,writes_revoked,cleanup_receipt FROM job_execution_ownership WHERE job_id=?1 ORDER BY rowid")?; + stmt.query_map([job_id], |r| Ok(JobExecutionOwnership { attempt_id:r.get(0)?,owner_id:r.get(1)?,run_id:r.get(2)?,writes_revoked:r.get(3)?,cleanup_receipt:r.get(4)? }))?.collect::,_>>()? + }, attempt: JobAttempt { attempt_id, number, @@ -953,8 +1122,10 @@ CREATE TABLE job_intents ( fingerprint TEXT NOT NULL, serialization_key TEXT, max_concurrent INTEGER NOT NULL CHECK (max_concurrent BETWEEN 1 AND 65535), - state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown', 'cancelled')), + state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'cancelled')), current_attempt INTEGER NOT NULL CHECK (current_attempt BETWEEN 1 AND 255), + input_disposition TEXT NOT NULL DEFAULT 'available' + CHECK (input_disposition IN ('available','reclaimed','erased','invalidated')), acknowledged INTEGER NOT NULL DEFAULT 0 CHECK (acknowledged IN (0, 1)), CHECK (acknowledged = 0 OR state = 'completed'), FOREIGN KEY (job_id, current_attempt) REFERENCES job_attempts(job_id, number) @@ -965,7 +1136,7 @@ CREATE TABLE job_attempts ( number INTEGER NOT NULL CHECK (number BETWEEN 1 AND 255), attempt_id TEXT NOT NULL UNIQUE, input_digest TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatched', 'completed', 'failed', 'unknown', 'cancelled')), + state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatched', 'completed', 'failed', 'cancelled')), result_json TEXT, result_digest TEXT, failure TEXT, @@ -974,10 +1145,10 @@ CREATE TABLE job_attempts ( CHECK (state <> 'completed' OR result_json IS NOT NULL) ); CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) - WHERE serialization_key IS NOT NULL AND state IN ('pending', 'unknown'); + WHERE serialization_key IS NOT NULL AND state = 'pending'; CREATE INDEX job_attempt_state ON job_attempts(state); CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF - job_id, request_json, fingerprint, serialization_key, max_concurrent ON job_intents + job_id, fingerprint, serialization_key, max_concurrent ON job_intents BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END; CREATE TRIGGER job_attempt_immutable BEFORE UPDATE OF job_id, number, attempt_id, input_digest ON job_attempts @@ -986,6 +1157,14 @@ CREATE TRIGGER job_result_immutable BEFORE UPDATE OF result_json, result_digest WHEN OLD.result_json IS NOT NULL AND (NEW.result_json IS NOT OLD.result_json OR NEW.result_digest IS NOT OLD.result_digest) BEGIN SELECT RAISE(ABORT, 'immutable accepted Job result'); END; +CREATE TRIGGER job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON job_intents +WHEN ( + ((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated')) + OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased')) + AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') + AND json_type(NEW.request_json,'$.input') = 'null' +) IS NOT TRUE +BEGIN SELECT RAISE(ABORT, 'immutable Job input retirement'); END; "; #[cfg(test)] @@ -996,7 +1175,7 @@ mod tests { use std::sync::{Arc, Barrier}; use tempfile::TempDir; - fn request(id: &str) -> JobRequest { + pub(super) fn request(id: &str) -> JobRequest { JobRequest { job_id: id.into(), purpose: "fixture".into(), @@ -1024,7 +1203,7 @@ mod tests { .finish( "first", &reserved.attempt.attempt_id, - JobAttemptState::Unknown, + JobAttemptState::Failed, Some("response lost"), ) .unwrap(); @@ -1039,6 +1218,13 @@ mod tests { .reconcile_granted_attempt("first", "foreign-attempt", "domain-fenced") .is_err() ); + assert!( + store + .reconcile_granted_attempt("first", &unknown.attempt.attempt_id, "domain-fenced") + .is_err() + ); + record_runner_cleanup(&mut store, "first"); + let unknown = store.get("first").unwrap(); store .reconcile_granted_attempt("first", &unknown.attempt.attempt_id, "domain-fenced") .unwrap(); @@ -1056,7 +1242,7 @@ mod tests { drop(store); let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); assert_eq!(reopened.get("first").unwrap(), unknown); - assert!(reopened.unknown_granted().unwrap().is_empty()); + assert!(reopened.unreleased_granted().unwrap().is_empty()); assert!( reopened .conn @@ -1067,42 +1253,53 @@ mod tests { } #[test] - fn schema_three_migration_preserves_unknown_grant_and_its_serialization_fence() { - let (dir, mut store) = fixture(); + fn schema_three_cutover_preserves_grant_history_and_execution_fence() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut conn = Connection::open(&path).unwrap(); + feature_storage::configure_connection(&conn).unwrap(); + let tx = conn.transaction().unwrap(); + tx.execute_batch(legacy_migrations::SCHEMA).unwrap(); + tx.execute_batch(GRANT_SCHEMA).unwrap(); let mut first = request("first"); first.serialization_key = Some("journal".into()); - let reserved = store - .reserve_granted(first, Some(json!({"domain":"journal"}))) - .unwrap(); - store - .dispatch("first", &reserved.attempt.attempt_id, 8) - .unwrap(); - let unknown = store - .finish( - "first", - &reserved.attempt.attempt_id, - JobAttemptState::Unknown, - Some("unconfirmed"), - ) - .unwrap(); - store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable; - ALTER TABLE job_intents DROP COLUMN input_disposition; - CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents - BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END; - DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap(); - drop(store); - let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); - assert_eq!(reopened.get("first").unwrap(), unknown); + tx.execute("INSERT INTO job_intents(job_id,request_json,fingerprint,serialization_key,max_concurrent,state,current_attempt) VALUES('first',?1,?2,'journal',8,'unknown',1)", params![serde_json::to_string(&first).unwrap(),first.fingerprint().unwrap()]).unwrap(); + tx.execute("INSERT INTO job_attempts(job_id,number,attempt_id,input_digest,state,failure) VALUES('first',1,?1,?2,'unknown','unconfirmed')", params![job::attempt_id("first",1),first.input_digest().unwrap()]).unwrap(); + tx.execute( + "INSERT INTO job_domain_grants VALUES('first',?1)", + [serde_json::to_string(&json!({"domain":"journal"})).unwrap()], + ) + .unwrap(); + tx.pragma_update(None, "user_version", 3).unwrap(); + tx.commit().unwrap(); + drop(conn); + let mut reopened = JobStore::open(&path).unwrap(); + let saved = reopened.get("first").unwrap(); + assert_eq!(saved.state, JobState::Failed); + assert_eq!( + saved.attempt.failure.as_deref(), + Some(recovery::UPDATE_INTERRUPTION) + ); + assert_eq!(saved.domain_grant, Some(json!({"domain":"journal"}))); + assert!(saved.version_stopped); + assert!(!saved.executions_released()); let mut next = request("next"); next.serialization_key = Some("journal".into()); assert!(matches!( - reopened.reserve(next.clone()), + reopened.reserve(next), Err(JobStoreError::SerializationConflict(_)) )); - reopened - .reconcile_granted_attempt("first", &unknown.attempt.attempt_id, "domain-fenced") - .unwrap(); - reopened.reserve(next).unwrap(); + assert!( + reopened + .reconcile_granted_attempt("first", &saved.attempt.attempt_id, "not-a-stop-proof") + .is_err() + ); + assert!( + reopened + .resume_granted("first", &saved.attempt.attempt_id) + .is_err() + ); + assert!(reopened.retry("first").is_err()); } fn fixture() -> (TempDir, JobStore) { @@ -1120,6 +1317,15 @@ mod tests { } } + // Store-level fixture only: the runner/gate proof is exercised by Jobs tests. + fn record_runner_cleanup(store: &mut JobStore, id: &str) { + for execution in store.get(id).unwrap().executions { + store + .confirm_execution_cleanup(id, &execution, "fixture-owned-run-joined") + .unwrap(); + } + } + fn dispatched(store: &mut JobStore, request: JobRequest) -> JobSnapshot { let reserved = store.reserve(request).unwrap(); store @@ -1139,7 +1345,9 @@ mod tests { .dispatch("retained", &reserved.attempt.attempt_id, 8) .unwrap(); let result = submission(&running, json!({"saved":true})); - let completed = store.accept(&result).unwrap(); + store.accept(&result).unwrap(); + record_runner_cleanup(&mut store, "retained"); + let completed = store.get("retained").unwrap(); store.ack("retained").unwrap(); let retired = store .reclaim_input( @@ -1219,7 +1427,7 @@ mod tests { let (_dir, mut store) = fixture(); for (id, state) in [ ("pending", None), - ("unknown", Some(JobAttemptState::Unknown)), + ("unknown", Some(JobAttemptState::Failed)), ("failed", Some(JobAttemptState::Failed)), ] { let original = request(id); @@ -1285,7 +1493,8 @@ mod tests { drop(store); let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); let recovered = reopened.get("erase-running").unwrap(); - assert_eq!(recovered.state, JobState::Unknown); + assert_eq!(recovered, erased); + assert!(!recovered.executions_released()); assert_eq!(recovered.input_disposition, JobInputDisposition::Erased); assert_eq!(recovered.input_digest, running.input_digest); assert!(reopened.accept(&result).is_err()); @@ -1588,6 +1797,8 @@ mod tests { ) .unwrap(); assert_eq!(store.reserve(original.clone()).unwrap(), failed); + assert!(store.retry("retry").is_err()); + record_runner_cleanup(&mut store, "retry"); let second = store.retry("retry").unwrap(); assert_eq!(second.request, original); assert_eq!(second.attempt.number, 2); @@ -1631,6 +1842,7 @@ mod tests { None, ) .unwrap(); + record_runner_cleanup(&mut store, "retry"); assert!(matches!( store.retry("retry"), Err(JobStoreError::InvalidTransition("attempt limit reached")) @@ -1736,7 +1948,6 @@ mod tests { .unwrap(); for state in [ JobAttemptState::Failed, - JobAttemptState::Unknown, JobAttemptState::Cancelled, JobAttemptState::Completed, ] { @@ -1803,10 +2014,11 @@ mod tests { assert_eq!(reopened.pending().unwrap(), vec![pending.clone()]); assert_eq!(reopened.get("pending").unwrap(), pending); let unknown = reopened.get("interrupted").unwrap(); - assert_eq!(unknown.state, JobState::Unknown); - assert_eq!(unknown.attempt.state, JobAttemptState::Unknown); + assert_eq!(unknown, active); + assert_eq!(unknown.state, JobState::Pending); + assert_eq!(unknown.attempt.state, JobAttemptState::Dispatched); assert_eq!(unknown.result, None); - assert!(unknown.attempt.failure.is_some()); + assert!(!unknown.executions_released()); assert_eq!(reopened.reserve(request("interrupted")).unwrap(), unknown); assert!(reopened.retry("interrupted").is_err()); assert!( @@ -1840,10 +2052,11 @@ mod tests { drop(store); let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); let retained = reopened.get("accepted-fixture").unwrap(); - assert_eq!(retained.state, JobState::Completed); - assert_eq!(retained.attempt.state, JobAttemptState::Completed); + assert_eq!(retained.state, JobState::Pending); + assert_eq!(retained.attempt.state, JobAttemptState::Dispatched); assert_eq!(retained.result, Some(bound.result.clone())); - assert_eq!(reopened.accept(&bound).unwrap(), retained); + assert!(!retained.executions_released()); + assert!(reopened.accept(&bound).is_err()); assert!(reopened.pending().unwrap().is_empty()); } @@ -1911,7 +2124,8 @@ mod tests { )); let cancelled_unknown = reopened.cancel("first").unwrap(); assert_eq!(cancelled_unknown.state, JobState::Cancelled); - assert_eq!(cancelled_unknown.attempt.state, JobAttemptState::Unknown); + assert_eq!(cancelled_unknown.attempt.state, JobAttemptState::Cancelled); + assert!(!cancelled_unknown.executions_released()); assert!(matches!( reopened.reserve(second.clone()), Err(JobStoreError::SerializationConflict(_)) @@ -1971,6 +2185,11 @@ mod tests { None, ) .unwrap(); + assert!(matches!( + store.dispatch("other", &other.attempt.attempt_id, 8), + Err(JobStoreError::ConcurrentLimit) + )); + record_runner_cleanup(&mut store, "strict"); store .dispatch("other", &other.attempt.attempt_id, 8) .unwrap(); @@ -1989,6 +2208,11 @@ mod tests { JobAttemptState::Reserved ); store.accept(&submission(&other, json!({}))).unwrap(); + assert!(matches!( + store.dispatch("host-bound", &host_bound.attempt.attempt_id, 1), + Err(JobStoreError::ConcurrentLimit) + )); + record_runner_cleanup(&mut store, "other"); store .dispatch("host-bound", &host_bound.attempt.attempt_id, 1) .unwrap(); @@ -2105,7 +2329,7 @@ mod tests { .finish( "failure", &reserved.attempt.attempt_id, - JobAttemptState::Unknown, + JobAttemptState::Cancelled, None ) .is_err() @@ -2122,8 +2346,8 @@ mod tests { ); assert!(store.accept(&submission(&reserved, json!({}))).is_err()); assert!(store.ack("failure").is_err()); - assert_eq!(store.cancel("failure").unwrap().state, JobState::Cancelled); - assert!(store.retry("failure").is_err()); + assert_eq!(store.cancel("failure").unwrap(), failed); + assert_eq!(store.retry("failure").unwrap().attempt.number, 2); } #[test] @@ -2176,6 +2400,8 @@ mod tests { Some("known failure"), ) .unwrap(); + record_runner_cleanup(&mut store, "atomic"); + let failed = store.get("atomic").unwrap(); store .conn .execute_batch( diff --git a/crates/standalone/src/job_store/legacy_migrations.rs b/crates/standalone/src/job_store/legacy_migrations.rs index f1b8cb539..00588470e 100644 --- a/crates/standalone/src/job_store/legacy_migrations.rs +++ b/crates/standalone/src/job_store/legacy_migrations.rs @@ -1,5 +1,48 @@ use super::*; +// Frozen pre-cutover schema, including the retired persisted state spelling. +pub(super) const SCHEMA: &str = " +CREATE TABLE job_intents ( + job_id TEXT PRIMARY KEY NOT NULL, + request_json TEXT NOT NULL, + fingerprint TEXT NOT NULL, + serialization_key TEXT, + max_concurrent INTEGER NOT NULL CHECK (max_concurrent BETWEEN 1 AND 65535), + state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown', 'cancelled')), + current_attempt INTEGER NOT NULL CHECK (current_attempt BETWEEN 1 AND 255), + acknowledged INTEGER NOT NULL DEFAULT 0 CHECK (acknowledged IN (0, 1)), + CHECK (acknowledged = 0 OR state = 'completed'), + FOREIGN KEY (job_id, current_attempt) REFERENCES job_attempts(job_id, number) + DEFERRABLE INITIALLY DEFERRED +); +CREATE TABLE job_attempts ( + job_id TEXT NOT NULL REFERENCES job_intents(job_id), + number INTEGER NOT NULL CHECK (number BETWEEN 1 AND 255), + attempt_id TEXT NOT NULL UNIQUE, + input_digest TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatched', 'completed', 'failed', 'unknown', 'cancelled')), + result_json TEXT, + result_digest TEXT, + failure TEXT, + PRIMARY KEY (job_id, number), + CHECK ((result_json IS NULL) = (result_digest IS NULL)), + CHECK (state <> 'completed' OR result_json IS NOT NULL) +); +CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) + WHERE serialization_key IS NOT NULL AND state IN ('pending', 'unknown'); +CREATE INDEX job_attempt_state ON job_attempts(state); +CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF + job_id, request_json, fingerprint, serialization_key, max_concurrent ON job_intents +BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END; +CREATE TRIGGER job_attempt_immutable BEFORE UPDATE OF + job_id, number, attempt_id, input_digest ON job_attempts +BEGIN SELECT RAISE(ABORT, 'immutable Job attempt identity'); END; +CREATE TRIGGER job_result_immutable BEFORE UPDATE OF result_json, result_digest ON job_attempts +WHEN OLD.result_json IS NOT NULL AND + (NEW.result_json IS NOT OLD.result_json OR NEW.result_digest IS NOT OLD.result_digest) +BEGIN SELECT RAISE(ABORT, 'immutable accepted Job result'); END; +"; + /// Frozen schema-1/2 adapter. Preserve original intent and attempt bindings /// before replacing counters with identities derived from immutable input. pub(super) fn migrate_legacy_jobs(conn: &Connection) -> Result<(), JobStoreError> { @@ -244,10 +287,15 @@ mod tests { .collect::>() .unwrap(); assert_eq!(digests, vec![request("done").input_digest().unwrap(); 2]); - assert_eq!(store.get("running").unwrap().state, JobState::Unknown); + assert_eq!(store.get("running").unwrap().state, JobState::Failed); + assert_eq!( + store.get("running").unwrap().attempt.failure.as_deref(), + Some(recovery::UPDATE_INTERRUPTION) + ); + assert!(store.get("running").unwrap().version_stopped); assert_eq!( store.reserve(request("pending")).unwrap().attempt.state, - JobAttemptState::Reserved + JobAttemptState::Failed ); assert!( store diff --git a/crates/standalone/src/job_store/recovery.rs b/crates/standalone/src/job_store/recovery.rs new file mode 100644 index 000000000..43c2026a7 --- /dev/null +++ b/crates/standalone/src/job_store/recovery.rs @@ -0,0 +1,1076 @@ +use super::*; + +pub(super) const UPDATE_INTERRUPTION: &str = "バージョン更新による実行打切り"; + +impl JobStore { + /// The Host supplies the identity from its revalidated exclusive Worker lease. + /// This binds admission, results and restart reconciliation to the same owner, + /// rather than a fresh connection UUID that could impersonate a restart. + pub(crate) fn bind_host_lease(&mut self, lease_id: &str) -> Result<(), JobStoreError> { + job::validate_identifier("Worker lease", lease_id, 128)?; + self.owner_id = format!("worker-lease:{lease_id}"); + Ok(()) + } + + /// A Host recovery event probes the exact retained attempt using an actual + /// write transaction without changing its cause, result or ownership proof. + pub(crate) fn probe_settlement_write( + &mut self, + job_id: &str, + attempt_id: &str, + ) -> Result<(), JobStoreError> { + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let saved = require_snapshot(&tx, job_id)?; + check_attempt(&saved, attempt_id)?; + if saved.version_stopped + || !saved.executions.iter().any(|execution| { + execution.attempt_id == attempt_id && execution.owner_id == self.owner_id + }) + { + return Err(JobStoreError::AttemptMismatch); + } + tx.execute( + "UPDATE job_attempts SET failure=failure WHERE job_id=?1 AND attempt_id=?2", + params![job_id, attempt_id], + )?; + tx.commit()?; + Ok(()) + } + + /// Normal restart only: the caller rechecks the acquired Host lease immediately + /// before this transaction. Failure fences subsequent callbacks/results but is + /// NOT a cleanup receipt, rollback, or permission to repeat domain side effects. + pub(crate) fn reconcile_host_restart(&mut self) -> Result, JobStoreError> { + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let ids = tx + .prepare( + "SELECT i.job_id FROM job_intents i JOIN job_attempts a + ON a.job_id=i.job_id AND a.number=i.current_attempt + WHERE i.state='pending' AND a.state='dispatched' + AND NOT EXISTS(SELECT 1 FROM job_version_stops s WHERE s.job_id=i.job_id) + ORDER BY i.rowid", + )? + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + let snapshots = ids + .iter() + .map(|id| require_snapshot(&tx, id)) + .collect::, _>>()?; + // A second connection under the SAME live lease is not a replacement Host. + // Reject the whole transaction before changing even an unrelated attempt. + if snapshots.iter().any(|saved| { + saved.executions.iter().any(|execution| { + execution.attempt_id == saved.attempt.attempt_id + && execution.owner_id == self.owner_id + }) + }) { + return Err(JobStoreError::InvalidTransition( + "the current Worker lease still owns a dispatched execution", + )); + } + for saved in snapshots { + let bound = saved + .executions + .iter() + .find(|execution| execution.attempt_id == saved.attempt.attempt_id); + let cause = match bound { + Some(_) if saved.executions_released() => "host_restart_result_receipt_missing: execution cleanup is confirmed but no structured result was accepted; reconcile same-attempt domain receipts; do not repeat non-idempotent effects".into(), + Some(execution) => format!( + "host_restart_execution_unconfirmed: prior owner={} run={}; no authoritative execution/Host callback drain proof; reconcile same-attempt domain/result receipts, retain cleanup ownership; do not redispatch", + execution.owner_id, execution.run_id), + None => "host_restart_acceptance_evidence_missing: dispatched attempt has no saved execution binding; cannot reconcile an owner or redispatch; retain immutable input and domain receipts".into(), + }; + tx.execute("UPDATE job_attempts SET state='failed',failure=?3 WHERE job_id=?1 AND attempt_id=?2 AND state='dispatched'", + params![saved.request.job_id,saved.attempt.attempt_id,cause])?; + tx.execute( + "UPDATE job_intents SET state='failed' WHERE job_id=?1 AND state='pending'", + [&saved.request.job_id], + )?; + } + let settled = ids + .iter() + .map(|id| require_snapshot(&tx, id)) + .collect::, _>>()?; + tx.commit()?; + Ok(settled) + } +} + +/// Rebuild with FK enforcement disabled outside the transaction: DROP must not +/// delete children or rewrite their canonical references. Check the entire graph +/// before commit, and restore connection settings even after a failed migration. +pub(super) fn initialize(conn: &mut Connection) -> Result<(), JobStoreError> { + if !conn.is_autocommit() { + return Err(JobStoreError::InvalidTransition( + "Job initialization requires an idle connection", + )); + } + let foreign_keys: bool = conn.pragma_query_value(None, "foreign_keys", |r| r.get(0))?; + let legacy_alter_table: bool = + conn.pragma_query_value(None, "legacy_alter_table", |r| r.get(0))?; + let result = (|| { + conn.pragma_update(None, "foreign_keys", false)?; + conn.pragma_update(None, "legacy_alter_table", true)?; + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let version: i64 = tx.pragma_query_value(None, "user_version", |r| r.get(0))?; + match version { + 0 => { + tx.execute_batch(super::SCHEMA)?; + tx.execute_batch(GRANT_SCHEMA)?; + tx.execute_batch(RECONCILIATION_SCHEMA)?; + tx.execute_batch(SCHEMA)?; + } + 1..=5 => { + if version == 1 { + tx.execute_batch(GRANT_SCHEMA)?; + } + if version <= 2 { + legacy_migrations::migrate_legacy_jobs(&tx)?; + } + if version <= 3 { + tx.execute_batch(RECONCILIATION_SCHEMA)?; + } + if version < 5 { + tx.execute_batch(INPUT_RETENTION_SCHEMA)?; + let has_legacy_archive: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema + WHERE type='table' AND name='legacy_job_archive')", + [], + |r| r.get(0), + )?; + if has_legacy_archive { + tx.execute_batch( + "DROP TRIGGER legacy_job_archive_immutable; + CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive + BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END; + CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive + WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available') + OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE + BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;", + )?; + } + } + migrate(&tx)?; + } + SCHEMA_VERSION => {} + other => return Err(JobStoreError::UnsupportedSchema(other)), + } + if tx.prepare("PRAGMA foreign_key_check")?.exists([])? { + return Err(JobStoreError::Corrupt( + "Job migration foreign key check failed".into(), + )); + } + tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; + tx.commit()?; + Ok(()) + })(); + // Attempt both restorations; an error in the first must not skip the second. + let restore_alter = conn.pragma_update(None, "legacy_alter_table", legacy_alter_table); + let restore_foreign_keys = conn.pragma_update(None, "foreign_keys", foreign_keys); + result?; + restore_alter?; + restore_foreign_keys?; + Ok(()) +} + +/// This cutover is not startup recovery. Preserve original states and causes +/// without copying input bodies into another persistent erasure domain. An old +/// execution has no stop/cleanup proof, even if its outcome was already terminal. +fn migrate(conn: &Connection) -> Result<(), JobStoreError> { + conn.execute_batch(SCHEMA)?; + conn.execute_batch( + "INSERT INTO job_version_history(job_id,attempt_id,job_state,attempt_state,failure) + SELECT i.job_id,a.attempt_id,i.state,a.state,a.failure + FROM job_intents i JOIN job_attempts a ON a.job_id=i.job_id; + INSERT INTO job_version_stops(job_id) SELECT job_id FROM job_intents; + INSERT INTO job_execution_ownership(job_id,attempt_id,owner_id,run_id,writes_revoked) + SELECT job_id,attempt_id,'schema-before-6',attempt_id,0 FROM job_attempts + WHERE state <> 'reserved';", + )?; + let objects = conn + .prepare( + "SELECT name,sql FROM sqlite_schema + WHERE tbl_name IN ('job_intents','job_attempts') + AND type IN ('index','trigger') AND sql IS NOT NULL + ORDER BY type,name", + )? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? + .collect::, _>>()?; + // Do not rename the parents: child FKs (including extension tables) must + // continue to name job_intents/job_attempts. Copy without running old update + // triggers, and restore extension triggers only after the rows are reloaded. + conn.execute_batch( + "CREATE TEMP TABLE cutover_intents AS SELECT * FROM job_intents; + CREATE TEMP TABLE cutover_attempts AS SELECT * FROM job_attempts; + DROP TABLE job_attempts; + DROP TABLE job_intents;", + )?; + conn.execute_batch(super::SCHEMA)?; + conn.execute_batch( + "INSERT INTO job_intents + (job_id,request_json,fingerprint,serialization_key,max_concurrent, + state,current_attempt,acknowledged,input_disposition) + SELECT job_id,request_json,fingerprint,serialization_key,max_concurrent, + CASE WHEN state IN ('pending','unknown') THEN 'failed' ELSE state END, + current_attempt,acknowledged,input_disposition FROM cutover_intents;", + )?; + conn.execute( + "INSERT INTO job_attempts + (job_id,number,attempt_id,input_digest,state,result_json,result_digest,failure) + SELECT job_id,number,attempt_id,input_digest, + CASE WHEN state IN ('reserved','dispatched','unknown') THEN 'failed' ELSE state END, + result_json,result_digest, + CASE WHEN state IN ('reserved','dispatched','unknown') THEN ?1 ELSE failure END + FROM cutover_attempts", + [UPDATE_INTERRUPTION], + )?; + conn.execute_batch("DROP TABLE cutover_intents; DROP TABLE cutover_attempts;")?; + for (name, sql) in objects { + let exists: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=?1)", + [&name], + |r| r.get(0), + )?; + if !exists { + conn.execute_batch(&sql)?; + } + } + Ok(()) +} + +pub(super) const SCHEMA: &str = " +CREATE TABLE job_version_history ( + job_id TEXT NOT NULL, + attempt_id TEXT NOT NULL PRIMARY KEY, + job_state TEXT NOT NULL, + attempt_state TEXT NOT NULL, + failure TEXT +); +CREATE TRIGGER job_version_history_immutable BEFORE UPDATE ON job_version_history +BEGIN SELECT RAISE(ABORT,'immutable previous Job state'); END; +CREATE TRIGGER job_version_history_retained BEFORE DELETE ON job_version_history +BEGIN SELECT RAISE(ABORT,'retained previous Job state'); END; +CREATE TABLE job_version_stops (job_id TEXT NOT NULL PRIMARY KEY); +CREATE TRIGGER job_version_stop_immutable BEFORE UPDATE ON job_version_stops +BEGIN SELECT RAISE(ABORT,'immutable version stop'); END; +CREATE TRIGGER job_version_stop_retained BEFORE DELETE ON job_version_stops +BEGIN SELECT RAISE(ABORT,'retained version stop'); END; +CREATE TABLE job_execution_ownership ( + job_id TEXT NOT NULL REFERENCES job_intents(job_id), + attempt_id TEXT NOT NULL PRIMARY KEY REFERENCES job_attempts(attempt_id), + owner_id TEXT NOT NULL, + run_id TEXT NOT NULL, + writes_revoked INTEGER NOT NULL DEFAULT 0 CHECK(writes_revoked IN (0,1)), + cleanup_receipt TEXT, + CHECK(cleanup_receipt IS NULL OR writes_revoked=1) +); +CREATE TRIGGER job_execution_identity_immutable BEFORE UPDATE OF job_id,attempt_id,owner_id,run_id ON job_execution_ownership +BEGIN SELECT RAISE(ABORT,'immutable Job execution binding'); END; +CREATE TRIGGER job_execution_proof_monotonic BEFORE UPDATE ON job_execution_ownership +WHEN NEW.writes_revoked < OLD.writes_revoked + OR (OLD.cleanup_receipt IS NOT NULL AND NEW.cleanup_receipt IS NOT OLD.cleanup_receipt) +BEGIN SELECT RAISE(ABORT,'immutable execution proof'); END; +CREATE TRIGGER job_execution_retained BEFORE DELETE ON job_execution_ownership +BEGIN SELECT RAISE(ABORT,'retained Job execution'); END; +"; + +#[cfg(test)] +mod tests { + //! Store integration tests use the production initializer and real temporary + //! SQLite databases, with no service/Worker execution or external effects. + use super::*; + + fn request(id: &str) -> JobRequest { + super::super::tests::request(id) + } + + fn seed(path: &Path, version: i64) -> Connection { + let mut conn = Connection::open(path).unwrap(); + feature_storage::configure_connection(&conn).unwrap(); + let tx = conn.transaction().unwrap(); + let baseline = if version <= 2 { + legacy_migrations::SCHEMA.replace("input_digest", "input_revision") + } else { + legacy_migrations::SCHEMA.to_owned() + }; + tx.execute_batch(&baseline).unwrap(); + if version >= 2 { + tx.execute_batch(GRANT_SCHEMA).unwrap(); + } + if version >= 4 { + tx.execute_batch(RECONCILIATION_SCHEMA).unwrap(); + } + if version >= 5 { + tx.execute_batch(INPUT_RETENTION_SCHEMA).unwrap(); + } + for (id, state, attempt) in [ + ("reserved", "pending", "reserved"), + ("running", "pending", "dispatched"), + ("uncertain", "unknown", "unknown"), + ("done", "completed", "completed"), + ("failed", "failed", "failed"), + ("cancelled", "cancelled", "cancelled"), + ] { + let req = request(id); + let mut body = serde_json::to_value(&req).unwrap(); + if version <= 2 { + body["input_revision"] = Value::String("old-token".into()); + } + tx.execute( + "INSERT INTO job_intents(job_id,request_json,fingerprint,max_concurrent,state,current_attempt,acknowledged) + VALUES(?1,?2,?3,8,?4,1,?5)", + params![id, serde_json::to_string(&body).unwrap(), req.fingerprint().unwrap(), state, id == "done"], + ).unwrap(); + tx.execute( + &format!("INSERT INTO job_attempts(job_id,number,attempt_id,{},state,result_json,result_digest,failure) + VALUES(?1,1,?2,?3,?4,?5,?6,'old cause')", + if version <= 2 { "input_revision" } else { "input_digest" }), + params![id, job::attempt_id(id, 1), req.input_digest().unwrap(), attempt, + (id == "done").then_some("{\"receipt\":true}"), (id == "done").then_some("digest")], + ).unwrap(); + } + // A previous failed attempt must remain distinct from the current one. + tx.execute( + &format!("INSERT INTO job_attempts(job_id,number,attempt_id,{},state,failure) + VALUES('done',2,'historic-attempt','historic-input','failed','historic cause')", + if version <= 2 { "input_revision" } else { "input_digest" }), + [], + ).unwrap(); + if version >= 2 { + tx.execute_batch( + "INSERT INTO job_domain_grants VALUES('done','{\"domain\":\"fixture\"}');", + ) + .unwrap(); + } + if version >= 4 { + tx.execute_batch("INSERT INTO job_domain_reconciliations VALUES('done','historic-attempt','checkpoint');").unwrap(); + } + if version == 5 { + tx.execute_batch("UPDATE job_intents SET request_json=json_set(request_json,'$.input',NULL),input_disposition='erased' WHERE job_id='cancelled';").unwrap(); + } + tx.execute_batch( + "CREATE TABLE fixture_children ( + job_id TEXT REFERENCES job_intents(job_id), + attempt_id TEXT REFERENCES job_attempts(attempt_id), + checkpoint TEXT NOT NULL); + INSERT INTO fixture_children VALUES('done','historic-attempt','child checkpoint'); + CREATE INDEX fixture_intent_index ON job_intents(fingerprint); + CREATE INDEX fixture_attempt_index ON job_attempts(failure); + CREATE TRIGGER fixture_intent_retained BEFORE DELETE ON job_intents + BEGIN SELECT RAISE(ABORT,'retained fixture intent'); END; + CREATE TRIGGER fixture_attempt_retained BEFORE DELETE ON job_attempts + BEGIN SELECT RAISE(ABORT,'retained fixture attempt'); END;", + ) + .unwrap(); + tx.pragma_update(None, "user_version", version).unwrap(); + tx.commit().unwrap(); + conn + } + + fn settings(conn: &Connection) -> (bool, bool) { + ( + conn.pragma_query_value(None, "foreign_keys", |r| r.get(0)) + .unwrap(), + conn.pragma_query_value(None, "legacy_alter_table", |r| r.get(0)) + .unwrap(), + ) + } + + fn schema(conn: &Connection) -> Vec<(String, String, String)> { + conn.prepare( + "SELECT type,name,sql FROM sqlite_schema WHERE sql IS NOT NULL + AND name NOT LIKE 'fixture_%' AND name NOT LIKE 'legacy_job_archive%' + ORDER BY type,name", + ) + .unwrap() + .query_map([], |r| { + Ok(( + r.get(0)?, + r.get(1)?, + r.get::<_, String>(2)? + .split_whitespace() + .collect::>() + .join(" "), + )) + }) + .unwrap() + .collect::>() + .unwrap() + } + + fn assert_integrity(conn: &Connection) { + assert!( + !conn + .prepare("PRAGMA foreign_key_check") + .unwrap() + .exists([]) + .unwrap() + ); + let targets = conn.prepare("SELECT \"table\" FROM pragma_foreign_key_list('fixture_children') ORDER BY \"table\"") + .unwrap().query_map([], |r| r.get::<_, String>(0)).unwrap() + .collect::, _>>().unwrap(); + assert_eq!(targets, ["job_attempts", "job_intents"]); + let checkpoint: String = conn + .query_row("SELECT checkpoint FROM fixture_children", [], |r| r.get(0)) + .unwrap(); + assert_eq!(checkpoint, "child checkpoint"); + for table in [ + "job_domain_grants", + "job_domain_reconciliations", + "job_execution_ownership", + ] { + let invalid: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM pragma_foreign_key_list(?1) WHERE \"table\" NOT IN ('job_intents','job_attempts'))", + [table], |r| r.get(0), + ).unwrap(); + assert!(!invalid, "noncanonical FK in {table}"); + } + let extension_objects: i64 = conn.query_row("SELECT count(*) FROM sqlite_schema WHERE name IN ('fixture_intent_index','fixture_attempt_index','fixture_intent_retained','fixture_attempt_retained')", [], |r| r.get(0)).unwrap(); + assert_eq!(extension_objects, 4); + } + + #[test] + fn schemas_one_through_five_preserve_receipts_history_checkpoints_and_constraints() { + let mut fresh = Connection::open_in_memory().unwrap(); + initialize(&mut fresh).unwrap(); + let canonical = schema(&fresh); + for version in 1..=5 { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut conn = seed(&path, version); + let before_settings = settings(&conn); + initialize(&mut conn).unwrap(); + assert_eq!(settings(&conn), before_settings); + assert_eq!(schema(&conn), canonical, "schema {version}"); + assert_integrity(&conn); + assert_eq!( + conn.query_row("SELECT count(*) FROM job_version_stops", [], |r| r + .get::<_, i64>(0)) + .unwrap(), + 6 + ); + let proofs: i64 = conn.query_row("SELECT count(*) FROM job_execution_ownership WHERE writes_revoked<>0 OR cleanup_receipt IS NOT NULL", [], |r| r.get(0)).unwrap(); + assert_eq!(proofs, 0); + let executions: i64 = conn + .query_row("SELECT count(*) FROM job_execution_ownership", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(executions, 6); + let historical: (String, String) = conn.query_row("SELECT attempt_state,failure FROM job_version_history WHERE attempt_id='historic-attempt'", [], |r| Ok((r.get(0)?,r.get(1)?))).unwrap(); + assert_eq!(historical, ("failed".into(), "historic cause".into())); + if version >= 4 { + assert_eq!( + conn.query_row( + "SELECT domain_receipt FROM job_domain_reconciliations", + [], + |r| r.get::<_, String>(0) + ) + .unwrap(), + "checkpoint" + ); + } + if version <= 2 { + let attempts: String = conn + .query_row( + "SELECT attempts_json FROM legacy_job_archive WHERE job_id='done'", + [], + |r| r.get(0), + ) + .unwrap(); + let attempts: Value = serde_json::from_str(&attempts).unwrap(); + assert_eq!(attempts[1]["failure"], "historic cause"); + } + drop(conn); + let mut store = JobStore::open(&path).unwrap(); + for (id, old_state) in [ + ("reserved", "reserved"), + ("running", "dispatched"), + ("uncertain", "unknown"), + ] { + let saved = store.get(id).unwrap(); + assert_eq!(saved.state, JobState::Failed); + assert_eq!(saved.attempt.state, JobAttemptState::Failed); + assert_eq!(saved.attempt.failure.as_deref(), Some(UPDATE_INTERRUPTION)); + assert_eq!(saved.request, request(id)); + let history: (String, String) = store + .conn + .query_row( + "SELECT attempt_state,failure FROM job_version_history WHERE job_id=?1", + [id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(history, (old_state.into(), "old cause".into())); + } + assert_eq!( + store.get("done").unwrap().result, + Some(serde_json::json!({"receipt":true})) + ); + assert!(store.get("done").unwrap().acknowledged); + assert_eq!( + store.get("failed").unwrap().attempt.failure.as_deref(), + Some("old cause") + ); + assert_eq!(store.get("cancelled").unwrap().state, JobState::Cancelled); + if version == 5 { + let retired = store.get("cancelled").unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Erased); + assert!(retired.request.input.is_null()); + assert_eq!( + retired.intent_fingerprint, + request("cancelled").fingerprint().unwrap() + ); + } + for id in [ + "reserved", + "running", + "uncertain", + "done", + "failed", + "cancelled", + ] { + let saved = store.get(id).unwrap(); + assert!(saved.version_stopped); + if id != "reserved" { + assert!(!saved.executions_released()); + assert!( + saved + .executions + .iter() + .all(|e| e.owner_id == "schema-before-6" + && !e.writes_revoked + && e.cleanup_receipt.is_none()) + ); + } + assert!(store.retry(id).is_err()); + assert!(store.dispatch(id, &saved.attempt.attempt_id, 8).is_err()); + } + for sql in [ + "UPDATE job_intents SET fingerprint='changed' WHERE job_id='done'", + "UPDATE job_attempts SET input_digest='changed' WHERE job_id='done'", + "UPDATE job_attempts SET result_json='{}' WHERE job_id='done' AND number=1", + "UPDATE job_intents SET request_json='{}' WHERE job_id='done'", + "UPDATE job_attempts SET state='unknown' WHERE job_id='failed'", + "UPDATE job_intents SET state='unknown' WHERE job_id='failed'", + "DELETE FROM job_version_stops", + "UPDATE job_version_history SET failure='changed'", + "DELETE FROM job_execution_ownership", + "DELETE FROM job_intents WHERE job_id='done'", + "DELETE FROM job_attempts WHERE job_id='done'", + ] { + assert!(store.conn.execute_batch(sql).is_err(), "{sql}"); + } + let fresh = store.reserve(request("new")).unwrap(); + assert!(!fresh.version_stopped); + drop(store); + let reopened = JobStore::open(&path).unwrap(); + assert_eq!(reopened.get("new").unwrap(), fresh); + assert_integrity(&reopened.conn); + } + } + + #[test] + fn failed_foreign_key_check_rolls_back_rebuild_and_allows_same_connection_retry() { + for version in 1..=5 { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut conn = seed(&path, version); + conn.pragma_update(None, "foreign_keys", false).unwrap(); + conn.execute( + "INSERT INTO fixture_children VALUES('missing','missing','bad checkpoint')", + [], + ) + .unwrap(); + conn.pragma_update(None, "foreign_keys", true).unwrap(); + let before_schema = schema(&conn); + let before_request: String = conn + .query_row( + "SELECT request_json FROM job_intents WHERE job_id='running'", + [], + |r| r.get(0), + ) + .unwrap(); + let before_settings = settings(&conn); + assert!(matches!( + initialize(&mut conn), + Err(JobStoreError::Corrupt(_)) + )); + assert_eq!(settings(&conn), before_settings); + assert!(conn.is_autocommit()); + assert_eq!(schema(&conn), before_schema); + assert_eq!( + conn.pragma_query_value(None, "user_version", |r| r.get::<_, i64>(0)) + .unwrap(), + version + ); + assert_eq!( + conn.query_row( + "SELECT request_json FROM job_intents WHERE job_id='running'", + [], + |r| r.get::<_, String>(0) + ) + .unwrap(), + before_request + ); + assert_eq!( + conn.query_row( + "SELECT state FROM job_attempts WHERE job_id='running'", + [], + |r| r.get::<_, String>(0) + ) + .unwrap(), + "dispatched" + ); + assert!( + !conn + .prepare("SELECT 1 FROM sqlite_temp_schema WHERE name LIKE 'cutover_%'") + .unwrap() + .exists([]) + .unwrap() + ); + assert!( + conn.execute( + "UPDATE job_intents SET fingerprint='changed' WHERE job_id='done'", + [] + ) + .is_err() + ); + conn.execute("DELETE FROM fixture_children WHERE job_id='missing'", []) + .unwrap(); + initialize(&mut conn).unwrap(); + assert_integrity(&conn); + let after_schema = schema(&conn); + initialize(&mut conn).unwrap(); + assert_eq!(schema(&conn), after_schema); + assert_eq!(settings(&conn), before_settings); + drop(conn); + let reopened = JobStore::open(&path).unwrap(); + assert!(reopened.get("failed").unwrap().version_stopped); + assert_integrity(&reopened.conn); + } + } + + #[test] + fn initialization_restores_both_pragma_values_on_success_and_failure() { + for foreign_keys in [false, true] { + for legacy_alter_table in [false, true] { + let mut conn = Connection::open_in_memory().unwrap(); + conn.pragma_update(None, "foreign_keys", foreign_keys) + .unwrap(); + conn.pragma_update(None, "legacy_alter_table", legacy_alter_table) + .unwrap(); + initialize(&mut conn).unwrap(); + assert_eq!(settings(&conn), (foreign_keys, legacy_alter_table)); + conn.pragma_update(None, "user_version", SCHEMA_VERSION + 1) + .unwrap(); + assert!(matches!( + initialize(&mut conn), + Err(JobStoreError::UnsupportedSchema(_)) + )); + assert_eq!(settings(&conn), (foreign_keys, legacy_alter_table)); + assert!(conn.is_autocommit()); + } + } + } + + #[tokio::test] + async fn settlement_recovers_sqlite_contention_for_same_attempt_without_dispatch() { + // Normal-operation storage fault, not migration or process-exit coverage. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut store = JobStore::open(&path).unwrap(); + store.conn.busy_timeout(std::time::Duration::ZERO).unwrap(); + let reserved = store.reserve(request("settlement")).unwrap(); + let dispatched = store + .dispatch("settlement", &reserved.attempt.attempt_id, 8) + .unwrap(); + let ownership = dispatched.executions[0].clone(); + let blocker = Connection::open(&path).unwrap(); + blocker.execute_batch("BEGIN IMMEDIATE").unwrap(); + let mut calls = 0; + crate::jobs::recover_settlement(|| { + calls += 1; + let result = (|| -> Result<(), JobStoreError> { + store.check_execution_binding( + "settlement", + &reserved.attempt.attempt_id, + &reserved.input_digest, + &ownership, + )?; + store.finish( + "settlement", + &reserved.attempt.attempt_id, + JobAttemptState::Failed, + Some("timeout"), + )?; + store.confirm_execution_cleanup( + "settlement", + &ownership, + "fixture-owned-run-joined", + ) + })(); + if calls == 1 { + assert!(matches!(result, Err(JobStoreError::Sqlite(_)))); + assert_eq!(store.get("settlement").unwrap(), dispatched); + blocker.execute_batch("ROLLBACK").unwrap(); + } + result.map_err(crate::jobs::JobServiceError::from) + }) + .await + .unwrap(); + assert_eq!(calls, 2); + let settled = store.get("settlement").unwrap(); + assert_eq!(settled.attempt.attempt_id, reserved.attempt.attempt_id); + assert_eq!(settled.attempt.number, 1); + assert_eq!(settled.attempt.failure.as_deref(), Some("timeout")); + assert!(settled.executions_released()); + drop(store); + assert_eq!( + JobStore::open(&path).unwrap().get("settlement").unwrap(), + settled + ); + } + + #[tokio::test] + async fn settlement_persistent_contention_exhausts_budget_and_retains_obligations() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut store = JobStore::open(&path).unwrap(); + store.conn.busy_timeout(std::time::Duration::ZERO).unwrap(); + let reserved = store.reserve(request("blocked")).unwrap(); + let dispatched = store + .dispatch("blocked", &reserved.attempt.attempt_id, 8) + .unwrap(); + let blocker = Connection::open(&path).unwrap(); + blocker.execute_batch("BEGIN IMMEDIATE").unwrap(); + let mut calls = 0; + let error = crate::jobs::recover_settlement(|| { + calls += 1; + store.finish( + "blocked", + &reserved.attempt.attempt_id, + JobAttemptState::Failed, + Some("timeout"), + )?; + Ok(()) + }) + .await + .unwrap_err(); + assert!( + matches!(error, crate::jobs::JobServiceError::SettlementRecoveryExhausted { attempts: 6, source } + if matches!(*source, crate::jobs::JobServiceError::Store(JobStoreError::Sqlite(_)))) + ); + assert_eq!(calls, 6); + assert_eq!(store.get("blocked").unwrap(), dispatched); + assert!(!store.execution_obligations().unwrap().is_empty()); + blocker.execute_batch("ROLLBACK").unwrap(); + } + + #[tokio::test] + async fn exhausted_settlement_detects_sqlite_recovery_without_user_action() { + use std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }; + use tokio::sync::Notify; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut store = JobStore::open(&path).unwrap(); + store.conn.busy_timeout(std::time::Duration::ZERO).unwrap(); + let reserved = store.reserve(request("event-recovery")).unwrap(); + let dispatched = store + .dispatch("event-recovery", &reserved.attempt.attempt_id, 8) + .unwrap(); + let ownership = dispatched.executions[0].clone(); + let store = Mutex::new(store); + let blocker = Connection::open(&path).unwrap(); + blocker.execute_batch("BEGIN IMMEDIATE").unwrap(); + let calls = AtomicUsize::new(0); + let exhausted = Notify::new(); + let recovered = Notify::new(); + let issue = Mutex::new(None); + let settle = crate::jobs::recover_owned_settlement( + &recovered, + &issue, + || { + let call = calls.fetch_add(1, Ordering::SeqCst) + 1; + let mut store = store.lock().unwrap(); + let result = store.finish( + "event-recovery", + &reserved.attempt.attempt_id, + JobAttemptState::Failed, + Some("original timeout"), + ); + if call == 6 { + exhausted.notify_one(); + } + result?; + store.confirm_execution_cleanup( + "event-recovery", + &ownership, + "fixture-owned-run-joined", + )?; + Ok(()) + }, + || { + store + .lock() + .unwrap() + .probe_settlement_write("event-recovery", &reserved.attempt.attempt_id)?; + Ok(()) + }, + ); + let restore_storage = async { + exhausted.notified().await; + assert!(matches!( + issue.lock().unwrap().as_ref(), + Some(crate::jobs::JobServiceError::SettlementRecoveryExhausted { attempts: 6, .. }) + )); + assert!( + issue + .lock() + .unwrap() + .as_ref() + .unwrap() + .to_string() + .contains("next action") + ); + assert_eq!( + store.lock().unwrap().get("event-recovery").unwrap(), + dispatched + ); + blocker.execute_batch("ROLLBACK").unwrap(); + // A restored read is not a write recovery event. Do not signal the + // Notify or call a Host API: the retained owner must detect recovery. + assert_eq!( + store.lock().unwrap().get("event-recovery").unwrap(), + dispatched + ); + tokio::task::yield_now().await; + assert_eq!(calls.load(Ordering::SeqCst), 6); + }; + let (result, ()) = tokio::time::timeout(std::time::Duration::from_secs(8), async { + tokio::join!(settle, restore_storage) + }) + .await + .unwrap(); + result.unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 7); + assert!(issue.lock().unwrap().is_none()); + let saved = store.lock().unwrap().get("event-recovery").unwrap(); + assert_eq!(saved.state, JobState::Failed); + assert_eq!(saved.attempt.attempt_id, reserved.attempt.attempt_id); + assert_eq!(saved.attempt.failure.as_deref(), Some("original timeout")); + assert!(saved.executions_released()); + } + + #[test] + fn restart_distinguishes_missing_result_from_saved_execution_cleanup() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut store = JobStore::open(&path).unwrap(); + store.bind_host_lease("prior-host").unwrap(); + let reserved = store.reserve(request("receipt-missing")).unwrap(); + let dispatched = store + .dispatch("receipt-missing", &reserved.attempt.attempt_id, 8) + .unwrap(); + store + .confirm_execution_cleanup( + "receipt-missing", + &dispatched.executions[0], + "fixture-owned-run-joined", + ) + .unwrap(); + let before = store.get("receipt-missing").unwrap(); + drop(store); + let mut reopened = JobStore::open(&path).unwrap(); + reopened.bind_host_lease("next-host").unwrap(); + let failed = reopened.reconcile_host_restart().unwrap().pop().unwrap(); + assert_eq!(failed.state, JobState::Failed); + assert_eq!(failed.executions, before.executions); + assert!(failed.executions_released()); + assert!( + failed + .attempt + .failure + .as_deref() + .unwrap() + .starts_with("host_restart_result_receipt_missing:") + ); + assert_eq!(failed.attempt.attempt_id, reserved.attempt.attempt_id); + assert!(failed.result.is_none()); + assert!(reopened.pending().unwrap().is_empty()); + } + + #[test] + fn restart_missing_dispatch_binding_fails_without_creating_retry_or_cleanup_proof() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut store = JobStore::open(&path).unwrap(); + store.bind_host_lease("first-owner").unwrap(); + let reserved = store.reserve(request("missing-binding")).unwrap(); + store + .conn + .execute( + "UPDATE job_attempts SET state='dispatched' WHERE job_id='missing-binding'", + [], + ) + .unwrap(); + drop(store); + let mut reopened = JobStore::open(&path).unwrap(); + reopened.bind_host_lease("next-owner").unwrap(); + let stopped = reopened.reconcile_host_restart().unwrap().pop().unwrap(); + assert_eq!(stopped.state, JobState::Failed); + assert_eq!(stopped.attempt.attempt_id, reserved.attempt.attempt_id); + assert!( + stopped + .attempt + .failure + .as_deref() + .unwrap() + .starts_with("host_restart_acceptance_evidence_missing:") + ); + assert!(stopped.executions.is_empty()); + assert!(!stopped.executions_released()); + assert!(matches!( + reopened.execution_obligations(), + Err(JobStoreError::Corrupt(_)) + )); + assert!(!stopped.version_stopped); + assert!(reopened.retry("missing-binding").is_err()); + assert!(reopened.pending().unwrap().is_empty()); + assert!(reopened.reconcile_host_restart().unwrap().is_empty()); + assert_eq!(reopened.get("missing-binding").unwrap(), stopped); + } + + #[test] + fn normal_restart_never_resumes_or_rewrites_migration_stopped_jobs() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let conn = seed(&path, 5); + drop(conn); + let mut store = JobStore::open(&path).unwrap(); + let migrated = store.get("running").unwrap(); + let fresh = store.reserve(request("post-upgrade")).unwrap(); + let dispatched = store + .dispatch("post-upgrade", &fresh.attempt.attempt_id, 8) + .unwrap(); + drop(store); + let mut reopened = JobStore::open(&path).unwrap(); + reopened.bind_host_lease("normal-restart").unwrap(); + let changed = reopened.reconcile_host_restart().unwrap(); + assert_eq!(changed.len(), 1); + assert_eq!(changed[0].request.job_id, "post-upgrade"); + assert!(!changed[0].version_stopped); + assert_eq!(changed[0].executions, dispatched.executions); + assert_eq!(reopened.get("running").unwrap(), migrated); + assert!( + reopened + .resume_granted("running", &migrated.attempt.attempt_id) + .is_err() + ); + assert!(reopened.retry("running").is_err()); + } + + #[test] + fn settlement_binding_rejects_wrong_run_digest_later_attempt_and_reopened_owner() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut store = JobStore::open(&path).unwrap(); + let reserved = store.reserve(request("binding")).unwrap(); + let dispatched = store + .dispatch("binding", &reserved.attempt.attempt_id, 8) + .unwrap(); + let ownership = dispatched.executions[0].clone(); + let mut wrong = ownership.clone(); + wrong.run_id = "foreign-run".into(); + assert!( + store + .check_execution_binding( + "binding", + &reserved.attempt.attempt_id, + &reserved.input_digest, + &wrong + ) + .is_err() + ); + assert!( + store + .check_execution_binding( + "binding", + &reserved.attempt.attempt_id, + "foreign-digest", + &ownership + ) + .is_err() + ); + assert_eq!(store.get("binding").unwrap(), dispatched); + store + .finish( + "binding", + &reserved.attempt.attempt_id, + JobAttemptState::Failed, + Some("known failure"), + ) + .unwrap(); + store + .confirm_execution_cleanup("binding", &ownership, "fixture-owned-run-joined") + .unwrap(); + let next = store.retry("binding").unwrap(); + assert!( + store + .check_execution_binding( + "binding", + &reserved.attempt.attempt_id, + &reserved.input_digest, + &ownership + ) + .is_err() + ); + assert_eq!(store.get("binding").unwrap(), next); + let current = store + .dispatch("binding", &next.attempt.attempt_id, 8) + .unwrap(); + drop(store); + let reopened = JobStore::open(&path).unwrap(); + assert!( + reopened + .check_execution_binding( + "binding", + &next.attempt.attempt_id, + &next.input_digest, + ¤t.executions[1] + ) + .is_err() + ); + assert_eq!(reopened.get("binding").unwrap(), current); + } + + #[test] + fn reopening_preserves_current_schema_dispatched_attempt_without_cutover() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + let mut store = JobStore::open(&path).unwrap(); + let reserved = store.reserve(request("current")).unwrap(); + let dispatched = store + .dispatch("current", &reserved.attempt.attempt_id, 8) + .unwrap(); + drop(store); + let reopened = JobStore::open(&path).unwrap(); + assert_eq!(reopened.get("current").unwrap(), dispatched); + assert!( + !reopened + .conn + .prepare("SELECT 1 FROM job_version_history") + .unwrap() + .exists([]) + .unwrap() + ); + assert!(!reopened.get("current").unwrap().version_stopped); + } +} diff --git a/crates/standalone/src/jobs.rs b/crates/standalone/src/jobs.rs index bc375e005..37d654f5f 100644 --- a/crates/standalone/src/jobs.rs +++ b/crates/standalone/src/jobs.rs @@ -11,24 +11,27 @@ //! Dropping a wait future does not release Host cancel/cleanup ownership. Consumer acknowledgement //! is separate from result lifetime; delivery failures never trigger model reexecution. //! -//! Timeout/shutdown makes unaccepted attempts Unknown before cancellation. Unconfirmed cleanup -//! keeps Host leases held. Clones cannot reopen the store after service closure or carry -//! authority over to another Host. +//! Timeout/shutdown fences unaccepted attempts before cancellation. Only owned runner shutdown +//! plus joined/revoked Host callbacks discharge execution reservations. Clones cannot reopen +//! the store after service closure or carry authority over to another Host. #[cfg(test)] mod tests; use std::collections::HashMap; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, +}; use std::time::Duration; use agen::llm_client::LlmClient; use job::{JobRequest, JobResultSubmission}; -use tokio::sync::watch; +use tokio::sync::{Notify, watch}; use crate::job_store::JobStore; pub use crate::job_store::{ - JobAttempt, JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, - JobState, JobStoreError, + JobAttempt, JobAttemptState, JobExecutionOwnership, JobInputDisposition, + JobInputRetirementReason, JobSnapshot, JobState, JobStoreError, }; #[derive(Debug, thiserror::Error)] @@ -45,12 +48,35 @@ pub enum JobServiceError { Cleanup(String), #[error("standalone Job shutdown is incomplete; execution resources are still owned")] ShutdownIncomplete, + #[error( + "Job settlement storage recovery exhausted after {attempts} tries; execution retained; next action: await a successful write/recovery probe in this Host store, then reconcile the same attempt: {source}" + )] + SettlementRecoveryExhausted { + attempts: usize, + #[source] + source: Box, + }, } +pub(crate) trait JobEffectGuard {} +impl JobEffectGuard for T {} + pub(crate) trait JobAttemptFence: Send + Sync { fn ensure_live(&self) -> Result<(), String>; + fn enter(&self) -> Result, String> { + self.ensure_live()?; + Ok(Box::new(())) + } } pub(crate) trait JobDomainProvider: Send + Sync { + /// Opt in only when every external callback holds `JobAttemptFence::enter` + /// through its complete synchronous effect and no ungated background work exists. + /// The service joins that gate after runner/finalizer shutdown; an ordinary + /// liveness check alone cannot provide this guarantee. + fn callbacks_use_attempt_fence(&self) -> bool { + false + } + fn grant( &self, snapshot: &JobSnapshot, @@ -64,10 +90,21 @@ struct Execution { attempt_id: String, resource_key: Option, max_concurrent: u16, + effect_gate: Arc>, + recovery_issue: Arc>>, task: tokio::task::JoinHandle>, + // Retain typed Worker failures and their cleanup resources until the owner can resolve them. + _owned: Arc< + Mutex< + Option>, + >, + >, } struct Executions { closed: bool, + // Set only after this service joined owned Tasks, checked durable obligations + // and dropped its stores/adapters. Empty maps or closed slots are not proof. + shutdown_confirmed: bool, jobs: HashMap, } @@ -90,6 +127,7 @@ pub struct StandaloneJobs { capacity: u16, domain: Arc>>>, model_client: Arc>>>, + storage_recovered: Arc, } impl StandaloneJobs { @@ -111,6 +149,12 @@ impl StandaloneJobs { "delegated Job execution/cleanup remains running or unconfirmed".into(), )); } + if !self + .with_store(|store| store.execution_obligations())? + .is_empty() + { + return Err(JobServiceError::ShutdownIncomplete); + } Ok(QuiescentJobClosure { executions }) } @@ -119,15 +163,117 @@ impl StandaloneJobs { store: Arc::new(Mutex::new(Some(JobStore::open(path)?))), executions: Arc::new(Mutex::new(Executions { closed: false, + shutdown_confirmed: false, jobs: HashMap::new(), })), cwd, capacity: job::DEFAULT_MAX_CONCURRENT_JOBS, domain: Arc::default(), model_client: Arc::default(), + storage_recovered: Arc::default(), }) } + /// Production Host entry: only a freshly revalidated Worker lease authorizes + /// restart reconciliation. The lease must remain owned until actual shutdown. + pub(crate) fn open_for_host( + path: &Path, + cwd: PathBuf, + lease: &crate::store::StandaloneWorkerLease, + ) -> Result { + let owner = lease + .verify_current_id() + .map_err(|error| JobServiceError::Configuration(error.to_string()))?; + let jobs = Self::open(path, cwd)?; + jobs.with_store_write(|store| { + store.bind_host_lease(&owner.to_string())?; + // Re-read after opening/migration, immediately before mutating attempts. + let current = lease.verify_current_id().map_err(|_| { + JobStoreError::InvalidTransition("Worker lease ownership changed during Job open") + })?; + if current != owner { + return Err(JobStoreError::AttemptMismatch); + } + store.reconcile_host_restart().map(|_| ()) + })?; + Ok(jobs) + } + + pub(crate) fn reserved_result_job_ids(&self) -> Result, JobServiceError> { + // Retain IDs, not a second queue of immutable bodies outside the owned + // execution barrier while awaiting an earlier Job's cleanup. + Ok(self + .pending()? + .into_iter() + .filter(|saved| { + saved.domain_grant.is_none() + && !saved.version_stopped + && saved.attempt.state == JobAttemptState::Reserved + }) + .map(|saved| saved.request.job_id) + .collect()) + } + + /// The restore caller captures IDs before returning the Host to clients, then + /// runs this once after binding adapters. Granted Jobs keep domain recovery. + pub(crate) async fn resume_reserved_result_jobs( + &self, + ids: Vec, + ) -> Result<(), JobServiceError> { + for id in ids { + self.start(&id).await?; + self.wait(&id).await?; + } + Ok(()) + } + + /// Storage exhaustion is owned by the same execution Task, not a new pending + /// Job state. The diagnostic retains its typed source until a successful Host + /// store write wakes reconciliation; no polling or model retry is scheduled. + pub fn settlement_recovery_diagnostic(&self, job_id: &str) -> Option { + let executions = self.executions.lock().expect("Job executions poisoned"); + let execution = executions.jobs.get(job_id)?; + let issue = execution + .recovery_issue + .lock() + .expect("Job recovery issue poisoned"); + issue.as_ref().map(|original| { + let completion = execution.complete.borrow(); + match completion.as_ref() { + Some(Err(final_error)) => format!("{original}; settlement ended: {final_error}; next action: resolve the retained typed execution/cleanup failure, not another storage retry"), + _ => original.to_string(), + } + }) + } + + /// Called by the existing Host/domain recovery hook when storage is available + /// again. A failed probe does not schedule another budget; a successful write + /// wakes only retained settlement Tasks, never starts another execution. + pub(crate) fn recover_storage(&self) -> Result<(), JobServiceError> { + let attempts: Vec<_> = { + let executions = self.executions.lock().expect("Job executions poisoned"); + executions + .jobs + .iter() + .filter_map(|(id, execution)| { + if execution.task.is_finished() { + return None; + } + execution + .recovery_issue + .lock() + .expect("Job recovery issue poisoned") + .is_some() + .then(|| (id.clone(), execution.attempt_id.clone())) + }) + .collect() + }; + for (id, attempt) in attempts { + self.with_store_write(|store| store.probe_settlement_write(&id, &attempt))?; + } + Ok(()) + } + pub(crate) fn bind_model_client(&self, client: Box) { *self .model_client @@ -157,7 +303,7 @@ impl StandaloneJobs { if executions.closed { return Err(JobServiceError::Closed); } - self.with_store(|store| store.reserve_granted(request, Some(grant))) + self.with_store_write(|store| store.reserve_granted(request, Some(grant))) } fn with_store( @@ -168,6 +314,17 @@ impl StandaloneJobs { Ok(operation(slot.as_mut().ok_or(JobServiceError::Closed)?)?) } + fn with_store_write( + &self, + operation: impl FnOnce(&mut JobStore) -> Result, + ) -> Result { + let value = self.with_store(operation)?; + // A successful commit in this same Host/store is a recovery event, unlike + // an arbitrary read, UI click, timer, or a newly issued replacement Job. + self.storage_recovered.notify_waiters(); + Ok(value) + } + /// Persist intent without starting a model. Exact resend returns the same /// current attempt, including after acknowledgement or cancellation. pub fn request(&self, request: JobRequest) -> Result { @@ -175,7 +332,7 @@ impl StandaloneJobs { if executions.closed { return Err(JobServiceError::Closed); } - Ok(self.with_store(|store| store.reserve(request))?) + Ok(self.with_store_write(|store| store.reserve(request))?) } /// Enumerate historical domain snapshots for a Session, including retired input receipts. @@ -205,7 +362,7 @@ impl StandaloneJobs { { return Err(JobServiceError::Capacity); } - self.with_store(|store| store.reclaim_input(job_id, expected_input_digest, reason)) + self.with_store_write(|store| store.reclaim_input(job_id, expected_input_digest, reason)) } pub fn get(&self, job_id: &str) -> Result { @@ -222,12 +379,12 @@ impl StandaloneJobs { self.with_store(|store| store.unacknowledged()) } - pub(crate) fn unknown_granted(&self) -> Result, JobServiceError> { - self.with_store(|store| store.unknown_granted()) + pub(crate) fn unreleased_granted(&self) -> Result, JobServiceError> { + self.with_store(|store| store.unreleased_granted()) } - /// Release only an Unknown attempt's serialization reservation after the bound domain - /// has reconciled its receipts and durably fenced further effects. Never a generic retry. + /// Release a failed granted attempt's domain reservation only after authoritative + /// reconciliation and independent execution cleanup. Never a generic retry. pub(crate) fn reconcile_granted_attempt( &self, job_id: &str, @@ -238,29 +395,63 @@ impl StandaloneJobs { if executions.closed { return Err(JobServiceError::Closed); } - if executions.jobs.contains_key(job_id) { + if executions.jobs.get(job_id).is_some_and(|run| { + !run.task.is_finished() || !matches!(run.complete.borrow().as_ref(), Some(Ok(()))) + }) { return Err(JobServiceError::Cleanup( "domain reconciliation requires observed execution cleanup".into(), )); } - self.with_store(|store| store.reconcile_granted_attempt(job_id, attempt_id, domain_receipt)) + self.with_store_write(|store| { + store.reconcile_granted_attempt(job_id, attempt_id, domain_receipt) + }) } - /// Explicit new attempt, bounded by immutable intent limits. Unknown - /// outcomes cannot be retried without domain reconciliation/new intent. + pub(crate) fn resume_granted( + &self, + job_id: &str, + attempt_id: &str, + ) -> Result { + let executions = self.executions.lock().expect("Job executions poisoned"); + if executions.closed { + return Err(JobServiceError::Closed); + } + if executions.jobs.get(job_id).is_some_and(|run| { + !run.task.is_finished() || !matches!(run.complete.borrow().as_ref(), Some(Ok(()))) + }) { + return Err(JobServiceError::ShutdownIncomplete); + } + self.with_store_write(|store| store.resume_granted(job_id, attempt_id)) + } + + pub(crate) fn accept_reconciled( + &self, + submission: &JobResultSubmission, + ) -> Result { + let executions = self.executions.lock().expect("Job executions poisoned"); + if executions.closed { + return Err(JobServiceError::Closed); + } + let _effects = executions + .jobs + .get(&submission.job_id) + .map(|run| run.effect_gate.lock().expect("Job effects poisoned")); + self.with_store_write(|store| store.accept_reconciled(submission)) + } + + /// Explicit new result-only attempt, bounded by immutable intent limits and + /// independent shutdown proof. Granted Jobs use domain-owned recovery instead. pub fn retry(&self, job_id: &str) -> Result { let executions = self.executions.lock().expect("Job executions poisoned"); if executions.closed { return Err(JobServiceError::Closed); } - if executions - .jobs - .get(job_id) - .is_some_and(|run| !run.task.is_finished()) - { + if executions.jobs.get(job_id).is_some_and(|run| { + !run.task.is_finished() || !matches!(run.complete.borrow().as_ref(), Some(Ok(()))) + }) { return Err(JobServiceError::Capacity); } - Ok(self.with_store(|store| store.retry(job_id))?) + Ok(self.with_store_write(|store| store.retry(job_id))?) } /// Successful model processing and consumer postprocessing have independent @@ -270,7 +461,7 @@ impl StandaloneJobs { if executions.closed { return Err(JobServiceError::Closed); } - Ok(self.with_store(|store| store.ack(job_id))?) + Ok(self.with_store_write(|store| store.ack(job_id))?) } pub fn cancel(&self, job_id: &str) -> Result { @@ -278,7 +469,11 @@ impl StandaloneJobs { if executions.closed { return Err(JobServiceError::Closed); } - let snapshot = self.with_store(|store| store.cancel(job_id))?; + let _effect = executions + .jobs + .get(job_id) + .map(|run| run.effect_gate.lock().expect("Job effects poisoned")); + let snapshot = self.with_store_write(|store| store.cancel(job_id))?; if let Some(execution) = executions.jobs.get(job_id) { execution.cancel.send_replace(true); } @@ -324,15 +519,35 @@ impl StandaloneJobs { }); let snapshot = self.get(job_id)?; snapshot.load_executable_request()?; + if snapshot.version_stopped + || snapshot.state != JobState::Pending + || snapshot.attempt.state != JobAttemptState::Reserved + { + return Err(JobStoreError::InvalidTransition( + "start requires an unstarted current attempt", + ) + .into()); + } + let effect_gate = Arc::new(Mutex::new(())); + let revoked = Arc::new(AtomicBool::new(false)); let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds)); let deadline = tokio::time::Instant::now() + timeout; let fence: Arc = Arc::new(BoundAttemptFence { + revoked: revoked.clone(), + effect_gate: effect_gate.clone(), store: self.store.clone(), job_id: job_id.into(), attempt_id: snapshot.attempt.attempt_id.clone(), input_digest: snapshot.input_digest.clone(), deadline, }); + let domain_fenced = snapshot.domain_grant.is_none() + || self + .domain + .lock() + .expect("Job domain poisoned") + .as_ref() + .is_some_and(|provider| provider.callbacks_use_attempt_fence()); let grant = match ( &snapshot.domain_grant, self.domain.lock().expect("Job domain poisoned").as_ref(), @@ -356,10 +571,9 @@ impl StandaloneJobs { // Count live resources, not only result states: acceptance can precede // the Internal Worker's cleanup and must not free a concurrency slot. let live = || { - executions - .jobs - .values() - .filter(|run| !run.task.is_finished()) + executions.jobs.values().filter(|run| { + !run.task.is_finished() || !matches!(run.complete.borrow().as_ref(), Some(Ok(()))) + }) }; let count = live().count(); let active_limit = live() @@ -377,28 +591,32 @@ impl StandaloneJobs { } let resource_key = snapshot.request.serialization_key.clone(); if let Some(key) = &resource_key { - if executions - .jobs - .values() - .any(|run| !run.task.is_finished() && run.resource_key.as_ref() == Some(key)) - { + if executions.jobs.values().any(|run| { + run.resource_key.as_ref() == Some(key) + && (!run.task.is_finished() + || !matches!(run.complete.borrow().as_ref(), Some(Ok(())))) + }) { return Err(JobServiceError::Capacity); } } // Retain errors from finished tasks until a caller observes them rather // than silently swallowing store/cleanup failures. if let Some(previous) = executions.jobs.get(job_id) { - if !previous.task.is_finished() { + if !previous.task.is_finished() + || !matches!(previous.complete.borrow().as_ref(), Some(Ok(()))) + { return Err(JobServiceError::Capacity); } } - let snapshot = self.with_store(|store| { + let snapshot = self.with_store_write(|store| { store.dispatch(job_id, &snapshot.attempt.attempt_id, self.capacity) })?; let store = self.store.clone(); let id = job_id.to_string(); let attempt_id = snapshot.attempt.attempt_id.clone(); let sink = Arc::new(BoundResultSink { + storage_recovered: self.storage_recovered.clone(), + revoked: revoked.clone(), store: store.clone(), job_id: id.clone(), attempt_id: attempt_id.clone(), @@ -408,40 +626,92 @@ impl StandaloneJobs { let (cancel, mut cancellation) = watch::channel(false); let (complete_tx, complete) = watch::channel(None); let execution_attempt = attempt_id.clone(); + let execution_gate = effect_gate.clone(); + let ownership = snapshot + .executions + .iter() + .find(|run| run.attempt_id == attempt_id) + .cloned() + .ok_or(JobStoreError::AttemptMismatch)?; + let expected_digest = snapshot.input_digest.clone(); + let owned = Arc::new(Mutex::new(None)); + let retained = owned.clone(); + let recovery_issue = Arc::new(Mutex::new(None)); + let task_issue = recovery_issue.clone(); + let storage_recovered = self.storage_recovered.clone(); let task = tokio::spawn(async move { - let outcome: Result<(), JobServiceError> = async { let (sender_tx, mut sender_rx) = tokio::sync::oneshot::channel(); - let run = prepared.run(attempt_id.clone(), sink, move |sender| { let _ = sender_tx.send(sender); }); + let run = prepared.run_owned(attempt_id.clone(), sink, move |sender| { + let _ = sender_tx.send(sender); + }); tokio::pin!(run); - let reason = tokio::select! { - result = &mut run => { - let (state, detail) = match result { - Ok(_) => (JobAttemptState::Failed, "Worker finished without an accepted structured result".to_string()), - Err(error @ worker::job::JobExecutionError::ResultUnconfirmed(_)) => (JobAttemptState::Unknown, error.to_string()), - Err(error) => (JobAttemptState::Failed, error.to_string()), - }; - finish_if_pending(&store, &id, &attempt_id, state, &detail)?; - return Ok(()); + let mut stop_reason = None; + let result = tokio::select! { + result = &mut run => result, + reason = async { + tokio::select! { + _ = tokio::time::sleep(timeout) => "timeout", + _ = cancellation.wait_for(|cancelled| *cancelled) => "Host cancellation/shutdown", + } + } => { + // The persisted terminal state revokes future domain calls, while the gate + // joins any synchronous call already inside the Host boundary. + { + let _effects = execution_gate.lock().expect("Job effects poisoned"); + revoked.store(true, Ordering::Release); + stop_reason = Some(reason); + // A transient store failure does not cancel ownership of settlement. + // The local revocation and effect gate still fence callbacks; + // retry the exact terminal write after the owned runner returns. + let _ = finish_if_pending(&store, &id, &attempt_id, JobAttemptState::Failed, reason); + } + if let Ok(Ok(sender)) = tokio::time::timeout(Duration::from_secs(1), &mut sender_rx).await { + let _ = sender.try_send(()); + } + match tokio::time::timeout(Duration::from_secs(2), &mut run).await { + Ok(result) => result, + Err(_) => { + // Report the overdue cancellation, but never drop the owned future. + // A later return can supply real shutdown evidence; until then both + // the task and durable execution reservation remain owned. + eprintln!("standalone Job {id} cancellation is still running; ownership retained"); + // Keep waiters attached to this recovery owner. An interim + // cleanup error would stop the Journal dispatcher before a + // later owned return/storage recovery can settle this Job. + run.await + } + } } - _ = tokio::time::sleep(timeout) => "timeout", - _ = cancellation.wait_for(|cancelled| *cancelled) => "Host cancellation/shutdown", }; - // Fence timeout/shutdown before cancelling so a late tool call can - // never turn an interrupted unaccepted execution into success. - let fence_result = finish_if_pending(&store, &id, &attempt_id, JobAttemptState::Unknown, reason); - // Cancellation is routed through Engine; keep the future alive to - // allow normal Feature cleanup. Hard stop is a fenced unknown - // outcome, never an automatic retry. Result acceptance wins races. - if let Ok(sender) = sender_rx.try_recv() { - let _ = sender.try_send(()); - } else if let Ok(Ok(sender)) = tokio::time::timeout(Duration::from_secs(1), &mut sender_rx).await { - let _ = sender.try_send(()); - } - let cleanup = tokio::time::timeout(Duration::from_secs(2), &mut run).await; - fence_result?; - if cleanup.is_err() { return Err(JobServiceError::Cleanup("Internal Worker cancellation deadline exceeded; outcome fenced unknown".into())); } - Ok(()) - }.await; + let allow_unconfirmed = stop_reason.is_none() && tokio::time::Instant::now() < deadline; + *retained.lock().expect("Job owned execution poisoned") = Some(result); + let outcome = recover_owned_settlement( + &storage_recovered, + &task_issue, + || { + let _effects = execution_gate.lock().expect("Job effects poisoned"); + revoked.store(true, Ordering::Release); + let owned = retained.lock().expect("Job owned execution poisoned"); + settle_owned_execution( + &store, + &id, + &attempt_id, + &expected_digest, + &ownership, + domain_fenced, + stop_reason, + allow_unconfirmed, + owned.as_ref().expect("owned execution retained"), + ) + }, + || { + let mut slot = store.lock().expect("Job store poisoned"); + let store = slot.as_mut().ok_or(JobServiceError::Closed)?; + store.probe_settlement_write(&id, &attempt_id)?; + Ok(()) + }, + ) + .await; complete_tx.send_replace(Some( outcome.as_ref().map(|_| ()).map_err(ToString::to_string), )); @@ -453,6 +723,9 @@ impl StandaloneJobs { cancel, complete, task, + _owned: owned, + effect_gate, + recovery_issue, attempt_id: execution_attempt, resource_key, max_concurrent: snapshot.request.limits.max_concurrent_jobs, @@ -515,7 +788,11 @@ impl StandaloneJobs { /// incomplete/error result retains the service resources and must retain the Host lease. pub(crate) fn try_shutdown(&self) -> Result<(), JobServiceError> { self.close(); + self.recover_storage()?; let mut executions = self.executions.lock().expect("Job executions poisoned"); + if executions.shutdown_confirmed { + return Ok(()); + } for execution in executions.jobs.values() { if !execution.task.is_finished() { return Err(JobServiceError::ShutdownIncomplete); @@ -530,6 +807,12 @@ impl StandaloneJobs { } } } + if !self + .with_store(|store| store.execution_obligations())? + .is_empty() + { + return Err(JobServiceError::ShutdownIncomplete); + } executions.jobs.clear(); // Retained external handles share these slots; they cannot retain a live store, // provider or model client after successful owner shutdown. @@ -539,11 +822,13 @@ impl StandaloneJobs { .lock() .expect("Job model transport poisoned") .take(); + executions.shutdown_confirmed = true; Ok(()) } pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> { self.close(); + self.recover_storage()?; let completions: Vec<_> = self .executions .lock() @@ -563,13 +848,188 @@ impl StandaloneJobs { loop { match self.try_shutdown() { // Only the task's final return follows its completion notification. - Err(JobServiceError::ShutdownIncomplete) => tokio::task::yield_now().await, + Err(JobServiceError::ShutdownIncomplete) => { + let has_running_task = self + .executions + .lock() + .expect("Job executions poisoned") + .jobs + .values() + .any(|execution| !execution.task.is_finished()); + if !has_running_task { + // Saved obligations cannot be discharged by yielding or an absent map. + return Err(JobServiceError::ShutdownIncomplete); + } + tokio::task::yield_now().await; + } result => return result, } } } } +// Retry only persistence, never dispatch, domain operations, or Worker shutdown. +// The owning execution Task retains the exact typed result throughout this finite +// recovery window, including when all observers drop their wait futures. +pub(crate) async fn recover_settlement( + mut settle: impl FnMut() -> Result<(), JobServiceError>, +) -> Result<(), JobServiceError> { + const DELAYS_MS: [u64; 5] = [100, 200, 400, 800, 1600]; + for pass in 0..=DELAYS_MS.len() { + match settle() { + Ok(()) => return Ok(()), + Err(error) if transient_settlement_storage_error(&error) => { + let Some(delay) = DELAYS_MS.get(pass) else { + return Err(JobServiceError::SettlementRecoveryExhausted { + attempts: pass + 1, + source: Box::new(error), + }); + }; + tokio::time::sleep(Duration::from_millis(*delay)).await; + } + Err(error) => return Err(error), + } + } + unreachable!("finite settlement retry returns on its final pass") +} + +pub(crate) async fn recover_owned_settlement( + storage_recovered: &Notify, + issue: &Mutex>, + mut settle: impl FnMut() -> Result<(), JobServiceError>, + mut probe: impl FnMut() -> Result<(), JobServiceError>, +) -> Result<(), JobServiceError> { + loop { + match recover_settlement(&mut settle).await { + Err(error @ JobServiceError::SettlementRecoveryExhausted { .. }) => { + { + let mut saved = issue.lock().expect("Job recovery issue poisoned"); + if saved.is_none() { + *saved = Some(error); + } + } + // The execution owner also owns recovery detection; no user action, + // other Job or detached monitor is needed. A timer only schedules an + // exact-owner storage probe, NEVER another settlement budget or effect. + // Only a committed probe permits settlement to resume. Permanent + // errors and stale ownership stop probing with the typed cause intact. + let mut delay = Duration::from_secs(1); + loop { + tokio::select! { + _ = storage_recovered.notified() => {}, + _ = tokio::time::sleep(delay) => {}, + } + match probe() { + Ok(()) => break, + Err(error) if transient_settlement_storage_error(&error) => { + delay = (delay * 2).min(Duration::from_secs(30)); + } + Err(error) => return Err(error), + } + } + } + result => { + if result.is_ok() { + issue.lock().expect("Job recovery issue poisoned").take(); + } + return result; + } + } + } +} + +fn transient_settlement_storage_error(error: &JobServiceError) -> bool { + matches!(error, + JobServiceError::Store(JobStoreError::Sqlite(rusqlite::Error::SqliteFailure(error, _))) + if matches!(error.code, + rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked + | rusqlite::ErrorCode::SystemIoFailure | rusqlite::ErrorCode::DiskFull + | rusqlite::ErrorCode::CannotOpen)) +} + +fn settle_owned_execution( + store: &Mutex>, + id: &str, + attempt: &str, + input_digest: &str, + ownership: &JobExecutionOwnership, + domain_fenced: bool, + stop_reason: Option<&str>, + allow_unconfirmed: bool, + result: &Result, +) -> Result<(), JobServiceError> { + let detail = match result { + Ok(execution) => execution + .result + .as_ref() + .err() + .map(ToString::to_string) + .unwrap_or_else(|| "Worker finished without an accepted structured result".into()), + Err(error) => error.to_string(), + }; + // Validate the saved owner before any terminal write. A stale result/cleanup + // must not finish a later attempt, even if that later attempt is still pending. + let mut slot = store.lock().expect("Job store poisoned"); + let store = slot.as_mut().ok_or(JobServiceError::Closed)?; + store.check_execution_binding(id, attempt, input_digest, ownership)?; + let snapshot = store.get(id)?; + if let Ok(execution) = result { + if execution.job_id != id + || execution.attempt_id != attempt + || execution.input_digest != input_digest + || execution.run_id.to_string() != ownership.run_id + { + return Err(JobStoreError::AttemptMismatch.into()); + } + let submission = execution + .result + .as_ref() + .ok() + .map(|accepted| &accepted.submission) + .or_else(|| { + (allow_unconfirmed || snapshot.state == JobState::Completed) + .then(|| execution.unconfirmed_submission()) + .flatten() + }); + if let Some(submission) = submission { + if submission.job_id != id + || submission.attempt_id != attempt + || submission.input_digest != input_digest + { + return Err(JobStoreError::AttemptMismatch.into()); + } + // Reconcile only the sealed, already-finalized payload. In particular, + // never rerun a non-idempotent domain finalizer after sink response loss. + store.accept(submission)?; + } + } + let snapshot = store.get(id)?; + if snapshot.state == JobState::Pending { + store.finish( + id, + attempt, + JobAttemptState::Failed, + Some(stop_reason.unwrap_or(&detail)), + )?; + } + let execution = result + .as_ref() + .map_err(|error| JobServiceError::Cleanup(error.to_string()))?; + if !domain_fenced + || !execution.feature_shutdown_complete() + || execution.worker_failure().is_some() + { + return Err(JobServiceError::Cleanup(format!( + "owned Worker shutdown unconfirmed ({} Feature shutdown failures); typed execution retained", + execution.feature_shutdown_failures().len() + ))); + } + // The caller holds the domain effect gate, run_owned has joined/revoked every + // result/finalizer clone, and terminal state rejects all future domain entries. + store.confirm_execution_cleanup(id, ownership, &format!("owned-run:{}", execution.run_id))?; + Ok(()) +} + fn finish_if_pending( store: &Mutex>, id: &str, @@ -587,6 +1047,8 @@ fn finish_if_pending( } struct BoundResultSink { + storage_recovered: Arc, + revoked: Arc, store: Arc>>, job_id: String, attempt_id: String, @@ -607,17 +1069,22 @@ impl worker::job::JobResultSink for BoundResultSink { .map_err(|_| "Job store poisoned".to_string())?; let store = slot.as_mut().ok_or_else(|| "Job Host closed".to_string())?; let snapshot = store.get(&self.job_id).map_err(|error| error.to_string())?; - if tokio::time::Instant::now() >= self.deadline && snapshot.state != JobState::Completed { - return Err("Job result deadline expired".into()); + if (self.revoked.load(Ordering::Acquire) || tokio::time::Instant::now() >= self.deadline) + && snapshot.state != JobState::Completed + { + return Err("Job result capability revoked or deadline expired".into()); } store .accept(&submission) - .map(|_| ()) - .map_err(|error| error.to_string()) + .map_err(|error| error.to_string())?; + self.storage_recovered.notify_waiters(); + Ok(()) } } struct BoundAttemptFence { + revoked: Arc, + effect_gate: Arc>, store: Arc>>, job_id: String, attempt_id: String, @@ -625,7 +1092,18 @@ struct BoundAttemptFence { deadline: tokio::time::Instant, } impl JobAttemptFence for BoundAttemptFence { + fn enter(&self) -> Result, String> { + let guard = self + .effect_gate + .lock() + .map_err(|_| "Job effects poisoned".to_string())?; + self.ensure_live()?; + Ok(Box::new(guard)) + } fn ensure_live(&self) -> Result<(), String> { + if self.revoked.load(Ordering::Acquire) { + return Err("Job domain capability revoked".into()); + } if tokio::time::Instant::now() >= self.deadline { return Err("Job domain deadline expired".into()); } diff --git a/crates/standalone/src/jobs/tests.rs b/crates/standalone/src/jobs/tests.rs index 06325c054..80d554f0a 100644 --- a/crates/standalone/src/jobs/tests.rs +++ b/crates/standalone/src/jobs/tests.rs @@ -21,6 +21,8 @@ fn request(id: &str) -> JobRequest { } fn sink(jobs: &StandaloneJobs, id: &str, attempt: &str) -> BoundResultSink { BoundResultSink { + storage_recovered: jobs.storage_recovered.clone(), + revoked: Arc::new(AtomicBool::new(false)), store: jobs.store.clone(), job_id: id.into(), attempt_id: attempt.into(), @@ -75,7 +77,9 @@ impl LlmClient for DomainTestClient { ) -> Result> + Send>>, ClientError> { self.requests.lock().unwrap().push(request); self.entered.notify_one(); - if let Some(release) = &self.failure_release { + if let Some(release) = &self.failure_release + && self.scripts.lock().unwrap().is_empty() + { release.notified().await; return Err(ClientError::Config( "known scripted provider failure".into(), @@ -106,6 +110,10 @@ fn domain_request(id: &str) -> JobRequest { intent } impl JobDomainProvider for RecordingDomainProvider { + fn callbacks_use_attempt_fence(&self) -> bool { + true + } + fn grant( &self, snapshot: &JobSnapshot, @@ -133,6 +141,269 @@ async fn await_job(jobs: &StandaloneJobs, id: &str) -> JobSnapshot { .unwrap() } +// The model and stopping hook are the only substituted boundaries. Dispatch, result +// persistence, the owned Internal Worker and service cleanup remain production code. +#[derive(Clone, Default)] +struct ControlledShutdown { + entered: Arc, + release: Arc, + block: bool, + fail: bool, +} +impl worker::feature::FeatureModule for ControlledShutdown { + fn descriptor(&self) -> worker::feature::FeatureDescriptor { + worker::feature::FeatureDescriptor::builtin("job-cleanup-test", "Controlled shutdown") + .with_hook(worker::feature::HookDeclaration::new( + "stop", + worker::feature::FeatureHookPoint::WorkerStopping, + )) + } + fn install( + &self, + context: &mut worker::feature::FeatureInstallContext<'_>, + ) -> Result<(), worker::feature::FeatureInstallError> { + context.hooks().add_worker_stopping( + "stop", + worker::hook::HookExecutionPolicy::fail_closed(), + self.clone(), + ) + } +} +#[async_trait] +impl worker::hook::Hook for ControlledShutdown { + async fn call( + &self, + _: &worker::hook::WorkerStoppingContext, + ) -> Result<(), worker::hook::HookError> { + self.entered.notify_one(); + if self.block { + self.release.notified().await; + } + if self.fail { + Err(worker::hook::HookError::new( + worker::hook::HookErrorCategory::Internal, + "scripted Feature shutdown failure", + )) + } else { + Ok(()) + } + } +} +impl JobDomainProvider for ControlledShutdown { + fn callbacks_use_attempt_fence(&self) -> bool { + true + } + + fn grant( + &self, + _: &JobSnapshot, + _: Arc, + ) -> Result { + Ok(worker::job::JobFeatureGrant { + features: worker::feature::FeatureRegistryBuilder::new().with_module(self.clone()), + satisfied_requirements: vec!["feature.subjektiv"], + ..Default::default() + }) + } +} +struct ReleaseShutdownOnDrop(Arc); +impl Drop for ReleaseShutdownOnDrop { + fn drop(&mut self) { + self.0.notify_one(); + } +} + +#[tokio::test] +async fn overdue_timeout_keeps_owned_future_and_capacity_until_feature_shutdown_returns() { + let temp = tempfile::tempdir().unwrap(); + let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap(); + let hook = ControlledShutdown { + block: true, + ..Default::default() + }; + let _release_on_failure = ReleaseShutdownOnDrop(hook.release.clone()); + jobs.bind_domain(Arc::new(hook.clone())).unwrap(); + let mut intent = domain_request("overdue"); + intent.limits.timeout_seconds = 1; + intent.limits.max_concurrent_jobs = 1; + intent.serialization_key = Some("owned-resource".into()); + let reserved = jobs + .request_granted(intent.clone(), domain_grant("overdue")) + .unwrap(); + jobs.request(request("next")).unwrap(); + let model = DomainTestClient { + stall: true, + ..Default::default() + }; + jobs.start_with_model_client("overdue", model.clone()) + .await + .unwrap(); + await_model(&model).await; + let dispatched = jobs.get("overdue").unwrap(); + assert_eq!(dispatched.executions.len(), 1); + let ownership = dispatched.executions[0].clone(); + assert_eq!( + ownership.run_id, + worker::job::execution_run_id(&intent, &reserved.attempt.attempt_id) + .unwrap() + .to_string() + ); + assert!(!ownership.writes_revoked); + assert!(ownership.cleanup_receipt.is_none()); + tokio::time::timeout(Duration::from_secs(5), hook.entered.notified()) + .await + .unwrap(); + // Grace expiry is diagnostic only: a dropped wait must not detach the owner + // or make the Journal dispatcher abandon the eventual settled result. + assert!( + tokio::time::timeout(Duration::from_secs(3), jobs.wait("overdue")) + .await + .is_err() + ); + let mut completion = { + let executions = jobs.executions.lock().unwrap(); + let execution = &executions.jobs["overdue"]; + assert!(!execution.task.is_finished()); + assert!(execution._owned.lock().unwrap().is_none()); + execution.complete.clone() + }; + let failed = jobs.get("overdue").unwrap(); + assert_eq!(failed.state, JobState::Failed); + assert_eq!(failed.attempt.failure.as_deref(), Some("timeout")); + assert_eq!(failed.executions, vec![ownership.clone()]); + assert!(jobs.reserve_quiescent_closure().is_err()); + assert!(jobs.retry("overdue").is_err()); + assert!( + jobs.resume_granted("overdue", &failed.attempt.attempt_id) + .is_err() + ); + assert!(matches!( + jobs.start_with_model_client("next", DomainTestClient::success()) + .await, + Err(JobServiceError::Capacity) + )); + let mut conflicting = request("conflicting"); + conflicting.serialization_key = intent.serialization_key.clone(); + assert!(jobs.request(conflicting).is_err()); + hook.release.notify_one(); + tokio::time::timeout( + Duration::from_secs(5), + completion.wait_for(|state| matches!(state, Some(Ok(())))), + ) + .await + .unwrap() + .unwrap(); + let cleaned = await_job(&jobs, "overdue").await; + assert!(cleaned.executions_released()); + assert_eq!(cleaned.request, reserved.request); + assert_eq!(cleaned.attempt, failed.attempt); + assert_eq!(cleaned.executions[0].owner_id, ownership.owner_id); + assert_eq!(cleaned.executions[0].run_id, ownership.run_id); + assert_eq!( + cleaned.executions[0].cleanup_receipt, + Some(format!("owned-run:{}", ownership.run_id)) + ); + jobs.start_with_model_client("next", DomainTestClient::success()) + .await + .unwrap(); + assert_eq!(await_job(&jobs, "next").await.state, JobState::Completed); + jobs.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn accepted_result_with_feature_shutdown_failure_retains_typed_owner_and_blocks_release() { + let temp = tempfile::tempdir().unwrap(); + let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap(); + jobs.bind_domain(Arc::new(ControlledShutdown { + fail: true, + ..Default::default() + })) + .unwrap(); + jobs.request_granted( + domain_request("cleanup-error"), + domain_grant("cleanup-error"), + ) + .unwrap(); + jobs.start_with_model_client("cleanup-error", DomainTestClient::success()) + .await + .unwrap(); + assert!(matches!( + tokio::time::timeout(Duration::from_secs(5), jobs.wait("cleanup-error")) + .await + .unwrap(), + Err(JobServiceError::Cleanup(_)) + )); + let accepted = jobs.get("cleanup-error").unwrap(); + assert_eq!(accepted.state, JobState::Completed); + assert_eq!(accepted.result, Some(json!({"valid":true}))); + assert!(!accepted.executions_released()); + { + let executions = jobs.executions.lock().unwrap(); + let owned = executions.jobs["cleanup-error"]._owned.lock().unwrap(); + let owned = owned.as_ref().unwrap().as_ref().unwrap(); + assert!(!owned.feature_shutdown_complete()); + assert!(!owned.feature_shutdown_failures().is_empty()); + assert_eq!(owned.run_id.to_string(), accepted.executions[0].run_id); + } + assert!(jobs.reserve_quiescent_closure().is_err()); + assert!( + jobs.reclaim_input( + "cleanup-error", + &accepted.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + assert!(matches!( + jobs.shutdown().await, + Err(JobServiceError::Cleanup(_)) + )); + assert!(jobs.store.lock().unwrap().is_some()); + assert_eq!(jobs.get("cleanup-error").unwrap(), accepted); +} + +#[tokio::test] +async fn accepted_result_with_worker_failure_does_not_confirm_execution_cleanup() { + let temp = tempfile::tempdir().unwrap(); + let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap(); + jobs.request(request("post-result-error")).unwrap(); + let release = Arc::new(tokio::sync::Notify::new()); + let model = DomainTestClient { + failure_release: Some(release.clone()), + ..DomainTestClient::success() + }; + model.scripts.lock().unwrap().pop_back(); + release.notify_one(); + jobs.start_with_model_client("post-result-error", model) + .await + .unwrap(); + assert!(matches!( + tokio::time::timeout(Duration::from_secs(5), jobs.wait("post-result-error")) + .await + .unwrap(), + Err(JobServiceError::Cleanup(_)) + )); + let accepted = jobs.get("post-result-error").unwrap(); + assert_eq!(accepted.state, JobState::Completed); + assert_eq!(accepted.result, Some(json!({"valid":true}))); + assert!(!accepted.executions_released()); + { + let executions = jobs.executions.lock().unwrap(); + let owned = executions.jobs["post-result-error"]._owned.lock().unwrap(); + let owned = owned.as_ref().unwrap().as_ref().unwrap(); + assert!(owned.feature_shutdown_complete()); + assert!(owned.worker_failure().is_some()); + assert!(owned.result.is_ok()); + } + assert!(jobs.retry("post-result-error").is_err()); + assert!(jobs.reserve_quiescent_closure().is_err()); + assert!(matches!( + jobs.shutdown().await, + Err(JobServiceError::Cleanup(_)) + )); + assert!(jobs.store.lock().unwrap().is_some()); +} + #[tokio::test] async fn pending_domain_grants_survive_service_reopen_and_remain_immutable() { let temp = tempfile::tempdir().unwrap(); @@ -285,12 +556,13 @@ async fn domain_fence_is_live_only_for_current_dispatched_attempt_and_revoked_on StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap(); let provider = Arc::new(RecordingDomainProvider::default()); jobs.bind_domain(provider.clone()).unwrap(); + let mut intent = domain_request("fenced"); + intent.limits.timeout_seconds = 1; let reserved = jobs - .request_granted(domain_request("fenced"), domain_grant("fenced")) + .request_granted(intent, domain_grant("fenced")) .unwrap(); - let failure_release = Arc::new(tokio::sync::Notify::new()); let first = DomainTestClient { - failure_release: Some(failure_release.clone()), + stall: true, ..Default::default() }; jobs.start_with_model_client("fenced", first.clone()) @@ -299,13 +571,27 @@ async fn domain_fence_is_live_only_for_current_dispatched_attempt_and_revoked_on await_model(&first).await; let old_fence = provider.0.lock().unwrap()[0].1.clone(); old_fence.ensure_live().unwrap(); - failure_release.notify_one(); let failed = await_job(&jobs, "fenced").await; assert_eq!(failed.state, JobState::Failed); assert!(failed.result.is_none()); assert!(old_fence.ensure_live().is_err()); - // Only known failure permits a retry; cancellation remains terminal. - let retried = jobs.retry("fenced").unwrap(); + assert!(failed.executions_released()); + assert!( + jobs.retry("fenced").is_err(), + "generic retry cannot bypass the domain" + ); + jobs.reconcile_granted_attempt( + "fenced", + &failed.attempt.attempt_id, + "test-domain-reconciled", + ) + .unwrap(); + let retried = jobs + .resume_granted("fenced", &failed.attempt.attempt_id) + .unwrap(); + assert_eq!(retried.request, reserved.request); + assert_eq!(retried.input_digest, reserved.input_digest); + assert_eq!(retried.attempt.number, 2); assert_ne!(retried.attempt.attempt_id, reserved.attempt.attempt_id); assert_eq!(retried.domain_grant, reserved.domain_grant); assert!(old_fence.ensure_live().is_err()); @@ -376,6 +662,263 @@ async fn bound_domain_does_not_change_root_generic_result_only_execution() { jobs.shutdown().await.unwrap(); } +#[tokio::test] +async fn host_restart_rejects_live_lease_then_fails_only_unconfirmed_dispatch_without_cleanup() { + // Real Worker lease admission and Job SQLite; no Worker/process is spawned + // for the interrupted rows. Releasing a lease is NOT execution drain proof. + use crate::store::{StaleLeasePolicy, StandaloneStoreError, StandaloneWorkerStore}; + let temp = tempfile::tempdir().unwrap(); + let workers = StandaloneWorkerStore::open(&temp.path().join("state")).unwrap(); + let allocation = workers + .allocate(temp.path(), StaleLeasePolicy::Reject) + .unwrap(); + let worker = allocation.worker_id(); + let path = workers.jobs_path(worker); + let lease = allocation.into_lease(); + let prior = StandaloneJobs::open_for_host(&path, temp.path().into(), &lease).unwrap(); + let reserved = prior.request(request("interrupted")).unwrap(); + let dispatched = prior + .with_store_write(|store| store.dispatch("interrupted", &reserved.attempt.attempt_id, 8)) + .unwrap(); + let receipt = prior.request(request("accepted")).unwrap(); + prior + .with_store_write(|store| store.dispatch("accepted", &receipt.attempt.attempt_id, 8)) + .unwrap(); + let accepted = prior + .with_store_write(|store| store.accept(&result("accepted", &receipt.attempt.attempt_id))) + .unwrap(); + let unstarted = prior.request(request("unstarted")).unwrap(); + assert!(matches!( + workers.acquire_lease(worker, StaleLeasePolicy::Recover), + Err(StandaloneStoreError::WorkerLeased(_)) + )); + assert!( + StandaloneJobs::open_for_host(&path, temp.path().into(), &lease).is_err(), + "same live lease/empty local map is not a restart" + ); + assert_eq!(prior.get("interrupted").unwrap(), dispatched); + let old_sink = sink(&prior, "interrupted", &reserved.attempt.attempt_id); + lease.release().unwrap(); + let replacement_lease = workers + .acquire_lease(worker, StaleLeasePolicy::Recover) + .unwrap(); + let reopened = + StandaloneJobs::open_for_host(&path, temp.path().into(), &replacement_lease).unwrap(); + let failed = reopened.get("interrupted").unwrap(); + assert_eq!(failed.state, JobState::Failed); + assert_eq!(failed.attempt.state, JobAttemptState::Failed); + assert_eq!(failed.attempt.attempt_id, reserved.attempt.attempt_id); + assert!( + failed + .attempt + .failure + .as_deref() + .unwrap() + .starts_with("host_restart_execution_unconfirmed:") + ); + assert!(!failed.version_stopped); + assert_eq!(failed.request, reserved.request); + assert_eq!(failed.executions, dispatched.executions); + assert!(!failed.executions_released()); + assert!( + old_sink + .submit(result("interrupted", &reserved.attempt.attempt_id)) + .is_err() + ); + assert!(reopened.retry("interrupted").is_err()); + assert_eq!(reopened.get("accepted").unwrap(), accepted); + assert_eq!(reopened.get("unstarted").unwrap(), unstarted); + reopened.bind_model_client(Box::new(DomainTestClient::success())); + tokio::time::timeout( + Duration::from_secs(5), + reopened.resume_reserved_result_jobs(reopened.reserved_result_job_ids().unwrap()), + ) + .await + .unwrap() + .unwrap(); + let resumed = reopened.get("unstarted").unwrap(); + assert_eq!(resumed.state, JobState::Completed); + assert_eq!(resumed.attempt.attempt_id, unstarted.attempt.attempt_id); + assert_eq!(resumed.attempt.number, 1); + assert!(!resumed.version_stopped); + assert_eq!(reopened.get("interrupted").unwrap(), failed); + assert!(matches!( + reopened.try_shutdown(), + Err(JobServiceError::ShutdownIncomplete) + )); + // The fixture never had a real prior runner to join; do not forge its receipt. +} + +struct UnconfirmedSink { + bound: BoundResultSink, + commit_before_error: bool, +} +impl JobResultSink for UnconfirmedSink { + fn submit(&self, submission: JobResultSubmission) -> Result<(), String> { + if self.commit_before_error { + self.bound.submit(submission)?; + } + Err("injected result persistence/acknowledgement outage".into()) + } +} + +#[tokio::test] +async fn sealed_non_journal_result_recovers_store_or_ack_loss_without_running_model_again() { + // Real owned Worker, result Feature and temporary SQLite store; only the + // model and sink acknowledgement boundary are substituted. No process death. + for committed in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let jobs = + StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap(); + let intent = request("receipt"); + let reserved = jobs.request(intent.clone()).unwrap(); + let dispatched = jobs + .with_store(|store| store.dispatch("receipt", &reserved.attempt.attempt_id, 8)) + .unwrap(); + let model = DomainTestClient::success(); + let prepared = + worker::job::prepare_job(&intent, temp.path(), Some(Box::new(model.clone()))).unwrap(); + let owned = tokio::time::timeout( + Duration::from_secs(5), + prepared.run_owned( + reserved.attempt.attempt_id.clone(), + Arc::new(UnconfirmedSink { + bound: sink(&jobs, "receipt", &reserved.attempt.attempt_id), + commit_before_error: committed, + }), + |_| {}, + ), + ) + .await + .unwrap() + .unwrap(); + assert!(matches!( + owned.result, + Err(worker::job::JobExecutionError::ResultUnconfirmed(_)) + )); + assert!(owned.feature_shutdown_complete()); + assert_eq!( + owned.unconfirmed_submission(), + Some(&result("receipt", &reserved.attempt.attempt_id)) + ); + assert_eq!( + jobs.get("receipt").unwrap().state, + if committed { + JobState::Completed + } else { + JobState::Pending + } + ); + let model_calls = model.requests.lock().unwrap().len(); + let owned = Ok(owned); + recover_settlement(|| { + settle_owned_execution( + &jobs.store, + "receipt", + &reserved.attempt.attempt_id, + &reserved.input_digest, + &dispatched.executions[0], + true, + None, + true, + &owned, + ) + }) + .await + .unwrap(); + let saved = jobs.get("receipt").unwrap(); + assert_eq!(saved.state, JobState::Completed); + assert_eq!(saved.result, Some(json!({"valid":true}))); + assert_eq!(saved.attempt.attempt_id, reserved.attempt.attempt_id); + assert_eq!(saved.attempt.number, 1); + assert!(saved.executions_released()); + assert_eq!(model.requests.lock().unwrap().len(), model_calls); + jobs.shutdown().await.unwrap(); + } +} + +#[tokio::test] +async fn stale_settlement_is_not_retried_and_cannot_fail_a_later_attempt() { + let temp = tempfile::tempdir().unwrap(); + let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap(); + let reserved = jobs.request(request("stale")).unwrap(); + let dispatched = jobs + .with_store(|store| store.dispatch("stale", &reserved.attempt.attempt_id, 8)) + .unwrap(); + let owner = dispatched.executions[0].clone(); + jobs.with_store(|store| { + store.finish( + "stale", + &reserved.attempt.attempt_id, + JobAttemptState::Failed, + Some("first failure"), + )?; + store.confirm_execution_cleanup("stale", &owner, "fixture-owned-run-joined") + }) + .unwrap(); + let next = jobs.retry("stale").unwrap(); + let mut calls = 0; + let outcome = recover_settlement(|| { + calls += 1; + settle_owned_execution( + &jobs.store, + "stale", + &reserved.attempt.attempt_id, + &reserved.input_digest, + &owner, + true, + None, + false, + &Err(worker::job::JobExecutionError::MissingResult), + ) + }) + .await; + assert!(matches!( + outcome, + Err(JobServiceError::Store(JobStoreError::AttemptMismatch)) + )); + assert_eq!(calls, 1); + assert_eq!(jobs.get("stale").unwrap(), next); + jobs.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn local_revocation_fences_callbacks_and_results_even_when_terminal_write_is_unavailable() { + let temp = tempfile::tempdir().unwrap(); + let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap(); + let reserved = jobs.request(request("revoked")).unwrap(); + let dispatched = jobs + .with_store(|store| store.dispatch("revoked", &reserved.attempt.attempt_id, 8)) + .unwrap(); + let revoked = Arc::new(AtomicBool::new(false)); + let fence = BoundAttemptFence { + revoked: revoked.clone(), + effect_gate: Arc::new(Mutex::new(())), + store: jobs.store.clone(), + job_id: "revoked".into(), + attempt_id: reserved.attempt.attempt_id.clone(), + input_digest: reserved.input_digest.clone(), + deadline: tokio::time::Instant::now() + Duration::from_secs(10), + }; + let mut sink = sink(&jobs, "revoked", &reserved.attempt.attempt_id); + sink.revoked = revoked.clone(); + assert!(fence.enter().is_ok()); + // Fault boundary: the cancellation's terminal write has not committed. + // No DB terminal state or elapsed deadline is available as revocation proof. + revoked.store(true, Ordering::Release); + assert!(fence.enter().is_err()); + assert!( + sink.submit(result("revoked", &reserved.attempt.attempt_id)) + .is_err() + ); + assert_eq!(jobs.get("revoked").unwrap(), dispatched); + assert!(!dispatched.executions_released()); + assert!(matches!( + jobs.shutdown().await, + Err(JobServiceError::ShutdownIncomplete) + )); +} + #[tokio::test] async fn host_result_capability_cannot_cross_job_attempt_digest_or_deadline() { let temp = tempfile::tempdir().unwrap(); @@ -418,12 +961,13 @@ async fn host_result_capability_cannot_cross_job_attempt_digest_or_deadline() { changed.result = serde_json::json!({"valid":false}); assert!(expired.submit(changed).is_err()); assert_eq!(jobs.get("second").unwrap().state, JobState::Pending); - jobs.shutdown().await.unwrap(); - assert!( - bound - .submit(result("first", &first.attempt.attempt_id)) - .is_err() - ); + assert!(matches!( + jobs.shutdown().await, + Err(JobServiceError::ShutdownIncomplete) + )); + assert!(jobs.store.lock().unwrap().is_some()); + assert!(!jobs.get("first").unwrap().executions_released()); + assert!(!jobs.get("second").unwrap().executions_released()); } #[tokio::test] @@ -437,17 +981,18 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han &jobs.store, "cancelled", &cancelled.attempt.attempt_id, - JobAttemptState::Unknown, + JobAttemptState::Failed, "cleanup", ) .unwrap(); assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled); let accepted = jobs.request(request("accepted")).unwrap(); - jobs.with_store(|store| store.dispatch("accepted", &accepted.attempt.attempt_id, 8)) - .unwrap(); - sink(&jobs, "accepted", &accepted.attempt.attempt_id) - .submit(result("accepted", &accepted.attempt.attempt_id)) + jobs.start_with_model_client("accepted", DomainTestClient::success()) + .await .unwrap(); + let completed = await_job(&jobs, "accepted").await; + assert!(completed.executions_released()); + let bound = sink(&jobs, "accepted", &accepted.attempt.attempt_id); finish_if_pending( &jobs.store, "accepted", @@ -459,6 +1004,15 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han assert_eq!(jobs.get("accepted").unwrap().state, JobState::Completed); jobs.shutdown().await.unwrap(); assert!(jobs.store.lock().unwrap().is_none()); + // Repeated close uses the retained proof of this service's real shutdown, + // not an absent store/map as a newly inferred completion. + jobs.try_shutdown().unwrap(); + jobs.shutdown().await.unwrap(); + assert!( + bound + .submit(result("accepted", &accepted.attempt.attempt_id)) + .is_err() + ); for operation in [ jobs.cancel("cancelled"), jobs.acknowledge("accepted"), @@ -469,8 +1023,17 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han } } +#[test] +fn absent_store_and_empty_execution_map_do_not_create_shutdown_confirmation() { + let temp = tempfile::tempdir().unwrap(); + let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap(); + jobs.close(); + jobs.store.lock().unwrap().take(); + assert!(matches!(jobs.try_shutdown(), Err(JobServiceError::Closed))); +} + #[tokio::test] -async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() { +async fn cancelled_shutdown_waiter_keeps_execution_barrier_and_failed_input_on_reopen() { // Actual generic runner + SQLite; only the provider stream is scripted/blocked. let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("jobs.sqlite3"); @@ -505,12 +1068,21 @@ async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() { .unwrap(); assert!(jobs.store.lock().unwrap().is_none()); let reopened = StandaloneJobs::open(&path, temp.path().into()).unwrap(); - let unknown = reopened.get(&reserved.request.job_id).unwrap(); - assert_eq!(unknown.attempt.state, JobAttemptState::Unknown); - assert_eq!(unknown.input_disposition, JobInputDisposition::Available); - assert_eq!(unknown.request, reserved.request); - assert_eq!(unknown.input_digest, reserved.input_digest); - assert!(reopened.retry(&unknown.request.job_id).is_err()); + let stopped = reopened.get(&reserved.request.job_id).unwrap(); + assert_eq!(stopped.state, JobState::Failed); + assert_eq!(stopped.attempt.state, JobAttemptState::Failed); + assert_eq!( + stopped.attempt.failure.as_deref(), + Some("Host cancellation/shutdown") + ); + assert_eq!(stopped.input_disposition, JobInputDisposition::Available); + assert_eq!(stopped.request, reserved.request); + assert_eq!(stopped.input_digest, reserved.input_digest); + assert!(stopped.executions_released()); + assert!(!stopped.version_stopped); + let retry = reopened.retry(&stopped.request.job_id).unwrap(); + assert_eq!(retry.attempt.number, 2); + assert_eq!(retry.request, reserved.request); reopened.shutdown().await.unwrap(); } @@ -528,6 +1100,8 @@ async fn input_retirement_service_fences_live_attempt_and_preserves_session_rece jobs.with_store(|store| store.dispatch("input-retirement", &reserved.attempt.attempt_id, 8)) .unwrap(); let fence = BoundAttemptFence { + revoked: Arc::new(AtomicBool::new(false)), + effect_gate: Arc::new(Mutex::new(())), store: jobs.store.clone(), job_id: "input-retirement".into(), attempt_id: reserved.attempt.attempt_id.clone(), @@ -567,5 +1141,10 @@ async fn input_retirement_service_fences_live_attempt_and_preserves_session_rece .is_empty() ); assert_eq!(jobs.request_granted(original, grant).unwrap(), erased); - jobs.shutdown().await.unwrap(); + assert!(matches!( + jobs.shutdown().await, + Err(JobServiceError::ShutdownIncomplete) + )); + assert!(jobs.store.lock().unwrap().is_some()); + assert!(!jobs.get("input-retirement").unwrap().executions_released()); } diff --git a/crates/standalone/src/store.rs b/crates/standalone/src/store.rs index bee8a7421..457233f06 100644 --- a/crates/standalone/src/store.rs +++ b/crates/standalone/src/store.rs @@ -523,6 +523,10 @@ impl StandaloneWorkerAllocation { &self.cwd } + pub(crate) fn lease(&self) -> &StandaloneWorkerLease { + &self.lease + } + pub fn into_lease(self) -> StandaloneWorkerLease { self.lease } @@ -536,6 +540,30 @@ pub struct StandaloneWorkerLease { } impl StandaloneWorkerLease { + /// Revalidate this already-held lease before Job recovery. This neither acquires a lease + /// nor proves any previous execution or Host effect has ended. + pub(crate) fn verify_current_id(&self) -> Result { + if self.released { + return Err(StandaloneStoreError::LeaseOwnershipLost); + } + let parent = self.path.parent().expect("lease parent"); + let _guard = LeaseMutationGuard::acquire(parent)?; + let metadata = fs::symlink_metadata(&self.path).map_err(StandaloneStoreError::Io)?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(StandaloneStoreError::LeaseOwnershipLost); + } + let bytes = fs::read(&self.path).map_err(StandaloneStoreError::Io)?; + let current: LeaseRecord = + serde_json::from_slice(&bytes).map_err(StandaloneStoreError::Json)?; + if current.lease_id != self.lease_id + || current.pid != std::process::id() + || current.liveness() != LeaseLiveness::Live + { + return Err(StandaloneStoreError::LeaseOwnershipLost); + } + Ok(current.lease_id) + } + pub fn release(mut self) -> Result<(), StandaloneStoreError> { self.release_inner() } @@ -900,6 +928,48 @@ permission = "write" ); } + #[test] + fn lease_verification_is_read_only_and_rejects_replacement_or_changed_process_identity() { + let root = tempfile::tempdir().unwrap(); + let store = StandaloneWorkerStore::open(root.path().join("workers")).unwrap(); + let allocation = store + .allocate(root.path(), StaleLeasePolicy::Reject) + .unwrap(); + let lease = allocation.lease(); + let original = fs::read(&lease.path).unwrap(); + assert_eq!(lease.verify_current_id().unwrap(), lease.lease_id); + assert_eq!(fs::read(&lease.path).unwrap(), original); + + let mut record: LeaseRecord = serde_json::from_slice(&original).unwrap(); + record.lease_id = Uuid::now_v7(); + let successor = serde_json::to_vec(&record).unwrap(); + fs::write(&lease.path, &successor).unwrap(); + assert!(matches!( + lease.verify_current_id(), + Err(StandaloneStoreError::LeaseOwnershipLost) + )); + assert_eq!(fs::read(&lease.path).unwrap(), successor); + + record.lease_id = lease.lease_id; + record.pid = std::process::id().wrapping_add(1); + fs::write(&lease.path, serde_json::to_vec(&record).unwrap()).unwrap(); + assert!(matches!( + lease.verify_current_id(), + Err(StandaloneStoreError::LeaseOwnershipLost) + )); + record.pid = std::process::id(); + record.process_start_marker = record + .process_start_marker + .map(|marker| marker.wrapping_add(1)); + fs::write(&lease.path, serde_json::to_vec(&record).unwrap()).unwrap(); + assert!(matches!( + lease.verify_current_id(), + Err(StandaloneStoreError::LeaseOwnershipLost) + )); + fs::write(&lease.path, original).unwrap(); + assert_eq!(lease.verify_current_id().unwrap(), lease.lease_id); + } + #[test] fn metadata_write_compares_observed_content_even_with_equal_timestamps() { let root = tempfile::tempdir().unwrap(); diff --git a/crates/standalone/src/subjektiv.rs b/crates/standalone/src/subjektiv.rs index 82a47b836..01e40d7ae 100644 --- a/crates/standalone/src/subjektiv.rs +++ b/crates/standalone/src/subjektiv.rs @@ -14,8 +14,9 @@ //! The selected Subject's Job store is shared across Workers; the generic Jobs service owns execution //! and cleanup. Consolidation grants allow only immutable candidate batches and limited operations, //! not body leases or parent filesystems. Startup pending processing follows Host connections. -//! Unknown attempts are not rerun; delivery failures for accepted results do not trigger model -//! reexecution. Memory application and surface repair use shared-domain CAS/receipts. +//! Execution receipts and domain results are reconciled without blindly replaying effects; +//! delivery failures for accepted results do not trigger model reexecution. Memory application +//! and surface repair use shared-domain CAS/receipts. mod journal; pub(crate) use journal::prepare_worker_deletion; @@ -712,20 +713,135 @@ impl LocalSubjectHost { } } } - if let Ok(pending) = jobs.pending() { - for job in pending { - if job - .domain_grant - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|g| g.binding == self.binding) - { - schedule_job(jobs.clone(), job.request.job_id, self.self_weak.clone()); + // Restart reconciliation can leave a failed attempt with a saved domain + // result. Pending alone lists only unstarted reservations, not those cuts. + let candidates = jobs.pending().and_then(|mut pending| { + pending.extend(jobs.unreleased_granted()?); + Ok(pending) + }); + match candidates { + Ok(candidates) => { + let mut seen = std::collections::HashSet::new(); + for job in candidates { + if job.request.purpose != "subjektiv_consolidation" + || !seen.insert(job.request.job_id.clone()) + || !job + .domain_grant + .as_ref() + .and_then(|v| { + serde_json::from_value::(v.clone()).ok() + }) + .is_some_and(|g| g.binding == self.binding) + { + continue; + } + match self.recover_consolidation_job(&jobs, &job) { + Ok(true) => {} + Ok(false) + if job.state == crate::jobs::JobState::Pending + && job.attempt.state == crate::jobs::JobAttemptState::Reserved + && job.executions_released() => + { + schedule_job(jobs.clone(), job.request.job_id, self.self_weak.clone()); + } + Ok(false) => {} // Missing receipts never authorize replay of effects. + Err(error) => eprintln!( + "standalone consolidation Job {} reconciliation blocked: {error}", + job.request.job_id + ), + } } } + Err(error) => eprintln!("standalone consolidation recovery lookup failed: {error}"), } self.recover_journals(); } + + /// Reconstructs a result from domain receipts, never by reentering the finalizer. + /// Acceptance does not assert prior owner shutdown or discharge cleanup ownership. + fn recover_consolidation_job( + &self, + jobs: &crate::jobs::StandaloneJobs, + observed: &crate::jobs::JobSnapshot, + ) -> Result { + let service_error = |error: crate::jobs::JobServiceError| { + SubjektivHostError::Unavailable(error.to_string()) + }; + self.ensure_live()?; + let current = jobs.get(&observed.request.job_id).map_err(service_error)?; + if current.attempt.attempt_id != observed.attempt.attempt_id + || current.input_digest != observed.input_digest + { + return Err(SubjektivHostError::InvalidArgument( + "consolidation recovery attempt/input changed".into(), + )); + } + if current.version_stopped + || current.input_disposition != crate::jobs::JobInputDisposition::Available + || current.state == crate::jobs::JobState::Cancelled + { + return Err(SubjektivHostError::InvalidArgument( + "stopped or retired consolidation cannot be recovered".into(), + )); + } + current + .load_executable_request() + .map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?; + let grant = self.consolidation_grant(¤t)?; + let Some(result) = subjektiv::job::lookup_result( + &self.catalog.store, + &grant.binding.subject_id, + &grant.candidate_ids, + ¤t.request.job_id, + ¤t.attempt.attempt_id, + ) + .map_err(operation_error)? + else { + return Ok(false); + }; + self.ensure_live()?; + let settled = jobs + .accept_reconciled(&job::JobResultSubmission { + job_id: current.request.job_id, + attempt_id: current.attempt.attempt_id, + input_digest: current.input_digest, + result, + }) + .map_err(service_error)?; + jobs.acknowledge(&settled.request.job_id) + .map_err(service_error)?; + Ok(true) + } + + fn consolidation_grant( + &self, + snapshot: &crate::jobs::JobSnapshot, + ) -> Result { + let grant: LocalConsolidationGrant = + serde_json::from_value(snapshot.domain_grant.clone().ok_or_else(|| { + SubjektivHostError::InvalidArgument("missing explicit Subject delegation".into()) + })?) + .map_err(|error| SubjektivHostError::InvalidArgument(error.to_string()))?; + if grant.binding != self.binding + || snapshot.request.purpose != "subjektiv_consolidation" + || snapshot.request.input + != serde_json::json!({"subject_id":self.binding.subject_id,"candidate_ids":grant.candidate_ids}) + { + return Err(SubjektivHostError::InvalidArgument( + "persisted Job grant/input does not match local Subject capability".into(), + )); + } + let subject = self + .catalog + .store + .subject(&self.binding.subject_id) + .map_err(domain_error)? + .ok_or_else(|| SubjektivHostError::Unavailable("Subject missing".into()))?; + if subject.state != subjektiv::SubjectState::Active { + return Err(SubjektivHostError::Unavailable("Subject is retired".into())); + } + Ok(grant) + } fn validate_evidence( &self, session: &str, @@ -1125,7 +1241,7 @@ impl SubjektivHost for LocalSubjectHost { .store .pending_staging_backlog(&self.binding.subject_id) .map_err(domain_error)?; - return Ok(memory::backend::MemoryConsolidationOutput { status:"resumed_pending".into(),summary:"Resumed the existing immutable reserved Subject Job; unknown attempts are never replayed".into(),candidate_count:count,total_bytes:bytes }); + return Ok(memory::backend::MemoryConsolidationOutput { status:"resumed_pending".into(),summary:"Resumed the existing immutable reserved Subject Job; prior execution receipts remain independently fenced".into(),candidate_count:count,total_bytes:bytes }); } let (count, bytes) = self .catalog @@ -1195,6 +1311,12 @@ struct LocalConsolidationGrant { } struct LocalDomainProvider(std::sync::Weak); impl crate::jobs::JobDomainProvider for LocalDomainProvider { + fn callbacks_use_attempt_fence(&self) -> bool { + // Consolidation memory effects and Journal execute/finish hold the attempt gate. + // Result finalizers are independently revoked and joined by the owned Job runner. + true + } + fn grant( &self, snapshot: &crate::jobs::JobSnapshot, @@ -1205,20 +1327,9 @@ impl crate::jobs::JobDomainProvider for LocalDomainProvider { if snapshot.request.purpose == journal::PURPOSE { return journal::grant(host, snapshot, fence); } - let grant: LocalConsolidationGrant = serde_json::from_value( - snapshot - .domain_grant - .clone() - .ok_or("missing explicit Subject delegation")?, - ) - .map_err(|e| e.to_string())?; - if grant.binding != host.binding - || snapshot.request.purpose != "subjektiv_consolidation" - || snapshot.request.input - != serde_json::json!({"subject_id":host.binding.subject_id,"candidate_ids":grant.candidate_ids}) - { - return Err("persisted Job grant/input does not match local Subject capability".into()); - } + let grant = host + .consolidation_grant(snapshot) + .map_err(|error| error.to_string())?; let adapter = Arc::new(LocalConsolidationHost { host, grant, @@ -1311,7 +1422,11 @@ impl SubjektivHost for LocalConsolidationHost { _context: &SubjektivHostContext, operation: SubjektivMemoryBackendOperation, ) -> Result { - self.live()?; + let _effects = self + .fence + .enter() + .map_err(SubjektivHostError::Unavailable)?; + self.host.ensure_live()?; let authority = subjektiv::api::HostOperationContext::validated_consolidation( &self.host.catalog.store, &self.grant.binding.subject_id, diff --git a/crates/standalone/src/subjektiv/journal.rs b/crates/standalone/src/subjektiv/journal.rs index 85bbe26f1..e5cd1c404 100644 --- a/crates/standalone/src/subjektiv/journal.rs +++ b/crates/standalone/src/subjektiv/journal.rs @@ -305,11 +305,11 @@ impl LocalSubjectHost { .pending() .map_err(service_error)? .into_iter() - .chain(jobs.unknown_granted().map_err(service_error)?) + .chain(jobs.unreleased_granted().map_err(service_error)?) .chain(jobs.unacknowledged_results().map_err(service_error)?) { let _ = job; - return Err(SubjektivHostError::Conflict { code: "journal_outcome_unconfirmed".into(), message: "Subject has queued/running or Unknown/unacknowledged delegated work; binding retained".into() }); + return Err(SubjektivHostError::Conflict { code: "journal_outcome_unconfirmed".into(), message: "Subject has queued/running, unreleased or unacknowledged delegated work; binding retained".into() }); } let released = self .catalog @@ -474,6 +474,7 @@ impl LocalSubjectHost { pub(super) fn recover_journals(&self) { let result = (|| { self.ensure_live()?; + self.jobs()?.recover_storage().map_err(service_error)?; for attribution in self .catalog .store @@ -565,9 +566,16 @@ impl LocalSubjectHost { // Continue even without a further user turn. A later admission has // remained in the ledger, never in this running Job's frozen input. } + Ok(snapshot) + if snapshot.state == JobState::Failed + && snapshot.executions_released() + && !snapshot.version_stopped => + { + // The next pass reconciles saved operations/checkpoints and resumes + // this exact Job/input within its attempt budget, without resampling. + } Ok(_) | Err(_) => { - // Failure is not permission to consume sources or blindly rerun the model. - // Reopen/reconnect performs domain reconciliation before replacement. + // Neither failure nor a missing task proves resource shutdown. *host .journal_dispatch .lock() @@ -593,54 +601,8 @@ impl LocalSubjectHost { if pending.request.purpose != PURPOSE { continue; } - self.journal_grant_scope(&pending)?; - if pending.request.input.get("capture").is_some() { - self.journal_grant(&pending)?; - return Ok(Some((jobs, pending.request.job_id))); - } - // Old descriptor jobs never acquire new execution authority. Their immutable - // identity/input is retired, not upgraded in place; saved checkpoint wins. - let replacement = self.replace_legacy_journal_job(&jobs, &pending)?; - if let Some(id) = replacement { - return Ok(Some((jobs, id))); - } - } - for unknown in jobs.unknown_granted().map_err(service_error)? { - let Ok(grant) = self.journal_grant_scope(&unknown) else { - continue; - }; - let context = authority(self, &grant.session_id)?; - if let Some(active) = self - .catalog - .store - .active_journal_generation(&context, &grant.journal_id) - .map_err(journal_error)? - { - if active.job.job_id != unknown.request.job_id - || active.job.attempt_id != unknown.attempt.attempt_id - { - continue; - } - self.catalog - .store - .close_journal_generation(&context, &active.id, &active.job) - .map_err(journal_error)?; - } - // This grant has no ambient side effects: every successful mutation and exact - // receipt is in the shared store. Its closed generation fences delayed tools. - // Observe actual saved current/open/cursor before freeing generic serialization; - // the replacement reads these receipts, never resubmits an aggregate old result. - let _ = self - .catalog - .store - .journal_resume_state(&context, &grant.journal_id) - .map_err(journal_error)?; - jobs.reconcile_granted_attempt( - &unknown.request.job_id, - &unknown.attempt.attempt_id, - &format!("journal-fenced:{}", unknown.attempt.attempt_id), - ) - .map_err(service_error)?; + self.journal_grant(&pending)?; + return Ok(Some((jobs, pending.request.job_id))); } let reader = JournalHostContext::validated_reader(&self.catalog.store, &self.binding.subject_id) @@ -662,112 +624,63 @@ impl LocalSubjectHost { continue; } let context = authority(self, &journal.source_session_id)?; - if let Some(active) = self - .catalog - .store - .active_journal_generation(&context, &journal.id) - .map_err(journal_error)? - { - let existing = jobs.get(&active.job.job_id).map_err(service_error)?; - if existing.attempt.attempt_id != active.job.attempt_id { - return Err(SubjektivHostError::Unavailable( - "Journal Job attempt changed without reconciliation".into(), - )); - } - match existing.attempt.state { - JobAttemptState::Reserved | JobAttemptState::Dispatched => continue, - JobAttemptState::Completed => { - self.complete_journal_job(&jobs, &existing)? - } - JobAttemptState::Unknown - | JobAttemptState::Failed - | JobAttemptState::Cancelled => { - // The generic Host has durably fenced this attempt. Close its domain - // generation before replacement; saved record receipts remain visible. - self.catalog - .store - .close_journal_generation(&context, &active.id, &active.job) - .map_err(journal_error)?; - } - } - } - let mut retained = None; - // Saved body snapshots always win over descriptor migration. In particular, - // a failed continuation must not recapture its original text even if an - // invalidated legacy grant still bounds a larger unprocessed suffix. - for legacy_pass in [false, true] { - let mut after_job = None; - loop { - let page = jobs - .list_session_snapshots( - &journal.source_session_id, - after_job.as_deref(), - 100, - ) - .map_err(service_error)?; - if page.is_empty() { - break; - } - for saved in &page { - if saved.request.purpose != PURPOSE { - continue; - } - let legacy = saved.input_disposition - == JobInputDisposition::Invalidated - || (saved.input_disposition == JobInputDisposition::Available - && saved.request.input.get("capture").is_none()); - if legacy != legacy_pass - || (!legacy - && saved.input_disposition != JobInputDisposition::Available) - { - continue; - } - let saved_grant = self.journal_grant_scope(saved)?; - if saved_grant.journal_id != journal.id { - continue; - } - if legacy { - if let Some(id) = self.replace_legacy_journal_job(&jobs, saved)? { - return Ok(Some((jobs, id))); - } - continue; - } - self.journal_grant(saved)?; - let original = JournalInput::decode(&saved.request.input) - .map_err(journal_error)?; - let continuation = self.build_journal_input( - &context, - &journal.id, - Some(&original.capture), - )?; - if !continuation.selected_ranges.is_empty() { - retained = Some(continuation); - break; - } - if saved.state == JobState::Completed && saved.acknowledged { - // Recover the acknowledgement-before-reclamation gap without - // turning a retained receipt into an executable empty input. - jobs.reclaim_input( - &saved.request.job_id, - &saved.input_digest, - JobInputRetirementReason::Completed, - ) - .map_err(service_error)?; - } - } - if retained.is_some() { - break; - } - after_job = page.last().map(|job| job.request.job_id.clone()); - } - if retained.is_some() { + // Resolve every saved owner before admitting a new prefix. A failed + // Job is not permission to resample its input into a replacement Job. + let mut after_job = None; + loop { + let saved_page = jobs + .list_session_snapshots( + &journal.source_session_id, + after_job.as_deref(), + 100, + ) + .map_err(service_error)?; + if saved_page.is_empty() { break; } + for saved in &saved_page { + if saved.request.purpose != PURPOSE { + continue; + } + let grant = self.journal_grant_scope(saved)?; + if grant.journal_id != journal.id { + continue; + } + if saved.version_stopped { + // Migration is not normal recovery or Journal completion. + // Keep the original ranges/input, even when fully checkpointed. + if !self.saved_ranges_reflected(&context, &grant)? + || !saved.executions_released() + { + return Err(SubjektivHostError::Conflict { code: "journal_version_stopped".into(), message: "Version-stopped Journal retains input/checkpoint and unproven execution obligations".into() }); + } + continue; + } + match saved.state { + JobState::Completed => self.complete_journal_job(&jobs, saved)?, + JobState::Pending + if saved.attempt.state == JobAttemptState::Reserved => + { + self.journal_grant(saved)?; + return Ok(Some((jobs, saved.request.job_id.clone()))); + } + JobState::Pending | JobState::Failed => { + if let Some(resumed) = self.recover_journal_job(&jobs, saved)? { + return Ok(Some((jobs, resumed.request.job_id))); + } + } + JobState::Cancelled => { + if !saved.executions_released() + || !self.saved_ranges_reflected(&context, &grant)? + { + return Err(SubjektivHostError::Conflict {code:"journal_cancelled_input_retained".into(),message:"Cancelled Journal has unreleased execution or unreflected saved input".into()}); + } + } + } + } + after_job = saved_page.last().map(|job| job.request.job_id.clone()); } - let input = match retained { - Some(input) => input, - None => self.build_journal_input(&context, &journal.id, None)?, - }; + let input = self.build_journal_input(&context, &journal.id, None)?; if input.selected_ranges.is_empty() { continue; @@ -797,120 +710,105 @@ impl LocalSubjectHost { } } - fn replace_legacy_journal_job( + fn saved_ranges_reflected( + &self, + context: &JournalHostContext, + grant: &LocalJournalGrant, + ) -> Result { + for range in &grant.ranges { + let status = self + .catalog + .store + .read_journal_source(context, &grant.journal_id, range.source.order) + .map_err(journal_error)?; + if status.source != range.source || status.reflected_bytes < range.bytes.end { + return Ok(false); + } + } + Ok(true) + } + + fn recover_journal_job( &self, jobs: &crate::jobs::StandaloneJobs, - old: &JobSnapshot, - ) -> Result, SubjektivHostError> { - let grant = self.journal_grant_scope(old)?; - if old.input_disposition == JobInputDisposition::Available { - old.load_executable_request() - .map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?; - if old.request.input - != serde_json::json!({"selected_ranges":grant.ranges,"origins":grant.origins}) - { - return Err(SubjektivHostError::InvalidArgument( - "invalid legacy Journal descriptor input".into(), - )); - } - } else if old.input_disposition != JobInputDisposition::Invalidated { - return Err(SubjektivHostError::InvalidArgument( - "legacy Journal input is unavailable".into(), - )); + saved: &JobSnapshot, + ) -> Result, SubjektivHostError> { + let current = jobs.get(&saved.request.job_id).map_err(service_error)?; + if current.attempt.attempt_id != saved.attempt.attempt_id + || current.input_digest != saved.input_digest + { + return Err(SubjektivHostError::Conflict { + code: "journal_attempt_changed".into(), + message: "Journal recovery snapshot no longer names the current attempt/input" + .into(), + }); } + let saved = ¤t; + if saved.version_stopped { + return Err(SubjektivHostError::Conflict { + code: "journal_version_stopped".into(), + message: + "Version-stopped Journal cannot resume or convert checkpoints into completion" + .into(), + }); + } + self.journal_grant(saved)?; // verifies the immutable body; never fetch live text + let grant = self.journal_grant_scope(saved)?; let context = authority(self, &grant.session_id)?; - if let Some(active) = self + let binding = JobAttemptBinding { + job_id: saved.request.job_id.clone(), + attempt_id: saved.attempt.attempt_id.clone(), + }; + let generation = self .catalog .store - .active_journal_generation(&context, &grant.journal_id) - .map_err(journal_error)? - { - if active.job.job_id != old.request.job_id { - return Err(SubjektivHostError::InvalidArgument( - "another generation owns legacy migration".into(), - )); + .retained_journal_generation(&context, &grant.journal_id, &binding) + .map_err(journal_error)?; + if let Some(generation) = generation.as_ref() { + let recovery = JournalJobContext::validated_recovery( + &self.catalog.store, + &context, + &generation.id, + binding.clone(), + ) + .map_err(journal_error)?; + match subjektiv::journal::service::finish(&self.catalog.store, &recovery) { + Ok(completion) => { + let settled = jobs + .accept_reconciled(&job::JobResultSubmission { + job_id: binding.job_id.clone(), + attempt_id: binding.attempt_id.clone(), + input_digest: saved.input_digest.clone(), + result: serde_json::to_value(completion) + .map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?, + }) + .map_err(service_error)?; + self.complete_journal_job(jobs, &settled)?; + return Ok(None); + } + Err(JournalError::Conflict(_)) => {} // incomplete, retained checkpoint + Err(error) => return Err(journal_error(error)), } + } + if !saved.executions_released() { + return Err(SubjektivHostError::Conflict {code:"journal_execution_unreleased".into(),message:"Saved Journal input/operations retained; preceding execution/cleanup is not proven stopped".into()}); + } + let receipt = if let Some(generation) = generation { self.catalog .store - .close_journal_generation(&context, &active.id, &active.job) + .close_journal_generation(&context, &generation.id, &binding) .map_err(journal_error)?; - } - jobs.cancel(&old.request.job_id).map_err(service_error)?; - let current = self - .catalog - .store - .read_journal(&context, &grant.journal_id) - .map_err(journal_error)?; - let end = grant - .ranges - .last() - .ok_or_else(|| SubjektivHostError::InvalidArgument("empty legacy selection".into()))?; - let end_position = if end.bytes.end == end.source.total_bytes { - (end.source.order + 1, 0) + format!("journal-generation-closed:{}", generation.id) } else { - (end.source.order, end.bytes.end) + // The authoritative domain lookup found no generation for this exact attempt. + "journal-no-generation".to_string() }; - let replacement = if (current.process_order, current.process_byte) >= end_position { - None - } else { - let mut capture = self.recording_capture(&grant.session_id)?; - capture - .entries - .retain(|entry| grant.ranges.iter().any(|r| r.source.order == entry.order)); - for entry in &mut capture.entries { - let range = grant - .ranges - .iter() - .find(|r| r.source.order == entry.order) - .unwrap(); - let session_store::capture::CaptureBody::CanonicalPublic { - descriptor, - byte_offset, - text, - .. - } = &mut entry.body - else { - unreachable!() - }; - if source_descriptor(&grant.session_id, descriptor) != range.source - || !grant.origins.iter().any(|origin| { - origin.pointer == range.source.pointer && origin.origin == descriptor.origin - }) - || !text.is_char_boundary(range.bytes.start as usize) - || !text.is_char_boundary(range.bytes.end as usize) - { - return Err(SubjektivHostError::InvalidArgument( - "legacy original source identity/range changed".into(), - )); - } - *text = text[range.bytes.start as usize..range.bytes.end as usize].to_owned(); - *byte_offset = range.bytes.start; - } - let input = self.build_journal_input(&context, &grant.journal_id, Some(&capture))?; - let request = self.journal_request(&grant.journal_id, &input)?; - let next_grant = LocalJournalGrant { - ranges: input.selected_ranges.clone(), - origins: vec![], - ..grant - }; - Some( - jobs.request_granted( - request, - serde_json::to_value(next_grant) - .map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?, - ) - .map_err(service_error)? - .request - .job_id, - ) - }; - jobs.reclaim_input( - &old.request.job_id, - &old.input_digest, - JobInputRetirementReason::LegacyInvalidation, - ) - .map_err(service_error)?; - Ok(replacement) + jobs.reconcile_granted_attempt(&binding.job_id, &binding.attempt_id, &receipt) + .map_err(service_error)?; + let resumed = jobs + .resume_granted(&binding.job_id, &binding.attempt_id) + .map_err(service_error)?; + Ok(Some(resumed)) } fn journal_request( @@ -1455,6 +1353,9 @@ impl LocalSubjectHost { } jobs.acknowledge(&snapshot.request.job_id) .map_err(service_error)?; + if !snapshot.executions_released() { + return Err(SubjektivHostError::Conflict {code:"journal_cleanup_unconfirmed".into(),message:"Checkpoint/result completed; execution cleanup remains independent and unconfirmed".into()}); + } jobs.reclaim_input( &snapshot.request.job_id, &snapshot.input_digest, @@ -1560,6 +1461,10 @@ impl JournalJobHost for LocalJournalHost { &self, request: JournalJobRequest, ) -> Result { + let _effects = self + .fence + .enter() + .map_err(SubjektivHostError::Unavailable)?; let context = self.live()?; // The generic runner has supplied this exact immutable input before it can // invoke this bound tool. Preparation alone is not evidence of delivery. @@ -1579,6 +1484,10 @@ impl JournalJobHost for LocalJournalHost { .map_err(journal_error) } fn finish(&self) -> Result { + let _effects = self + .fence + .enter() + .map_err(SubjektivHostError::Unavailable)?; let context = self.live()?; subjektiv::journal::service::finish(&self.host.catalog.store, &context) .map_err(journal_error) diff --git a/crates/standalone/src/subjektiv/journal/tests.rs b/crates/standalone/src/subjektiv/journal/tests.rs index 0e4d76f5f..ff5ced557 100644 --- a/crates/standalone/src/subjektiv/journal/tests.rs +++ b/crates/standalone/src/subjektiv/journal/tests.rs @@ -11,6 +11,7 @@ mod compaction; mod connected_compaction; mod handoff; mod memory_links; +mod recovery; struct Fixture { _temp: tempfile::TempDir, @@ -1092,7 +1093,7 @@ async fn persisted_pre_binding_job_recovers_only_under_original_session_lifetime } #[tokio::test] -async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_create() { +async fn shutdown_reopen_resumes_same_job_and_frozen_input_without_duplicate_create() { let mut fixture = Fixture::new("proposed, corrected, partially performed, then interrupted"); let state = fixture.host().catalog.state_root.clone(); let worker = fixture._allocation.as_ref().unwrap().worker_id(); @@ -1129,12 +1130,27 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c .items; assert_eq!(entries.len(), 1); let saved_entry = entries[0].id.clone(); - let original_input = jobs.get(&id).unwrap().request.input; + let original = jobs.get(&id).unwrap(); + let original_input = original.request.input.clone(); fixture.append( "newer-than-snapshot", "NEW INPUT MUST NOT MIX INTO RESUMED SNAPSHOT", ); jobs.shutdown().await.unwrap(); + let stopped_store = crate::jobs::StandaloneJobs::open( + &fixture.connection.jobs_path, + fixture._temp.path().into(), + ) + .unwrap(); + let stopped = stopped_store.get(&id).unwrap(); + assert_eq!(stopped.state, JobState::Failed); + assert_eq!(stopped.attempt.state, JobAttemptState::Failed); + assert_eq!(stopped.attempt.attempt_id, original.attempt.attempt_id); + assert_eq!(stopped.request, original.request); + assert!(!stopped.version_stopped); + assert!(!stopped.executions.is_empty()); + assert!(stopped.executions_released()); + stopped_store.shutdown().await.unwrap(); fixture.connection.close().unwrap(); let mut reopened = StandaloneSubjects::open_existing(&state) .unwrap() @@ -1143,8 +1159,8 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c let jobs = crate::jobs::StandaloneJobs::open(&reopened.jobs_path, fixture._temp.path().into()) .unwrap(); reopened.host.bind_jobs(jobs.clone()).unwrap(); - assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown); - // Deny every original-source read while preparing and running the replacement. + assert_eq!(jobs.get(&id).unwrap(), stopped); + // Deny every original-source read while preparing and running the new attempt. let source_path = fixture .sessions .root_dir() @@ -1154,28 +1170,35 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c .path() .join("temporarily-unavailable-original"); fs::rename(&source_path, &denied).unwrap(); - let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap(); - let replacement_input = jobs.get(&replacement).unwrap().request.input; - assert_eq!(replacement_input["capture"], original_input["capture"]); + let (_, resumed_id) = reopened.host.next_journal_job().unwrap().unwrap(); + assert_eq!(resumed_id, id); + let resumed = jobs.get(&resumed_id).unwrap(); + assert_eq!(resumed.request, original.request); + assert_eq!(resumed.input_digest, original.input_digest); + assert_eq!(resumed.intent_fingerprint, original.intent_fingerprint); + assert_eq!(resumed.domain_grant, original.domain_grant); + assert_eq!(resumed.executions, stopped.executions); + assert_eq!(resumed.state, JobState::Pending); + assert_eq!(resumed.attempt.state, JobAttemptState::Reserved); + assert_eq!(resumed.attempt.number, original.attempt.number + 1); + assert_ne!(resumed.attempt.attempt_id, original.attempt.attempt_id); + assert_eq!(resumed.input_disposition, JobInputDisposition::Available); assert!( - !replacement_input + !resumed + .request + .input .to_string() .contains("NEW INPUT MUST NOT MIX") ); - assert_eq!(jobs.get(&id).unwrap().request.input, original_input); - assert_eq!( - jobs.get(&id).unwrap().input_disposition, - JobInputDisposition::Available - ); - assert_ne!( - replacement, id, - "unknown external execution is not blindly replayed" - ); - jobs.start_with_model_client(&replacement, JournalScript::default()) + let history = jobs + .list_session_snapshots(&fixture.context.session_id, None, 100) + .unwrap(); + assert_eq!(history.len(), 1, "recovery must not reserve a second Job"); + jobs.start_with_model_client(&resumed_id, JournalScript::default()) .await .unwrap(); let completed = - tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&replacement)) + tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&resumed_id)) .await .unwrap() .unwrap(); @@ -1894,7 +1917,7 @@ async fn saved_body_jobs_process_over_a_hundred_entries_and_utf8_without_live_so } #[tokio::test] -async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful_records() { +async fn descriptor_job_version_cutover_retains_input_and_records_without_live_recapture() { let fixture = Fixture::new("original descriptor-only input"); let (adapter, _, modern) = fixture.adapter(); let jobs = fixture.host().jobs().unwrap(); @@ -1929,13 +1952,8 @@ async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful request.job_id = format!("legacy-descriptor:{}", uuid::Uuid::now_v7()); request.input = serde_json::json!({"selected_ranges":old_grant.ranges,"origins":old_grant.origins}); - let old = jobs - .request_granted(request, serde_json::to_value(&old_grant).unwrap()) - .unwrap(); - assert!( - fixture.host().journal_grant(&old).is_err(), - "descriptor jobs must not execute as snapshots" - ); + let path = fixture._temp.path().join("descriptor-schema-five.sqlite3"); + recovery::seed_schema_five_journal(&path, &request, &old_grant); let generation = fixture .host() .catalog @@ -1944,8 +1962,8 @@ async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful &adapter.context, &old_grant.journal_id, JobAttemptBinding { - job_id: old.request.job_id.clone(), - attempt_id: old.attempt.attempt_id.clone(), + job_id: request.job_id.clone(), + attempt_id: job::attempt_id(&request.job_id, 1), }, old_grant.ranges.clone(), ) @@ -1978,56 +1996,52 @@ async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful }, ) .unwrap(); - fixture.append( - "new-not-selected", - "LATEST SOURCE MUST WAIT UNTIL OLD INPUT FINISHES", + // Normal recovery is not the migration runner. Seed the old running attempt, + // then use the production initializer before consulting any live Session. + let prior = rusqlite::Connection::open(&path).unwrap(); + prior + .execute("UPDATE job_attempts SET state='dispatched'", []) + .unwrap(); + drop(prior); + let migrated = crate::jobs::StandaloneJobs::open(&path, fixture._temp.path().into()).unwrap(); + let old = migrated.get(&request.job_id).unwrap(); + assert!(old.version_stopped); + assert_eq!(old.state, JobState::Failed); + assert_eq!(old.attempt.state, JobAttemptState::Failed); + assert_eq!( + old.attempt.failure.as_deref(), + Some("バージョン更新による実行打切り") ); - fixture.host().admit_journal(&fixture.context).unwrap(); - let (_, next) = fixture.host().next_journal_job().unwrap().unwrap(); - assert_ne!(next, old.request.job_id); - let replacement = jobs.get(&next).unwrap(); - let replacement_input = JournalInput::decode(&replacement.request.input).unwrap(); + assert_eq!(old.request, request); + assert_eq!(old.input_disposition, JobInputDisposition::Available); + assert!(!old.executions_released()); + assert!(fixture.host().journal_grant(&old).is_err()); + fixture.append("new-not-selected", "LATEST SOURCE MUST NOT BE RECAPTURED"); + let error = fixture + .host() + .recover_journal_job(&migrated, &old) + .unwrap_err(); assert!( - !replacement - .request - .input - .to_string() - .contains("LATEST SOURCE MUST WAIT") + matches!(error, SubjektivHostError::Conflict { ref code, .. } if code == "journal_version_stopped") ); - assert_eq!(replacement_input.selected_ranges, old_grant.ranges); - let retired = jobs.get(&old.request.job_id).unwrap(); - assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated); - assert_eq!(retired.input_digest, old.input_digest); - assert_eq!(retired.intent_fingerprint, old.intent_fingerprint); - assert!(retired.request.input.is_null()); + assert!(migrated.retry(&request.job_id).is_err()); assert!( + migrated + .resume_granted(&request.job_id, &old.attempt.attempt_id) + .is_err() + ); + assert!(migrated.pending().unwrap().is_empty()); + assert_eq!(migrated.get(&request.job_id).unwrap(), old); + assert_eq!( fixture .host() .catalog .store - .execute_journal( - &context, - JournalOperation { - request_id: "legacy-delayed".into(), - expected_state: saved.state_token, - action: JournalAction::ConfirmUnchanged, - covered_ranges: old_grant.ranges, - } - ) - .is_err() + .active_journal_generation(&adapter.context, &adapter.grant.journal_id,) + .unwrap() + .unwrap(), + generation ); - jobs.start_with_model_client(&next, JournalScript::default()) - .await - .unwrap(); - let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&next)) - .await - .unwrap() - .unwrap(); - assert_eq!(completed.state, JobState::Completed); - fixture - .host() - .complete_journal_job(&jobs, &completed) - .unwrap(); let entries = fixture .host() .catalog @@ -2036,6 +2050,16 @@ async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful .unwrap(); assert_eq!(entries.items.len(), 1); assert_eq!(entries.items[0].id, saved.entry_id.unwrap()); + drop(migrated); + let reopened = crate::jobs::StandaloneJobs::open(&path, fixture._temp.path().into()).unwrap(); + assert_eq!(reopened.get(&request.job_id).unwrap(), old); + assert_eq!( + reopened + .list_session_snapshots(&fixture.context.session_id, None, 100) + .unwrap() + .len(), + 1 + ); jobs.shutdown().await.unwrap(); } @@ -2373,23 +2397,31 @@ async fn close_cancels_retained_local_job_before_erase_can_acquire_exclusive_lea .is_ok(); // Always tear down before asserting the observed violation, so a failed regression // cannot leak the blocked model future into another test. - let _ = retained.shutdown().await; + tokio::time::timeout(std::time::Duration::from_secs(20), retained.shutdown()) + .await + .unwrap() + .unwrap(); assert!(model.request_dropped.load(Ordering::Acquire)); fixture.connection.close().unwrap(); assert!(fixture.connection.lease.is_none()); assert!(fixture.host().jobs.lock().unwrap().is_none()); - // Cancellation is not completion/erasure. Reopening retains the same unknown - // attempt's immutable body until the explicit erasure transaction retires it. + // Cancellation is not completion/erasure. The owned shutdown has confirmed cleanup, + // but reopening must preserve the failed attempt and its immutable body. let reopened = crate::jobs::StandaloneJobs::open( &fixture.connection.jobs_path, fixture._temp.path().into(), ) .unwrap(); - let unknown = reopened.get(&id).unwrap(); - assert_eq!(unknown.attempt.state, crate::jobs::JobAttemptState::Unknown); - assert_eq!(unknown.input_disposition, JobInputDisposition::Available); + let failed = reopened.get(&id).unwrap(); + assert_eq!(failed.state, JobState::Failed); + assert_eq!(failed.attempt.state, JobAttemptState::Failed); + assert!(!failed.version_stopped); + assert_eq!(failed.attempt.number, 1); + assert!(!failed.executions.is_empty()); + assert!(failed.executions_released()); + assert_eq!(failed.input_disposition, JobInputDisposition::Available); assert!( - unknown + failed .request .input .to_string() diff --git a/crates/standalone/src/subjektiv/journal/tests/connected_compaction.rs b/crates/standalone/src/subjektiv/journal/tests/connected_compaction.rs index 50ad7cfbc..d769eb95a 100644 --- a/crates/standalone/src/subjektiv/journal/tests/connected_compaction.rs +++ b/crates/standalone/src/subjektiv/journal/tests/connected_compaction.rs @@ -111,7 +111,7 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab assert_eq!(journal.process_byte, before.current_journal.process_byte); assert_eq!(journal.process_order, 0); assert_eq!(journal.process_byte, 0, "saved open records do not advance processing checkpoint"); - let expected_json = serde_json::to_string(&expected).unwrap(); + let expected_json = serde_json::to_value(&expected).unwrap(); let adapter = Arc::new(adapter); let model = MockClient { adapter: adapter.clone(), requests: Arc::default(), updated: Arc::new(AtomicBool::new(false)) }; let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap(); @@ -142,7 +142,11 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab assert_eq!(journal_items.len(), 1); let journal_item = Item::from(journal_items[0].item.clone()); let body = journal_item.as_text().unwrap(); - assert!(body.contains(&expected_json), "durable seed contains the full selected JSON and processing checkpoint"); + // Display is concise YAML; the Feature payload retains the full frozen + // snapshot, including processing checkpoints omitted from display. + let payload: serde_json::Value = journal_items[0].metadata.input_metadata.as_ref() + .unwrap().feature_payload().unwrap().decode("journal:compaction", 1).unwrap(); + assert_eq!(payload["snapshot"], expected_json, "durable payload retains the complete frozen snapshot and processing checkpoint"); assert!(body.contains(BEFORE)); assert!(!body.contains(AFTER)); assert!(seed.iter().any(|entry| entry.metadata.input_metadata.as_ref().is_some_and(|input| input.usage().compaction == session_store::InputCompactionUse::OpeningSummary) && Item::from(entry.item.clone()).as_text().is_some_and(|text| text.contains(COMPLEMENT)))); @@ -152,7 +156,7 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab _ => None, }).expect("actual compactor user request"); assert!(input.ends_with(body), "exact same frozen reference goes to actual compactor and seed"); - assert_eq!(input.matches(&expected_json).count(), 1); + assert_eq!(input.matches(body).count(), 1); assert!(!input.contains(AFTER)); drop(requests); let latest = fixture.host().journal_compaction_context(&fixture.context).unwrap(); diff --git a/crates/standalone/src/subjektiv/journal/tests/recovery.rs b/crates/standalone/src/subjektiv/journal/tests/recovery.rs new file mode 100644 index 000000000..ac7a35da4 --- /dev/null +++ b/crates/standalone/src/subjektiv/journal/tests/recovery.rs @@ -0,0 +1,321 @@ +//! Journal recovery connects the real local Host, Journal store and temporary Job SQLite +//! stores. Persisted prior-owner rows are injected without spawning a Worker; they cannot +//! supply cleanup proof. Scripted owned-Worker shutdown is covered in the parent suite. + +use super::*; + +#[test] +fn restart_reconciles_complete_checkpoint_but_retains_partial_input_and_all_unproven_cleanup() { + for checkpoint_complete in [false, true] { + let fixture = Fixture::new("restart uses only this frozen body"); + let (adapter, _, original) = fixture.adapter(); + let initial = adapter.state().unwrap(); + let saved = save( + &adapter, + "restart-record", + &initial.current_journal.state_token, + JournalAction::Create { + title: "saved once".into(), + body: "retained operation".into(), + ongoing: "ongoing".into(), + state: JournalEntryState::Open, + source_end: None, + correction_refs: vec![], + }, + initial.selected_ranges.clone(), + ); + if checkpoint_complete { + let position = save( + &adapter, + "restart-position", + &saved.state_token, + JournalAction::UpdatePosition { + current_position: "selected input reflected".into(), + }, + vec![], + ); + save( + &adapter, + "restart-checkpoint", + &position.state_token, + JournalAction::Checkpoint { + record_operation_ids: vec!["restart-record".into()], + }, + initial.selected_ranges, + ); + } + let path = fixture._temp.path().join("restart-job.sqlite3"); + let mut prior = crate::job_store::JobStore::open(&path).unwrap(); + prior.bind_host_lease("prior-host").unwrap(); + let reserved = prior + .reserve_granted(original.request.clone(), original.domain_grant.clone()) + .unwrap(); + let dispatched = prior + .dispatch(&original.request.job_id, &reserved.attempt.attempt_id, 8) + .unwrap(); + drop(prior); + let mut next = crate::job_store::JobStore::open(&path).unwrap(); + next.bind_host_lease("replacement-host").unwrap(); + let failed = next.reconcile_host_restart().unwrap().pop().unwrap(); + assert_eq!(failed.state, JobState::Failed); + assert_eq!(failed.executions, dispatched.executions); + assert!(!failed.version_stopped); + drop(next); + let jobs = crate::jobs::StandaloneJobs::open(&path, fixture._temp.path().into()).unwrap(); + // A missing live Session makes any source re-fetch an observable failure. + let source = fixture + .sessions + .root_dir() + .join(&fixture.context.session_id); + let unavailable = fixture._temp.path().join("unavailable-live-session"); + fs::rename(&source, &unavailable).unwrap(); + let error = fixture + .host() + .recover_journal_job(&jobs, &failed) + .unwrap_err(); + let settled = jobs.get(&original.request.job_id).unwrap(); + if checkpoint_complete { + assert!( + matches!(error, SubjektivHostError::Conflict {ref code, ..} if code == "journal_cleanup_unconfirmed") + ); + assert_eq!(settled.state, JobState::Completed); + let completion: JournalJobCompletion = + serde_json::from_value(settled.result.clone().unwrap()).unwrap(); + assert_eq!(completion.generation_id, adapter.generation.id); + assert!(settled.acknowledged); + } else { + assert!( + matches!(error, SubjektivHostError::Conflict {ref code, ..} if code == "journal_execution_unreleased") + ); + assert_eq!(settled, failed); + assert!(settled.result.is_none()); + } + assert_eq!(settled.attempt.attempt_id, reserved.attempt.attempt_id); + assert_eq!(settled.attempt.number, 1); + assert_eq!(settled.request, original.request); + assert_eq!(settled.input_disposition, JobInputDisposition::Available); + assert_eq!(settled.executions, dispatched.executions); + assert!(!settled.executions_released()); + assert!(jobs.pending().unwrap().is_empty()); + assert!( + jobs.resume_granted(&settled.request.job_id, &settled.attempt.attempt_id) + .is_err() + ); + let entries = fixture + .host() + .catalog + .store + .list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32) + .unwrap(); + assert_eq!(entries.items.len(), 1); + assert_eq!(entries.items[0].id, saved.entry_id.unwrap()); + fs::rename(&unavailable, &source).unwrap(); + } +} + +#[test] +fn current_schema_reopen_preserves_unconfirmed_execution_and_blocks_journal_resume() { + let fixture = Fixture::new("saved input must not be recaptured"); + let (adapter, _, original) = fixture.adapter(); + let state = adapter.state().unwrap(); + let saved = save( + &adapter, + "saved-before-owner-loss", + &state.current_journal.state_token, + JournalAction::Create { + title: "partial operation".into(), + body: "durable record without checkpoint".into(), + ongoing: "unconfirmed execution".into(), + state: JournalEntryState::Open, + source_end: None, + correction_refs: vec![], + }, + state.selected_ranges, + ); + let source = fixture + .sessions + .root_dir() + .join(&fixture.context.session_id); + let denied = fixture._temp.path().join("unavailable-recovery-source"); + fs::rename(&source, &denied).unwrap(); + for fail_attempt in [false, true] { + let path = fixture._temp.path().join(if fail_attempt { + "prior-failed.sqlite3" + } else { + "prior-dispatched.sqlite3" + }); + let mut prior = crate::job_store::JobStore::open(&path).unwrap(); + let reserved = prior + .reserve_granted(original.request.clone(), original.domain_grant.clone()) + .unwrap(); + let mut retained = prior + .dispatch(&original.request.job_id, &reserved.attempt.attempt_id, 8) + .unwrap(); + if fail_attempt { + retained = prior + .finish( + &original.request.job_id, + &reserved.attempt.attempt_id, + JobAttemptState::Failed, + Some("prior execution failed without cleanup evidence"), + ) + .unwrap(); + } + drop(prior); + let reopened = + crate::jobs::StandaloneJobs::open(&path, fixture._temp.path().into()).unwrap(); + assert_eq!(reopened.get(&original.request.job_id).unwrap(), retained); + assert!(!retained.version_stopped); + assert!(!retained.executions_released()); + assert_eq!(retained.executions.len(), 1); + assert!(!retained.executions[0].writes_revoked); + assert!(retained.executions[0].cleanup_receipt.is_none()); + let error = match fixture.host().recover_journal_job(&reopened, &retained) { + Err(error) => error, + Ok(_) => panic!("unconfirmed prior execution must block Journal recovery"), + }; + assert!(matches!( + error, + SubjektivHostError::Conflict { ref code, .. } + if code == "journal_execution_unreleased" + )); + assert!(reopened.retry(&original.request.job_id).is_err()); + assert!( + reopened + .reconcile_granted_attempt( + &original.request.job_id, + &retained.attempt.attempt_id, + "domain-receipts-alone-are-not-cleanup", + ) + .is_err() + ); + assert!( + reopened + .resume_granted(&original.request.job_id, &retained.attempt.attempt_id) + .is_err() + ); + let mut replacement = original.request.clone(); + replacement.job_id = format!("replacement-{fail_attempt}"); + assert!( + reopened + .request_granted(replacement, original.domain_grant.clone().unwrap()) + .is_err(), + "unconfirmed execution retains the serialization fence" + ); + assert_eq!(reopened.get(&original.request.job_id).unwrap(), retained); + assert_eq!( + reopened + .list_session_snapshots(&fixture.context.session_id, None, 100) + .unwrap() + .len(), + 1 + ); + assert!(matches!( + reopened.try_shutdown(), + Err(crate::jobs::JobServiceError::ShutdownIncomplete) + )); + // Only prior-owner DB rows exist here: dropping this observer is not cleanup proof. + drop(reopened); + let reread = crate::job_store::JobStore::open(&path).unwrap(); + assert_eq!(reread.get(&original.request.job_id).unwrap(), retained); + } + fs::rename(&denied, &source).unwrap(); + let entries = fixture + .host() + .catalog + .store + .list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32) + .unwrap(); + assert_eq!(entries.items.len(), 1); + assert_eq!(entries.items[0].id, saved.entry_id.unwrap()); + assert_eq!( + fixture + .host() + .catalog + .store + .active_journal_generation(&adapter.context, &adapter.grant.journal_id) + .unwrap() + .unwrap(), + adapter.generation + ); +} + +/// Minimal schema-five storage graph for the Journal cutover boundary. Legacy state +/// spellings belong to this persisted input fixture, never to the current Rust enums. +pub(super) fn seed_schema_five_journal( + path: &std::path::Path, + request: &job::JobRequest, + grant: &LocalJournalGrant, +) { + let mut conn = rusqlite::Connection::open(path).unwrap(); + feature_storage::configure_connection(&conn).unwrap(); + let tx = conn.transaction().unwrap(); + tx.execute_batch( + "CREATE TABLE job_intents ( + job_id TEXT PRIMARY KEY NOT NULL, + request_json TEXT NOT NULL, + fingerprint TEXT NOT NULL, + serialization_key TEXT, + max_concurrent INTEGER NOT NULL, + state TEXT NOT NULL CHECK(state IN ('pending','completed','failed','unknown','cancelled')), + current_attempt INTEGER NOT NULL, + acknowledged INTEGER NOT NULL DEFAULT 0, + input_disposition TEXT NOT NULL DEFAULT 'available', + FOREIGN KEY(job_id,current_attempt) REFERENCES job_attempts(job_id,number) + DEFERRABLE INITIALLY DEFERRED + ); + CREATE TABLE job_attempts ( + job_id TEXT NOT NULL REFERENCES job_intents(job_id), + number INTEGER NOT NULL, + attempt_id TEXT NOT NULL UNIQUE, + input_digest TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('reserved','dispatched','completed','failed','unknown','cancelled')), + result_json TEXT, + result_digest TEXT, + failure TEXT, + PRIMARY KEY(job_id,number) + ); + CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) + WHERE serialization_key IS NOT NULL AND state='pending'; + CREATE TABLE job_domain_grants ( + job_id TEXT PRIMARY KEY NOT NULL REFERENCES job_intents(job_id), + grant_json TEXT NOT NULL + ); + CREATE TABLE job_domain_reconciliations ( + job_id TEXT NOT NULL REFERENCES job_intents(job_id), + attempt_id TEXT NOT NULL, + domain_receipt TEXT NOT NULL, + PRIMARY KEY(job_id,attempt_id) + );", + ) + .unwrap(); + tx.execute( + "INSERT INTO job_intents(job_id,request_json,fingerprint,serialization_key,max_concurrent,state,current_attempt) + VALUES(?1,?2,?3,?4,?5,'pending',1)", + rusqlite::params![ + request.job_id, + serde_json::to_string(request).unwrap(), + request.fingerprint().unwrap(), + request.serialization_key, + request.limits.max_concurrent_jobs, + ], + ) + .unwrap(); + tx.execute( + "INSERT INTO job_attempts(job_id,number,attempt_id,input_digest,state) + VALUES(?1,1,?2,?3,'reserved')", + rusqlite::params![ + request.job_id, + job::attempt_id(&request.job_id, 1), + request.input_digest().unwrap(), + ], + ) + .unwrap(); + tx.execute( + "INSERT INTO job_domain_grants(job_id,grant_json) VALUES(?1,?2)", + rusqlite::params![request.job_id, serde_json::to_string(grant).unwrap()], + ) + .unwrap(); + tx.pragma_update(None, "user_version", 5).unwrap(); + tx.commit().unwrap(); +} diff --git a/crates/standalone/src/subjektiv/tests.rs b/crates/standalone/src/subjektiv/tests.rs index 7966751ef..5c1d70c1f 100644 --- a/crates/standalone/src/subjektiv/tests.rs +++ b/crates/standalone/src/subjektiv/tests.rs @@ -1,5 +1,425 @@ +//! Consolidation recovery connects the production Subject receipt store, SQLite Job +//! store and local recovery dispatcher. Fixtures persist an interrupted attempt; +//! the named joined-execution contrast substitutes that execution boundary's +//! cleanup receipt, not the domain lookup. No test here proves an OS crash/drain. use super::*; +struct ConsolidationRecoveryFixture { + body: SubjectConnection, + jobs: crate::jobs::StandaloneJobs, + saved: crate::jobs::JobSnapshot, + execution_owner: crate::job_store::JobStore, + _temp: tempfile::TempDir, +} + +impl Drop for ConsolidationRecoveryFixture { + fn drop(&mut self) { + // Revoke this fixture's Host before deleting its isolated storage. This + // does not settle the retained prior execution or manufacture drain proof. + self.body.host.live.store(false, Ordering::Release); + } +} + +impl ConsolidationRecoveryFixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap(); + let subject = catalog.create("recovery-owner", None).unwrap(); + let body = catalog + .connect(&subject.id, protocol::WorkerId::now_v7()) + .unwrap(); + body.host + .catalog + .store + .stage_candidate(subjektiv::SubjectStagingRecord::attach( + &subject.id, + memory::extract::StagingRecord::from_candidate( + "candidate", + "dedup", + memory::schema::SourceRef { + segment_id: "segment".into(), + range: [1, 1], + }, + memory::extract::ExtractedCandidate { + kind: memory::extract::CandidateKind::Lesson, + claim: "claim".into(), + why_useful: "reason".into(), + staleness: None, + evidence_ids: vec![], + }, + vec![], + vec![], + ), + )) + .unwrap(); + let grant = LocalConsolidationGrant { + binding: body.binding.clone(), + candidate_ids: vec!["candidate".into()], + }; + let request = job::JobRequest { + job_id: "consolidation-recovery".into(), + purpose: "subjektiv_consolidation".into(), + input_ref: format!("subjektiv://{}/consolidation", subject.id), + input: serde_json::json!({"subject_id":subject.id,"candidate_ids":grant.candidate_ids}), + instruction: "Resolve the immutable batch".into(), + profile: "builtin:standalone-subjektiv-consolidation".into(), + serialization_key: Some(format!("subjektiv:{}", subject.id)), + limits: Default::default(), + }; + let mut store = crate::job_store::JobStore::open(&body.jobs_path).unwrap(); + let reserved = store + .reserve_granted(request, Some(serde_json::to_value(grant).unwrap())) + .unwrap(); + store + .dispatch(&reserved.request.job_id, &reserved.attempt.attempt_id, 1) + .unwrap(); + let saved = store + .finish( + &reserved.request.job_id, + &reserved.attempt.attempt_id, + crate::jobs::JobAttemptState::Failed, + Some("host_restart_execution_unconfirmed: retained prior owner"), + ) + .unwrap(); + let jobs = + crate::jobs::StandaloneJobs::open(&body.jobs_path, temp.path().to_path_buf()).unwrap(); + body.host.bind_jobs(jobs.clone()).unwrap(); + Self { + _temp: temp, + body, + jobs, + saved, + execution_owner: store, + } + } + + fn confirm_execution_joined(&mut self) { + // The execution boundary is substituted in this domain-consumer fixture; + // use its exact owner/run to commit the receipt through the real store. + self.execution_owner + .confirm_execution_cleanup( + &self.saved.request.job_id, + &self.saved.executions[0], + "fixture-owned-execution-joined", + ) + .unwrap(); + self.saved = self.jobs.get(&self.saved.request.job_id).unwrap(); + } + + fn resolve_candidate(&self) { + self.body + .host + .catalog + .store + .decide_candidate( + &self.body.binding.subject_id, + subjektiv::CandidateDecisionRequest { + request_id: "decision".into(), + candidate_id: "candidate".into(), + reason: "invalid".into(), + decision: subjektiv::CandidateDecision::Close { + action: subjektiv::StagingResolutionAction::Invalid, + affected_memory: vec![], + }, + }, + ) + .unwrap(); + } + + fn generation(&self, attempt_id: &str) -> subjektiv::SurfaceGeneration { + self.body + .host + .catalog + .store + .prepare_job_surface_generation( + &self.body.binding.subject_id, + Some((&self.saved.request.job_id, attempt_id)), + ) + .unwrap() + } + + fn publish(&self, generation: &subjektiv::SurfaceGeneration) { + self.body + .host + .catalog + .store + .publish_surface_generation(&self.body.binding.subject_id, &generation.id, vec![]) + .unwrap(); + } +} + +#[tokio::test] +async fn consolidation_recovery_accepts_same_attempt_receipts_without_claiming_owner_drain() { + for availability in ["ready", "failed"] { + let fixture = ConsolidationRecoveryFixture::new(); + fixture.resolve_candidate(); + let generation = fixture.generation(&fixture.saved.attempt.attempt_id); + if availability == "ready" { + fixture.publish(&generation); + } else { + fixture + .body + .host + .catalog + .store + .fail_surface_generation( + &fixture.body.binding.subject_id, + &generation.id, + "model_failure", + ) + .unwrap(); + } + let resolution = serde_json::to_value( + fixture + .body + .host + .catalog + .store + .staging_resolution(&fixture.body.binding.subject_id, "candidate") + .unwrap(), + ) + .unwrap(); + assert!(fixture.jobs.pending().unwrap().is_empty()); + fixture.body.host.recover_jobs(); + let completed = fixture.jobs.get(&fixture.saved.request.job_id).unwrap(); + assert_eq!(completed.state, crate::jobs::JobState::Completed); + assert_eq!( + completed.attempt.attempt_id, + fixture.saved.attempt.attempt_id + ); + assert_eq!(completed.attempt.number, fixture.saved.attempt.number); + assert_eq!(completed.input_digest, fixture.saved.input_digest); + assert_eq!(completed.executions, fixture.saved.executions); + assert!(!completed.executions_released()); + assert!(completed.acknowledged); + let result = completed.result.as_ref().unwrap(); + assert_eq!(result["surface"]["generation_id"], generation.id); + assert_eq!(result["surface"]["availability"], availability); + assert_eq!( + result["candidate_dispositions"], + serde_json::json!([{"candidate_id":"candidate","action":"invalid"}]) + ); + if availability == "failed" { + assert_eq!(result["surface"]["reason_code"], "model_failure"); + } + fixture.body.host.recover_jobs(); + assert_eq!( + fixture.jobs.get(&completed.request.job_id).unwrap(), + completed + ); + assert_eq!( + serde_json::to_value( + fixture + .body + .host + .catalog + .store + .staging_resolution(&fixture.body.binding.subject_id, "candidate") + .unwrap(), + ) + .unwrap(), + resolution + ); + // A separate production store read proves settlement was persisted, not + // merely projected by the dispatcher; no execution cleanup is inferred. + let reopened = crate::job_store::JobStore::open(&fixture.body.jobs_path).unwrap(); + assert_eq!(reopened.get(&completed.request.job_id).unwrap(), completed); + } +} + +#[tokio::test] +async fn consolidation_recovery_reads_saved_result_after_execution_join_and_writer_fence() { + let mut fixture = ConsolidationRecoveryFixture::new(); + fixture.resolve_candidate(); + let generation = fixture.generation(&fixture.saved.attempt.attempt_id); + fixture.publish(&generation); + fixture.confirm_execution_joined(); + assert!(fixture.saved.executions_released()); + assert!(fixture.saved.executions[0].writes_revoked); + assert!( + fixture + .body + .host + .catalog + .store + .journal_writer_active(&fixture.body.host.writer) + .unwrap() + ); + + fixture.body.host.recover_jobs(); + let completed = fixture.jobs.get(&fixture.saved.request.job_id).unwrap(); + assert_eq!(completed.state, crate::jobs::JobState::Completed); + assert_eq!( + completed.attempt.attempt_id, + fixture.saved.attempt.attempt_id + ); + assert_eq!(completed.input_digest, fixture.saved.input_digest); + assert_eq!(completed.executions, fixture.saved.executions); + assert!(completed.executions_released()); + assert_eq!( + completed.result.as_ref().unwrap()["surface"]["generation_id"], + generation.id + ); +} + +#[tokio::test] +async fn consolidation_recovery_never_replays_incomplete_foreign_or_stale_receipts() { + for case in ["missing_surface", "unresolved", "foreign_attempt", "stale"] { + let fixture = ConsolidationRecoveryFixture::new(); + if case != "unresolved" { + fixture.resolve_candidate(); + } + if case != "missing_surface" { + let attempt = if case == "foreign_attempt" { + "other-attempt" + } else { + &fixture.saved.attempt.attempt_id + }; + let generation = fixture.generation(attempt); + fixture.publish(&generation); + } + if case == "stale" { + fixture + .body + .host + .catalog + .store + .create_memory( + &fixture.body.binding.subject_id, + subjektiv::MemoryDraft::active( + memory::extract::CandidateKind::Lesson, + "new lesson", + "new body", + "reason", + "change", + ), + ) + .unwrap(); + } + let before_surface = fixture + .body + .host + .catalog + .store + .resident_surface(&fixture.body.binding.subject_id) + .unwrap(); + fixture.body.host.recover_jobs(); + tokio::task::yield_now().await; + assert_eq!( + fixture.jobs.get(&fixture.saved.request.job_id).unwrap(), + fixture.saved, + "{case}" + ); + assert_eq!( + fixture + .body + .host + .catalog + .store + .resident_surface(&fixture.body.binding.subject_id) + .unwrap(), + before_surface, + "{case}" + ); + } +} + +#[tokio::test] +async fn consolidation_recovery_rejects_retired_input_version_stop_and_released_writer() { + for case in [ + "erased", + "invalidated", + "version_stopped", + "released_writer", + ] { + let fixture = ConsolidationRecoveryFixture::new(); + fixture.resolve_candidate(); + let generation = fixture.generation(&fixture.saved.attempt.attempt_id); + fixture.publish(&generation); + match case { + "erased" | "invalidated" => { + fixture + .jobs + .reclaim_input( + &fixture.saved.request.job_id, + &fixture.saved.input_digest, + if case == "erased" { + crate::jobs::JobInputRetirementReason::ExplicitErasure + } else { + crate::jobs::JobInputRetirementReason::LegacyInvalidation + }, + ) + .unwrap(); + } + "version_stopped" => { + // Seed the persisted cutover marker, not a new migration or an + // execution termination proof. + rusqlite::Connection::open(&fixture.body.jobs_path) + .unwrap() + .execute( + "INSERT INTO job_version_stops(job_id) VALUES (?1)", + [&fixture.saved.request.job_id], + ) + .unwrap(); + } + "released_writer" => { + fixture + .body + .host + .catalog + .store + .release_journal_writer(&fixture.body.host.writer) + .unwrap(); + } + _ => unreachable!(), + } + let before = fixture.jobs.get(&fixture.saved.request.job_id).unwrap(); + assert!( + fixture + .body + .host + .recover_consolidation_job(&fixture.jobs, &fixture.saved) + .is_err(), + "{case}" + ); + fixture.body.host.recover_jobs(); + assert_eq!( + fixture.jobs.get(&fixture.saved.request.job_id).unwrap(), + before, + "{case}" + ); + assert!(before.result.is_none()); + assert!(!before.executions_released()); + } +} + +#[test] +fn consolidation_recovery_rejects_stale_attempt_and_digest_observations() { + let fixture = ConsolidationRecoveryFixture::new(); + fixture.resolve_candidate(); + let generation = fixture.generation(&fixture.saved.attempt.attempt_id); + fixture.publish(&generation); + for change_attempt in [false, true] { + let mut stale = fixture.saved.clone(); + if change_attempt { + stale.attempt.attempt_id = "other-attempt".into(); + } else { + stale.input_digest = "other-digest".into(); + } + assert!( + fixture + .body + .host + .recover_consolidation_job(&fixture.jobs, &stale) + .is_err() + ); + assert_eq!( + fixture.jobs.get(&fixture.saved.request.job_id).unwrap(), + fixture.saved + ); + } +} + #[test] fn catalog_reopen_preserves_random_scope_subject_and_behavior() { let temp = tempfile::tempdir().unwrap(); @@ -182,8 +602,16 @@ fn different_local_roots_do_not_authorize_shared_subject_ids() { #[test] fn consolidation_operation_policy_is_attempt_bound_and_does_not_inherit_body_rights() { - struct Fence(AtomicBool); + struct Fence(AtomicBool, AtomicBool); impl crate::jobs::JobAttemptFence for Fence { + fn enter(&self) -> Result, String> { + if !self.1.load(Ordering::Acquire) { + return Err("attempt admission closed".into()); + } + self.ensure_live()?; + Ok(Box::new(())) + } + fn ensure_live(&self) -> Result<(), String> { self.0 .load(Ordering::Acquire) @@ -197,7 +625,7 @@ fn consolidation_operation_policy_is_attempt_bound_and_does_not_inherit_body_rig let mut body = catalog .connect(&subject.id, protocol::WorkerId::now_v7()) .unwrap(); - let fence = Arc::new(Fence(AtomicBool::new(true))); + let fence = Arc::new(Fence(AtomicBool::new(true), AtomicBool::new(true))); let job = LocalConsolidationHost { host: body.host.clone(), grant: LocalConsolidationGrant { @@ -255,6 +683,21 @@ fn consolidation_operation_policy_is_attempt_bound_and_does_not_inherit_body_rig ) .is_err() ); + fence.1.store(false, Ordering::Release); + assert!( + job.authorize_operation(&context, "memory.staging.read") + .is_ok() + ); + assert!(matches!( + job.memory( + &context, + SubjektivMemoryBackendOperation::ReadCandidate(SubjektivMemoryCandidateReadRequest { + candidate_id: "outside-batch".into() + }) + ), + Err(SubjektivHostError::Unavailable(reason)) if reason == "attempt admission closed" + )); + fence.1.store(true, Ordering::Release); fence.0.store(false, Ordering::Release); assert!( job.authorize_operation(&context, "memory.staging.list") @@ -297,6 +740,13 @@ fn session_registration_uses_real_capability_and_attribution_decisions() { .session_registration_cause(), SessionRegistrationCause::binding_mismatch(), ); + // Normal factory startup creates the Worker Session store before registration. + // Keep that real filesystem prerequisite without pre-creating attribution. + let _sessions = session_store::FsStore::new_with_retention( + root.join(id.to_string()).join("sessions"), + crate::store::session_retention(&root), + ) + .unwrap(); let success = port.record_session(&context, true).unwrap(); assert_eq!(success.subject_id, subject.id); assert_eq!(success.session_id, context.session_id); diff --git a/crates/standalone/tests/jobs.rs b/crates/standalone/tests/jobs.rs index c7c35346a..5f549275f 100644 --- a/crates/standalone/tests/jobs.rs +++ b/crates/standalone/tests/jobs.rs @@ -137,6 +137,18 @@ async fn host_job_runs_without_dialog_and_result_survives_restore_and_ack() { assert_eq!(completed.state, JobState::Completed); assert_eq!(completed.result, Some(serde_json::json!({"valid":true}))); assert!(!completed.acknowledged); + assert!(completed.executions_released()); + assert_eq!(completed.executions.len(), 1); + assert_eq!( + completed.executions[0].run_id, + worker::job::execution_run_id(&intent, &completed.attempt.attempt_id) + .unwrap() + .to_string() + ); + assert_eq!( + completed.executions[0].cleanup_receipt.as_deref(), + Some(format!("owned-run:{}", completed.executions[0].run_id).as_str()) + ); let neutral = completed.outcome(); assert_eq!(neutral.state, job::JobState::Completed); assert_eq!(neutral.result, completed.result); @@ -196,6 +208,46 @@ async fn host_job_runs_without_dialog_and_result_survives_restore_and_ack() { restored.shutdown().await.unwrap(); } +#[tokio::test] +async fn host_restore_resumes_reserved_result_job_on_same_attempt_without_client_action() { + let temp = tempfile::tempdir().unwrap(); + let (host, dialog) = host(&temp).await; + let worker_id = host.worker_id(); + let saved = host + .jobs() + .request(request("reserved-before-dispatch")) + .unwrap(); + host.shutdown().await.unwrap(); + assert!(dialog.requests().is_empty()); + + let model = success(); + let restored = StandaloneHost::restore_with_model_client( + temp.path().join("state"), + worker_id, + model.clone(), + ) + .await + .unwrap(); + // The production from_started dispatcher, not a test start/retry/probe call, + // must supply the already-accepted immutable input to the result-only Job. + wait_started(&model).await; + let completed = tokio::time::timeout( + Duration::from_secs(10), + restored.jobs().wait("reserved-before-dispatch"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.state, JobState::Completed); + assert_eq!(completed.attempt.attempt_id, saved.attempt.attempt_id); + assert_eq!(completed.input_digest, saved.input_digest); + assert_eq!(completed.request, saved.request); + assert_eq!(completed.executions.len(), 1); + assert!(completed.executions_released()); + assert_eq!(model.requests().len(), 2); + restored.shutdown().await.unwrap(); +} + #[tokio::test] async fn live_resource_limits_cancellation_and_wait_are_host_owned() { let temp = tempfile::tempdir().unwrap(); @@ -261,13 +313,13 @@ async fn prose_failure_requires_explicit_bounded_new_attempt_and_exact_replay_do } #[tokio::test] -async fn timeout_is_unknown_not_prose_success_or_automatic_retry() { +async fn timeout_fails_without_automatic_retry_and_cleanup_allows_explicit_retry() { let temp = tempfile::tempdir().unwrap(); let (host, _) = host(&temp).await; let jobs = host.jobs(); let mut intent = request("timeout"); intent.limits.timeout_seconds = 1; - jobs.request(intent).unwrap(); + let reserved = jobs.request(intent.clone()).unwrap(); let stalled = ScriptedClient::stall(); jobs.start_with_model_client("timeout", stalled.clone()) .await @@ -277,22 +329,45 @@ async fn timeout_is_unknown_not_prose_success_or_automatic_retry() { .await .unwrap() .unwrap(); - assert_eq!(outcome.state, JobState::Unknown); + assert_eq!(outcome.state, JobState::Failed); + assert_eq!( + outcome.attempt.state, + standalone::jobs::JobAttemptState::Failed + ); + assert_eq!(outcome.attempt.failure.as_deref(), Some("timeout")); + assert_eq!(outcome.attempt.attempt_id, reserved.attempt.attempt_id); + assert_eq!(outcome.request, intent); assert!(outcome.result.is_none()); - assert!(jobs.retry("timeout").is_err()); + assert!(outcome.executions_released()); + assert_eq!(jobs.request(intent).unwrap(), outcome); + assert_eq!( + stalled.requests().len(), + 1, + "timeout never reexecutes the model" + ); + let retried = jobs.retry("timeout").unwrap(); + assert_eq!(retried.attempt.number, 2); + assert_ne!(retried.attempt.attempt_id, outcome.attempt.attempt_id); + jobs.start_with_model_client("timeout", success()) + .await + .unwrap(); + assert_eq!( + jobs.wait("timeout").await.unwrap().state, + JobState::Completed + ); host.shutdown().await.unwrap(); } #[tokio::test] -async fn shutdown_keeps_reserved_results_and_fences_running_attempts_on_reopen() { +async fn shutdown_preserves_reserved_intent_and_reopen_preserves_confirmed_failure() { let temp = tempfile::tempdir().unwrap(); let (host, _) = host(&temp).await; let id = host.worker_id(); let jobs = host.jobs(); - jobs.request(request("reserved")).unwrap(); + let reserved = jobs.request(request("reserved")).unwrap(); jobs.request(request("cancelled")).unwrap(); - jobs.cancel("cancelled").unwrap(); - jobs.request(request("running")).unwrap(); + let cancelled = jobs.cancel("cancelled").unwrap(); + let running = jobs.request(request("running")).unwrap(); let stalled = ScriptedClient::stall(); jobs.start_with_model_client("running", stalled.clone()) .await @@ -300,32 +375,51 @@ async fn shutdown_keeps_reserved_results_and_fences_running_attempts_on_reopen() wait_started(&stalled).await; host.shutdown().await.unwrap(); assert!(matches!(jobs.get("running"), Err(JobServiceError::Closed))); + let resume_model = success(); let restored = StandaloneHost::restore_with_model_client( temp.path().join("state"), id, - ScriptedClient::new(Vec::new()), + resume_model.clone(), ) .await .unwrap(); + wait_started(&resume_model).await; let jobs = restored.jobs(); + let resumed = jobs.wait("reserved").await.unwrap(); + assert_eq!(resumed.state, JobState::Completed); + assert_eq!(resumed.attempt.attempt_id, reserved.attempt.attempt_id); + assert_eq!(resumed.input_digest, reserved.input_digest); + assert_eq!(resumed.request, reserved.request); + assert!(jobs.pending().unwrap().is_empty()); + assert_eq!(jobs.get("cancelled").unwrap(), cancelled); + let stopped = jobs.get("running").unwrap(); + assert_eq!(stopped.state, JobState::Failed); assert_eq!( - jobs.pending() - .unwrap() - .iter() - .map(|snapshot| snapshot.request.job_id.as_str()) - .collect::>(), - ["reserved"] + stopped.attempt.state, + standalone::jobs::JobAttemptState::Failed ); - assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled); - assert_eq!(jobs.get("running").unwrap().state, JobState::Unknown); - assert!(jobs.retry("running").is_err()); - jobs.start_with_model_client("reserved", success()) + assert_eq!( + stopped.attempt.failure.as_deref(), + Some("Host cancellation/shutdown") + ); + assert_eq!(stopped.request, running.request); + assert_eq!(stopped.attempt.attempt_id, running.attempt.attempt_id); + assert!(stopped.executions_released()); + assert!( + !stopped.version_stopped, + "ordinary shutdown is not a version cutover" + ); + let retry = jobs.retry("running").unwrap(); + assert_eq!(retry.attempt.number, 2); + jobs.start_with_model_client("running", success()) .await .unwrap(); assert_eq!( - jobs.wait("reserved").await.unwrap().state, + jobs.wait("running").await.unwrap().state, JobState::Completed ); + assert_eq!(jobs.get("reserved").unwrap(), resumed); + assert_eq!(resume_model.requests().len(), 2); restored.shutdown().await.unwrap(); } @@ -357,7 +451,7 @@ async fn profile_requirements_are_rejected_without_model_execution_or_implicit_f // The child exits deliberately without destructors to prove actual process-loss // recovery, rather than treating a hand-edited SQLite fixture as that evidence. #[tokio::test] -#[ignore = "entry point for process_loss_recovers_durable_jobs_without_replaying_unknown"] +#[ignore = "entry point for process_loss_fails_same_attempt_without_claiming_execution_cleanup"] async fn process_job_fixture() { let root = std::path::PathBuf::from( std::env::var_os("YOI_JOB_PROCESS_FIXTURE_DIR").expect("fixture root"), @@ -392,12 +486,17 @@ async fn process_job_fixture() { .await .unwrap(); wait_started(&stalled).await; + std::fs::write( + root.join("interrupted-snapshot.json"), + serde_json::to_vec(&jobs.get("interrupted").unwrap()).unwrap(), + ) + .unwrap(); // Test-owned process only, never the operating Yoi Worker or dogfood daemon. std::process::exit(23); } #[tokio::test] -async fn process_loss_recovers_durable_jobs_without_replaying_unknown() { +async fn process_loss_fails_same_attempt_without_claiming_execution_cleanup() { let temp = tempfile::tempdir().unwrap(); let output = std::process::Command::new(std::env::current_exe().unwrap()) .args(["--exact", "process_job_fixture", "--ignored", "--nocapture"]) @@ -415,34 +514,73 @@ async fn process_loss_recovers_durable_jobs_without_replaying_unknown() { .unwrap() .parse() .unwrap(); + let recovery_model = success(); let host = StandaloneHost::restore_with_model_client( temp.path().join("state"), worker_id, - ScriptedClient::new(Vec::new()), + recovery_model.clone(), ) .await .unwrap(); let jobs = host.jobs(); - assert_eq!(jobs.get("interrupted").unwrap().state, JobState::Unknown); + let persisted: standalone::jobs::JobSnapshot = serde_json::from_slice( + &std::fs::read(temp.path().join("interrupted-snapshot.json")).unwrap(), + ) + .unwrap(); + let interrupted = jobs.get("interrupted").unwrap(); + assert_eq!(interrupted.request, persisted.request); + assert_eq!(interrupted.input_digest, persisted.input_digest); + assert_eq!(interrupted.attempt.attempt_id, persisted.attempt.attempt_id); + assert_eq!(interrupted.attempt.number, persisted.attempt.number); + assert_eq!(interrupted.executions, persisted.executions); + assert_eq!(interrupted.result, persisted.result); + assert_eq!(interrupted.state, JobState::Failed); + assert_eq!( + interrupted.attempt.state, + standalone::jobs::JobAttemptState::Failed + ); + assert!( + interrupted + .attempt + .failure + .as_deref() + .unwrap() + .starts_with("host_restart_execution_unconfirmed:") + ); + assert_eq!(interrupted.executions.len(), 1); + assert!(!interrupted.executions_released()); + assert!(interrupted.executions[0].cleanup_receipt.is_none()); + assert!(!interrupted.executions[0].writes_revoked); + assert!(!interrupted.version_stopped); assert!(jobs.retry("interrupted").is_err()); + let replay = success(); + assert!( + jobs.start_with_model_client("interrupted", replay.clone()) + .await + .is_err() + ); + assert!(replay.requests().is_empty()); + assert_eq!(jobs.wait("interrupted").await.unwrap(), interrupted); let completed = jobs.get("accepted").unwrap(); assert_eq!(completed.state, JobState::Completed); assert_eq!(completed.result, Some(serde_json::json!({"valid":true}))); assert!(!completed.acknowledged); assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled); - assert_eq!(jobs.pending().unwrap().len(), 1); - jobs.start_with_model_client("reserved", success()) - .await - .unwrap(); + wait_started(&recovery_model).await; + let reserved = jobs.wait("reserved").await.unwrap(); + assert_eq!(reserved.state, JobState::Completed); + assert_eq!(reserved.attempt.number, 1); + assert_eq!(recovery_model.requests().len(), 2); + assert!(jobs.pending().unwrap().is_empty()); + assert_eq!(jobs.get("interrupted").unwrap(), interrupted); assert_eq!( - jobs.wait("reserved").await.unwrap().state, - JobState::Completed + host.shutdown().await, + Err(standalone::StandaloneShutdownError::JobCleanup) ); - host.shutdown().await.unwrap(); } #[tokio::test] -async fn model_failure_is_durable_known_failure_with_explicit_successful_retry() { +async fn model_failure_retains_unconfirmed_execution_and_forbids_retry() { let temp = tempfile::tempdir().unwrap(); let (host, _) = host(&temp).await; let jobs = host.jobs(); @@ -454,24 +592,26 @@ async fn model_failure_is_durable_known_failure_with_explicit_successful_retry() jobs.start_with_model_client("model-failure", client) .await .unwrap(); - let failure = jobs.wait("model-failure").await.unwrap(); + assert!(matches!( + jobs.wait("model-failure").await, + Err(JobServiceError::Cleanup(_)) + )); + let failure = jobs.get("model-failure").unwrap(); assert_eq!(failure.state, JobState::Failed); assert!(failure.result.is_none()); assert!( failure .attempt .failure + .as_deref() .unwrap() .contains("scripted model failure") ); - let next = jobs.retry("model-failure").unwrap(); - assert_eq!(next.attempt.number, 2); - jobs.start_with_model_client("model-failure", success()) - .await - .unwrap(); + assert!(!failure.executions_released()); + assert!(jobs.retry("model-failure").is_err()); + assert_eq!(jobs.get("model-failure").unwrap(), failure); assert_eq!( - jobs.wait("model-failure").await.unwrap().state, - JobState::Completed + host.shutdown().await, + Err(standalone::StandaloneShutdownError::JobCleanup) ); - host.shutdown().await.unwrap(); } diff --git a/crates/subjektiv/src/job.rs b/crates/subjektiv/src/job.rs index 656d341d2..7c7b0b21c 100644 --- a/crates/subjektiv/src/job.rs +++ b/crates/subjektiv/src/job.rs @@ -118,6 +118,93 @@ pub fn validate_result( }) } +/// Reconstructs only a stored terminal outcome for this exact Job/attempt. +/// Missing work returns `None`; stale, erased, retired or inconsistent evidence is +/// an error. No candidate decision, generation, model call or finalizer is rerun. +/// Hosts still fence current grants and independently retain execution cleanup. +pub fn lookup_result( + store: &SubjektivStore, + subject_id: &str, + batch: &[String], + job_id: &str, + attempt_id: &str, +) -> OperationResult> { + use crate::SubjektivError; + use rusqlite::{OptionalExtension, params}; + + if batch.len() > CANDIDATE_BATCH_LIMIT { + return Err(OperationError::InvalidInput( + "consolidation batch exceeds limit".into(), + )); + } + let result = store.database.try_transaction(|tx| { + crate::require_active_subject(tx, subject_id)?; + crate::validate_label("Job id", job_id)?; + crate::validate_label("attempt id", attempt_id)?; + let mut complete = true; + for id in batch { + let candidate: Option = tx.query_row( + "SELECT record_json FROM staging_records WHERE subject_id=?1 AND candidate_id=?2", + params![subject_id, id], |row| row.get(0) + ).optional()?; + let candidate = candidate.ok_or_else(|| SubjektivError::CandidateNotFound(id.clone()))?; + crate::parse_staging(store.workspace_id(), &candidate)?; + let resolution: Option = tx.query_row( + "SELECT resolution_json FROM staging_resolutions WHERE subject_id=?1 AND candidate_id=?2", + params![subject_id, id], |row| row.get(0) + ).optional()?; + match resolution { + Some(raw) => { + let saved = crate::parse_resolution(&raw)?; + if saved.subject_id != subject_id || saved.candidate_id != *id { + return Err(SubjektivError::InvalidRecord("consolidation disposition identity mismatch".into())); + } + } + None => complete = false, + } + } + // Use the existing immutable generation binding and terminal fingerprint + // contract, not arrival time or a newly prepared generation. Several + // preparations for the same input may share a stored outcome. + let fingerprint = crate::memory_fingerprint_in_connection(tx, subject_id)?; + let generation: Option = tx.query_row( + "SELECT generation_id FROM surface_generation_runs + WHERE subject_id=?1 AND job_id=?2 AND attempt_id=?3 AND memory_fingerprint=?4 + ORDER BY generation_id LIMIT 1", + params![subject_id, job_id, attempt_id, fingerprint], |row| row.get(0) + ).optional()?; + let Some(generation_id) = generation else { + let prior = tx.prepare( + "SELECT 1 FROM surface_generation_runs WHERE subject_id=?1 AND job_id=?2 AND attempt_id=?3" + )?.exists(params![subject_id, job_id, attempt_id])?; + if prior { + return Err(SubjektivError::InvalidRecord("consolidation generation input is stale".into())); + } + return Ok(None); + }; + let state: Option<(String, String, Option, Option)> = tx.query_row( + "SELECT memory_fingerprint,status,snapshot_id,reason_code FROM surface_generation_state WHERE subject_id=?1", + [subject_id], |row| Ok((row.get(0)?,row.get(1)?,row.get(2)?,row.get(3)?)) + ).optional()?; + let Some((saved_fingerprint, status, snapshot_id, reason_code)) = state else { return Ok(None) }; + if saved_fingerprint != fingerprint || !matches!(status.as_str(), "ready" | "failed") { + return Err(SubjektivError::InvalidRecord("consolidation terminal outcome is stale or inconsistent".into())); + } + if !complete { return Ok(None) } + let mut surface = serde_json::json!({ + "availability": status, "generation_id": generation_id, "memory_fingerprint": fingerprint + }); + if let Some(id) = snapshot_id { surface["snapshot_id"] = id.into(); } + if let Some(reason) = reason_code { surface["reason_code"] = reason.into(); } + Ok(Some(serde_json::json!({"subject_id":subject_id, "candidate_ids":batch,"surface":surface}))) + }).map_err(|error: SubjektivError| OperationError::Store(error.to_string()))?; + let Some(result) = result else { + return Ok(None); + }; + validate_result(store, subject_id, batch, job_id, attempt_id, &result)?; + with_dispositions(store, subject_id, batch, &result).map(Some) +} + /// Adds a compact projection of stored dispositions to a validated result. /// Reasons, change references, and provenance stay in the receipt store, not duplicated in bounded Job results. /// Neither reapplies candidates nor creates a new ledger. diff --git a/crates/subjektiv/src/job/tests.rs b/crates/subjektiv/src/job/tests.rs index 5cc3dcf6d..3419173b9 100644 --- a/crates/subjektiv/src/job/tests.rs +++ b/crates/subjektiv/src/job/tests.rs @@ -127,6 +127,11 @@ fn result_checks_durable_dispositions_and_exact_job_surface_outcome() { .contains("unresolved") ); assert!(with_dispositions(&store, &subject.id, &batch, &ready).is_err()); + assert!( + lookup_result(&store, &subject.id, &batch, "job", "attempt") + .unwrap() + .is_none() + ); store .decide_candidate( &subject.id, @@ -143,6 +148,10 @@ fn result_checks_durable_dispositions_and_exact_job_surface_outcome() { .unwrap(); validate_result(&store, &subject.id, &batch, "job", "attempt", &ready).unwrap(); let enriched = with_dispositions(&store, &subject.id, &batch, &ready).unwrap(); + assert_eq!( + lookup_result(&store, &subject.id, &batch, "job", "attempt").unwrap(), + Some(enriched.clone()) + ); assert_eq!( enriched["candidate_dispositions"], serde_json::json!([{"candidate_id": "candidate", "action": "invalid"}]) @@ -176,6 +185,12 @@ fn result_checks_durable_dispositions_and_exact_job_surface_outcome() { let failed = serde_json::json!({"subject_id": subject.id, "candidate_ids": batch, "surface": {"availability": "failed", "generation_id": failure.id, "memory_fingerprint": memory_fingerprint, "reason_code": "model_failure"}}); validate_result(&store, &subject.id, &batch, "job", "attempt", &failed).unwrap(); + // The same attempt can prepare again after committing Memory. Recovery must + // select its compatible stored outcome, not its first/last-arriving generation. + assert_eq!( + lookup_result(&store, &subject.id, &batch, "job", "attempt").unwrap(), + Some(with_dispositions(&store, &subject.id, &batch, &failed).unwrap()) + ); assert!(validate_result(&store, &subject.id, &batch, "job", "attempt", &ready).is_err()); let mut false_reason = failed; false_reason["surface"]["reason_code"] = serde_json::json!("invented"); diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index bb19a307c..969908f2c 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -80,20 +80,86 @@ pub struct WorkerNotificationAck { pub notification_request_id: String, } +/// Actual execution-owner termination, not Backend Host drain, writer revocation, +/// disposable-resource cleanup, or completion of the owning domain operation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerExecutionTermination { + pub scope: WorkerExecutionTerminationScope, + pub worker_ref: WorkerRef, + pub stop_operation_id: WorkerLifecycleOperationId, + pub execution_id: uuid::Uuid, + pub shutdown_command_id: Option, + /// Matching durable acknowledgements observed by this execution owner; + /// absence does not establish that another submission was unaccepted. + pub submissions: Vec, + /// Nonempty failures leave Feature scope termination/cleanup unconfirmed, + /// even though the Controller and adapter-owned tasks have joined. + pub feature_failures: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkerExecutionTerminationScope { + Controller, + UnpublishedStartupCandidate, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerFeatureShutdownFailure { + pub category: worker::hook::HookErrorCategory, + pub diagnostic: String, +} + +impl WorkerExecutionTermination { + pub(crate) fn joined_startup_candidate( + worker_ref: WorkerRef, + stop_operation_id: WorkerLifecycleOperationId, + execution_id: uuid::Uuid, + ) -> Self { + Self { + scope: WorkerExecutionTerminationScope::UnpublishedStartupCandidate, + worker_ref, + stop_operation_id, + execution_id, + shutdown_command_id: None, + submissions: Vec::new(), + feature_failures: Vec::new(), + } + } + + pub fn confirms_scope_shutdown( + &self, + worker_ref: &WorkerRef, + operation_id: WorkerLifecycleOperationId, + ) -> bool { + &self.worker_ref == worker_ref + && self.stop_operation_id == operation_id + && self.feature_failures.is_empty() + } +} + /// Typed execution result class. Results are transient operation outcomes and /// are not persisted as Worker lifecycle authority. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkerExecutionResult { pub operation: WorkerExecutionOperation, pub outcome: WorkerExecutionOutcome, + /// Actual Controller targeted or observed by this operation. Receipt lookup + /// does not identify the originating run of a receipt retained across restore. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub execution_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub worker_state: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub message: Option, + /// Durable acceptance, independent of a later operation failure. Receipt + /// reconciliation can retain this on an Errored result; it is not run completion. #[serde(default, skip_serializing_if = "Option::is_none")] pub submission: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub notification: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub termination: Option, /// Projected primary facts. Never stop, cleanup, or retry evidence. #[serde(default, skip_serializing_if = "Option::is_none")] pub startup_failure: Option, @@ -123,10 +189,12 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Accepted, + execution_id: None, worker_state: None, message: None, submission: None, notification: None, + termination: None, startup_failure: None, secondary_failures: Vec::new(), startup_resources: None, @@ -149,6 +217,7 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Accepted, + execution_id: None, worker_state: None, message: None, submission: Some(WorkerSubmissionAck { @@ -157,6 +226,7 @@ impl WorkerExecutionResult { disposition, }), notification: None, + termination: None, startup_failure: None, secondary_failures: Vec::new(), startup_resources: None, @@ -172,12 +242,14 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Accepted, + execution_id: None, worker_state: None, message: None, submission: None, notification: Some(WorkerNotificationAck { notification_request_id: notification_request_id.into(), }), + termination: None, startup_failure: None, secondary_failures: Vec::new(), startup_resources: None, @@ -190,10 +262,12 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Busy, + execution_id: None, worker_state: None, message: Some(message.into()), submission: None, notification: None, + termination: None, startup_failure: None, secondary_failures: Vec::new(), startup_resources: None, @@ -206,10 +280,12 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Rejected, + execution_id: None, worker_state: None, message: Some(message.into()), submission: None, notification: None, + termination: None, startup_failure: None, secondary_failures: Vec::new(), startup_resources: None, @@ -222,10 +298,12 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Errored, + execution_id: None, worker_state: None, message: Some(message.into()), submission: None, notification: None, + termination: None, startup_failure: None, secondary_failures: Vec::new(), startup_resources: None, @@ -238,10 +316,12 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Unsupported, + execution_id: None, worker_state: None, message: Some(message.into()), submission: None, notification: None, + termination: None, startup_failure: None, secondary_failures: Vec::new(), startup_resources: None, @@ -730,6 +810,7 @@ pub struct WorkerProtocolTransport { dispatch: Arc Result, WorkerExecutionResult> + Send + Sync>, validate: Arc Result<(), WorkerExecutionResult> + Send + Sync>, + stop: Option WorkerExecutionResult + Send + Sync>>, } #[cfg(feature = "ws-server")] @@ -749,6 +830,25 @@ impl WorkerProtocolTransport { events, dispatch, validate, + stop: None, + } + } + + pub fn with_stop_dispatch( + mut self, + stop: Arc WorkerExecutionResult + Send + Sync>, + ) -> Self { + self.stop = Some(stop); + self + } + + pub fn stop(&self, request: WorkerExecutionStopRequest) -> WorkerExecutionResult { + match &self.stop { + Some(stop) => stop(request), + None => WorkerExecutionResult::unsupported( + WorkerExecutionOperation::Stop, + "bound protocol transport cannot confirm scope shutdown", + ), } } @@ -1034,6 +1134,29 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { Ok(()) } + /// Look up only the receipt for this exact input (request ID, source namespace + /// and payload digest). Missing or inaccessible receipts remain unknown; this + /// operation must not submit new work. + fn reconcile_submission( + &self, + _worker_ref: &WorkerRef, + _input: WorkerInput, + ) -> WorkerExecutionResult { + WorkerExecutionResult::errored( + WorkerExecutionOperation::Input, + "submission acceptance is unknown; exact receipt lookup is unavailable", + ) + } + + /// Read an actual retained scope-termination receipt. A backend must reject + /// receipts belonging to a replaced execution; absence is not shutdown proof. + fn termination_receipt( + &self, + _worker_ref: &WorkerRef, + ) -> Result, String> { + Err("execution backend has no scope-termination receipt reader".into()) + } + fn dispatch_input(&self, worker_ref: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult; fn upload_file( @@ -1327,6 +1450,21 @@ impl WorkerExecutionBackendRef { self.backend.observe_workspace_prompt_projection(projection) } + pub(crate) fn reconcile_submission( + &self, + worker_ref: &WorkerRef, + input: WorkerInput, + ) -> WorkerExecutionResult { + self.backend.reconcile_submission(worker_ref, input) + } + + pub(crate) fn termination_receipt( + &self, + worker_ref: &WorkerRef, + ) -> Result, String> { + self.backend.termination_receipt(worker_ref) + } + pub(crate) fn dispatch_input( &self, worker_ref: &WorkerRef, diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 5d04e4c6d..0a4729380 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -3556,6 +3556,29 @@ mod tests { fn stop_worker(&self, _worker_ref: &WorkerRef) -> WorkerExecutionResult { WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) } + + fn stop_worker_operation( + &self, + request: crate::execution::WorkerExecutionStopRequest, + ) -> WorkerExecutionResult { + // Only execution is substituted in these HTTP/retention fixtures. + // Model its bound stop report explicitly; do not relax Runtime's + // production requirement or claim real Controller/Host coverage. + let execution_id = + uuid::Uuid::parse_str(&request.worker_ref.worker_id.to_string()).unwrap(); + let mut result = self.stop_worker(&request.worker_ref); + result.execution_id = Some(execution_id); + result.termination = Some(crate::execution::WorkerExecutionTermination { + scope: crate::execution::WorkerExecutionTerminationScope::Controller, + worker_ref: request.worker_ref, + stop_operation_id: request.operation_id, + execution_id, + shutdown_command_id: None, + submissions: Vec::new(), + feature_failures: Vec::new(), + }); + result + } } #[cfg(feature = "fs-store")] diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 3c57b93c0..01c8445a6 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1842,6 +1842,44 @@ impl Runtime { Ok(worker.detail()) } + /// Look up the originally reserved Job Worker by its saved creation identity. + /// This confirms identity only: it never creates/activates a Controller or + /// proves Submit acceptance, execution termination, or Host drain. + pub fn lookup_job_worker_creation( + &self, + workspace_id: &str, + worker_ref: &WorkerRef, + create_fingerprint: &str, + binding: &crate::catalog::BackendJobExecutionBinding, + ) -> Result, RuntimeError> { + if workspace_id.is_empty() || create_fingerprint.is_empty() { + return Err(RuntimeError::InvalidRequest( + "Job creation lookup requires Workspace and fingerprint".into(), + )); + } + let state = self.lock()?; + let Some(worker) = state.workers.get(&worker_ref.worker_id) else { + return Ok(None); + }; + let request = worker + .request + .as_ref() + .filter(|_| worker.execution_metadata_available) + .ok_or_else(|| RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "saved Job Worker creation metadata unavailable".into(), + })?; + if worker.workspace_id.as_deref() != Some(workspace_id) + || request.create_fingerprint != create_fingerprint + || request.backend_job.as_ref() != Some(binding) + { + return Err(RuntimeError::InvalidRequest( + "saved Job Worker creation binding mismatch".into(), + )); + } + Ok(Some(worker.detail())) + } + fn ensure_worker_in_workspace( &self, scope: &RuntimeWorkspaceScope, @@ -3217,6 +3255,102 @@ impl Runtime { self.send_input_with_scope(worker_ref, input, None) } + /// Read the execution owner's retained termination receipt without stopping, + /// restoring or inferring shutdown from persisted status or process restart. + pub fn execution_termination( + &self, + worker_ref: &WorkerRef, + ) -> Result, RuntimeError> { + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + let backend = { + let state = self.lock()?; + state.ensure_worker_ref(worker_ref)?; + state.worker(worker_ref)?; + state.execution_backend.clone().ok_or_else(|| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "runtime has no execution backend".into(), + } + })? + }; + let receipt = backend.termination_receipt(worker_ref).map_err(|message| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message, + } + })?; + if receipt + .as_ref() + .is_some_and(|receipt| &receipt.worker_ref != worker_ref) + { + return Err(RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "scope-termination receipt Worker mismatch".into(), + }); + } + Ok(receipt) + } + + /// Read the existing exact Submit receipt without dispatching input, restoring a + /// Controller, changing lifecycle state, or replacing retained execution failures. + /// A missing receipt does not establish nonacceptance. + pub fn reconcile_input( + &self, + worker_ref: &WorkerRef, + input: WorkerInput, + ) -> Result { + validate_worker_input(&input)?; + let expected = input + .submission_request_id + .as_deref() + .filter(|id| !id.trim().is_empty()) + .ok_or_else(|| { + RuntimeError::InvalidRequest( + "receipt lookup requires a stable submission request id".into(), + ) + })?; + if !matches!( + input.kind, + WorkerInputKind::User | WorkerInputKind::UserIfIdle + ) { + return Err(RuntimeError::InvalidRequest( + "receipt lookup requires Submit input".into(), + )); + } + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + let backend = { + let state = self.lock()?; + state.ensure_running()?; + state.ensure_worker_ref(worker_ref)?; + state.worker(worker_ref)?; + state.execution_backend.clone().ok_or_else(|| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "runtime has no execution backend".into(), + } + })? + }; + let mut receipt = backend.reconcile_submission(worker_ref, input.clone()); + if receipt.is_accepted() + && (receipt.operation != WorkerExecutionOperation::Input + || receipt.submission.as_ref().is_none_or(|ack| { + ack.submission_request_id != expected || ack.submission_id.trim().is_empty() + })) + { + receipt.outcome = crate::execution::WorkerExecutionOutcome::Errored; + receipt.submission = None; + receipt.message = + Some("exact submission receipt mismatch; acceptance is not established".into()); + } + Ok(receipt) + } + fn send_input_with_scope( &self, worker_ref: &WorkerRef, @@ -3283,7 +3417,47 @@ impl Runtime { })? }; - let dispatch_result = backend.dispatch_input(worker_ref, input.clone()); + let mut dispatch_result = backend.dispatch_input(worker_ref, input.clone()); + if let Some(expected) = expected_submission_id.as_ref() + && matches!( + dispatch_result.outcome, + crate::execution::WorkerExecutionOutcome::Accepted + | crate::execution::WorkerExecutionOutcome::Errored + ) + && dispatch_result.submission.as_ref().is_none_or(|ack| { + &ack.submission_request_id != expected || ack.submission_id.trim().is_empty() + }) + { + let receipt = backend.reconcile_submission(worker_ref, input.clone()); + if receipt.is_accepted() + && receipt.operation == WorkerExecutionOperation::Input + && dispatch_result + .execution_id + .is_none_or(|execution_id| receipt.execution_id == Some(execution_id)) + && receipt.submission.as_ref().is_some_and(|ack| { + &ack.submission_request_id == expected && !ack.submission_id.trim().is_empty() + }) + { + // Receipt acceptance is independent of a later dispatch failure. + // Keep the original outcome and startup failure attachments intact. + dispatch_result.submission = receipt.submission; + if dispatch_result.is_accepted() && receipt.worker_state.is_some() { + dispatch_result.worker_state = receipt.worker_state; + } + } else { + // A missing receipt, changed Controller or mismatched acknowledgement + // cannot establish rejection. Do not retain a foreign request's ack. + dispatch_result.submission = None; + dispatch_result.outcome = crate::execution::WorkerExecutionOutcome::Errored; + let uncertainty = format!( + "submission acceptance is unknown for request {expected}; exact receipt reconciliation did not confirm acceptance on the same execution" + ); + dispatch_result.message = Some(match dispatch_result.message.take() { + Some(message) => format!("{message}; {uncertainty}"), + None => uncertainty, + }); + } + } if !dispatch_result.is_accepted() { self.record_execution_result(worker_ref, dispatch_result.clone())?; return Err(RuntimeError::WorkerExecutionRejected { @@ -3294,25 +3468,6 @@ impl Runtime { result: dispatch_result, }); } - if let Some(expected_submission_id) = expected_submission_id - && dispatch_result - .submission - .as_ref() - .is_none_or(|ack| ack.submission_request_id != expected_submission_id) - { - let result = WorkerExecutionResult::rejected( - WorkerExecutionOperation::Input, - "execution backend did not acknowledge the committed Runtime submission request id", - ); - self.record_execution_result(worker_ref, result.clone())?; - return Err(RuntimeError::WorkerExecutionRejected { - worker_id: worker_ref.worker_id.clone(), - operation: result.operation, - outcome: result.outcome, - message: result.message_or_default(), - result, - }); - } if let Some(expected_notification_id) = expected_notification_id && dispatch_result @@ -3805,9 +3960,12 @@ impl Runtime { if matches!(&method, Method::Shutdown { .. }) { let operation_id = WorkerLifecycleOperationId::new(); self.begin_stop_operation(worker_ref, operation_id)?; - let events = transport.dispatch(method).map_err(rejection)?; - self.commit_stopped_worker(worker_ref, operation_id)?; - return Ok(events); + let result = transport.stop(WorkerExecutionStopRequest { + operation_id, + worker_ref: worker_ref.clone(), + }); + self.commit_stopped_worker(worker_ref, operation_id, result)?; + return Ok(Vec::new()); } transport.dispatch(method).map_err(rejection) } @@ -3852,8 +4010,16 @@ impl Runtime { backend: &WorkerExecutionBackendRef, worker_ref: &WorkerRef, ) -> Result<(), RuntimeError> { - let stop_result = backend.stop_worker(worker_ref); - if stop_result.is_accepted() { + let operation_id = WorkerLifecycleOperationId::new(); + let stop_result = backend.stop_worker_operation(WorkerExecutionStopRequest { + operation_id, + worker_ref: worker_ref.clone(), + }); + if stop_result.is_accepted() + && stop_result.termination.as_ref().is_some_and(|termination| { + termination.confirms_scope_shutdown(worker_ref, operation_id) + }) + { return self.rollback_failed_create(worker_ref); } let mut state = self.lock()?; @@ -4197,7 +4363,7 @@ impl Runtime { result, }); } - self.commit_stopped_worker(worker_ref, operation_id) + self.commit_stopped_worker(worker_ref, operation_id, result) } fn admit_worker_stop_under_lock( @@ -4945,7 +5111,7 @@ impl Runtime { result, }); } - self.commit_stopped_worker(worker_ref, operation_id)?; + self.commit_stopped_worker(worker_ref, operation_id, result)?; Ok(()) } @@ -5011,7 +5177,27 @@ impl Runtime { &self, worker_ref: &WorkerRef, operation_id: WorkerLifecycleOperationId, + mut result: WorkerExecutionResult, ) -> Result { + if !result.is_accepted() + || result.operation != WorkerExecutionOperation::Stop + || result.termination.as_ref().is_none_or(|termination| { + !termination.confirms_scope_shutdown(worker_ref, operation_id) + }) + { + if result.is_accepted() { + result.outcome = crate::execution::WorkerExecutionOutcome::Errored; + result.message = Some("stop has no matching actual scope termination evidence; reconciliation remains pending".into()); + } + self.record_execution_result(worker_ref, result.clone())?; + return Err(RuntimeError::WorkerExecutionRejected { + worker_id: worker_ref.worker_id, + operation: result.operation, + outcome: result.outcome, + message: result.message_or_default(), + result, + }); + } let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); if candidate @@ -9030,6 +9216,63 @@ mod tests { } } + #[test] + fn job_creation_lookup_checks_saved_identity_without_activating_or_resubmitting() { + let (runtime, backend) = runtime_and_backend(); + let mut request = scoped_task_request("saved allocation", "workspace-a"); + let binding = crate::catalog::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_digest: Some(format!("sha256:{}", "a".repeat(64))), + subjektiv_consolidation: false, + subjektiv_journal: false, + }; + request.backend_job = Some(binding.clone()); + let worker = runtime + .create_worker_scoped(&scope("workspace-a", "server-a"), request.clone()) + .unwrap(); + let bindings_before = backend.job_bindings.lock().unwrap().clone(); + let activations_before = backend.activation_operation_ids.lock().unwrap().clone(); + let query = |workspace: &str, + fingerprint: &str, + binding: &crate::catalog::BackendJobExecutionBinding| { + runtime.lookup_job_worker_creation(workspace, &worker.worker_ref, fingerprint, binding) + }; + assert!( + query("workspace-a", &request.create_fingerprint, &binding) + .unwrap() + .is_some() + ); + assert!(query("workspace-b", &request.create_fingerprint, &binding).is_err()); + assert!(query("workspace-a", "foreign-fingerprint", &binding).is_err()); + let mut foreign = binding.clone(); + foreign.attempt_id = "attempt-other".into(); + assert!(query("workspace-a", &request.create_fingerprint, &foreign).is_err()); + foreign = binding.clone(); + foreign.input_digest = Some("foreign-input".into()); + assert!(query("workspace-a", &request.create_fingerprint, &foreign).is_err()); + let mut absent = worker.worker_ref.clone(); + absent.worker_id = WorkerId::now_v7(); + assert!( + runtime + .lookup_job_worker_creation( + "workspace-a", + &absent, + &request.create_fingerprint, + &binding + ) + .unwrap() + .is_none() + ); + assert_eq!(*backend.job_bindings.lock().unwrap(), bindings_before); + assert_eq!( + *backend.activation_operation_ids.lock().unwrap(), + activations_before + ); + assert!(backend.dispatched_inputs.lock().unwrap().is_empty()); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + } + #[cfg(feature = "fs-store")] #[test] fn backend_job_binding_survives_restart_and_restore_without_profile_inference() { @@ -9565,10 +9808,18 @@ mod tests { before_spawn_result: Mutex>>, job_bindings: Mutex>>, dispatch_result: Mutex>, + submission_receipt_result: Mutex>, + submission_receipt_queries: Mutex>, stop_result: Mutex>, stop_gate: Mutex>>, stop_count: Mutex, stop_operation_ids: Mutex>, + stop_terminations: Mutex< + std::collections::HashMap< + WorkerLifecycleOperationId, + crate::execution::WorkerExecutionTermination, + >, + >, restore_result: Mutex>, preflight_restore_result: Mutex>, restore_gate: Mutex>>, @@ -10009,6 +10260,24 @@ mod tests { }] } + fn reconcile_submission( + &self, + _worker_ref: &WorkerRef, + input: WorkerInput, + ) -> WorkerExecutionResult { + self.submission_receipt_queries.lock().unwrap().push(input); + self.submission_receipt_result + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::Input, + "receipt missing; acceptance unknown", + ) + }) + } + fn dispatch_input( &self, worker_ref: &WorkerRef, @@ -10068,7 +10337,30 @@ mod tests { .lock() .unwrap() .push(request.operation_id); - self.stop_worker(&request.worker_ref) + let configured = self.stop_result.lock().unwrap().is_some(); + let mut result = self.stop_worker(&request.worker_ref); + if !configured && result.is_accepted() { + // This catalog fixture substitutes only the backend response; + // real Controller completion/join is tested in worker_backend. + let termination = self + .stop_terminations + .lock() + .unwrap() + .entry(request.operation_id) + .or_insert_with(|| crate::execution::WorkerExecutionTermination { + scope: crate::execution::WorkerExecutionTerminationScope::Controller, + worker_ref: request.worker_ref.clone(), + stop_operation_id: request.operation_id, + execution_id: uuid::Uuid::now_v7(), + shutdown_command_id: None, + submissions: Vec::new(), + feature_failures: Vec::new(), + }) + .clone(); + result.execution_id = Some(termination.execution_id); + result.termination = Some(termination); + } + result } fn stop_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { @@ -14796,6 +15088,214 @@ mod tests { assert_eq!(after.worker_state, Some(fresh)); } + #[test] + fn input_ack_mismatch_queries_exact_receipt_without_resubmitting_or_claiming_rejection() { + for found in [false, true] { + let (runtime, backend) = runtime_and_backend(); + let detail = runtime + .create_worker(task_request("receipt reconciliation")) + .unwrap(); + backend.preserve_submission_acknowledgement_id(); + backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( + WorkerExecutionOperation::Input, + "another-request", + "another-submission", + protocol::SubmissionDisposition::Started, + )); + let mut input = WorkerInput::user("same payload"); + input.submission_request_id = Some("exact-request".into()); + if found { + *backend.submission_receipt_result.lock().unwrap() = + Some(WorkerExecutionResult::accepted_submission( + WorkerExecutionOperation::Input, + "exact-request", + "retained-submission", + protocol::SubmissionDisposition::Started, + )); + } + let response = runtime.send_input(&detail.worker_ref, input); + if found { + assert_eq!( + response.unwrap().submission.unwrap().submission_id, + "retained-submission" + ); + } else { + assert!(matches!( + response, + Err(RuntimeError::WorkerExecutionRejected { + outcome: crate::execution::WorkerExecutionOutcome::Errored, + .. + }) + )); + } + let queries = backend.submission_receipt_queries.lock().unwrap(); + assert_eq!(queries.len(), 1); + assert_eq!( + queries[0].submission_request_id.as_deref(), + Some("exact-request") + ); + assert_eq!(queries[0].content, "same payload"); + assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + } + } + + #[test] + fn receipt_from_replacement_controller_does_not_repair_original_dispatch_ack() { + let (runtime, backend) = runtime_and_backend(); + let detail = runtime + .create_worker(task_request("receipt execution binding")) + .unwrap(); + let original_execution_id = uuid::Uuid::now_v7(); + let mut dispatched = WorkerExecutionResult::accepted(WorkerExecutionOperation::Input); + dispatched.execution_id = Some(original_execution_id); + backend.set_dispatch_result(dispatched); + let mut receipt = WorkerExecutionResult::accepted_submission( + WorkerExecutionOperation::Input, + "bound-request", + "retained-submission", + protocol::SubmissionDisposition::Started, + ); + receipt.execution_id = Some(uuid::Uuid::now_v7()); + *backend.submission_receipt_result.lock().unwrap() = Some(receipt); + let mut input = WorkerInput::user("same payload"); + input.submission_request_id = Some("bound-request".into()); + let RuntimeError::WorkerExecutionRejected { result, .. } = + runtime.send_input(&detail.worker_ref, input).unwrap_err() + else { + panic!("replacement Controller cannot repair original execution acknowledgement"); + }; + assert_eq!( + result.outcome, + crate::execution::WorkerExecutionOutcome::Errored + ); + assert_eq!(result.execution_id, Some(original_execution_id)); + assert!(result.submission.is_none()); + assert!(result.termination.is_none()); + assert_eq!(backend.submission_receipt_queries.lock().unwrap().len(), 1); + assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1); + } + + #[test] + fn reconciled_receipt_preserves_independent_dispatch_failure_facts() { + use protocol::startup_failure::*; + let (runtime, backend) = runtime_and_backend(); + let detail = runtime + .create_worker(task_request("receipt and Worker failure")) + .unwrap(); + let mut failure = WorkerExecutionResult::errored( + WorkerExecutionOperation::Input, + "post-admission Worker failure", + ) + .with_startup_failure( + worker::startup_failure::runtime_startup_attachment(RuntimeStartupCause { + stage: StartupStage::Feature, + reason: RuntimeStartupReason::FeatureFailed, + }) + .unwrap(), + ); + failure.execution_id = Some(uuid::Uuid::now_v7()); + failure + .secondary_failures + .push(StartupSecondaryFailure::Cleanup { + failure: StartupCleanupFailure::Feature, + }); + failure.startup_resources = Some(StartupResourceFacts { + resources: StartupResources::WorkerStartup, + cleanup: StartupCleanup::Failed(StartupCleanupFailure::Feature), + durable_effects_possible: true, + }); + failure.startup_failure_persistence = Some(StartupPersistenceOutcome::NotSaved); + backend.set_dispatch_result(failure.clone()); + let mut receipt = WorkerExecutionResult::accepted_submission( + WorkerExecutionOperation::Input, + "accepted-before-failure", + "retained-submission", + protocol::SubmissionDisposition::Started, + ); + receipt.execution_id = failure.execution_id; + failure.submission = receipt.submission.clone(); + *backend.submission_receipt_result.lock().unwrap() = Some(receipt); + let mut input = WorkerInput::user("same payload"); + input.submission_request_id = Some("accepted-before-failure".into()); + let RuntimeError::WorkerExecutionRejected { result, .. } = runtime + .send_input(&detail.worker_ref, input.clone()) + .unwrap_err() + else { + panic!("receipt must not erase a later Worker failure"); + }; + assert_eq!(result, failure); + assert_eq!( + backend + .submission_receipt_queries + .lock() + .unwrap() + .as_slice(), + &[input] + ); + assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + } + + #[test] + fn explicit_receipt_lookup_is_read_only_and_rejects_foreign_acknowledgements() { + let (runtime, backend) = runtime_and_backend(); + let detail = runtime + .create_worker(task_request("read-only receipt lookup")) + .unwrap(); + let mut input = WorkerInput::user("frozen input"); + input.submission_request_id = Some("exact-request".into()); + let before = runtime.worker_detail(&detail.worker_ref).unwrap(); + let receipts = [None, Some("exact-request"), Some("foreign-request")]; + for request_id in receipts { + *backend.submission_receipt_result.lock().unwrap() = request_id.map(|id| { + WorkerExecutionResult::accepted_submission( + WorkerExecutionOperation::Input, + id, + "retained-submission", + protocol::SubmissionDisposition::Started, + ) + }); + let result = runtime + .reconcile_input(&detail.worker_ref, input.clone()) + .unwrap(); + assert_eq!(result.is_accepted(), request_id == Some("exact-request")); + assert_eq!(runtime.worker_detail(&detail.worker_ref).unwrap(), before); + assert!(backend.dispatched_inputs.lock().unwrap().is_empty()); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + } + let queries = backend.submission_receipt_queries.lock().unwrap().len(); + input.submission_request_id = None; + assert!(runtime.reconcile_input(&detail.worker_ref, input).is_err()); + assert_eq!( + backend.submission_receipt_queries.lock().unwrap().len(), + queries + ); + } + + #[test] + fn accepted_stop_without_bound_termination_keeps_operation_pending() { + let (runtime, backend) = runtime_and_backend(); + let detail = runtime + .create_worker(task_request("stop evidence")) + .unwrap(); + backend.set_stop_result(WorkerExecutionResult::accepted( + WorkerExecutionOperation::Stop, + )); + let error = runtime.stop_worker(&detail.worker_ref, None).unwrap_err(); + assert!(matches!( + error, + RuntimeError::WorkerExecutionRejected { + outcome: crate::execution::WorkerExecutionOutcome::Errored, + .. + } + )); + let state = runtime.lock().unwrap(); + let worker = state.worker(&detail.worker_ref).unwrap(); + assert!(worker.pending_stop.is_some()); + assert_ne!(worker.status, WorkerStatus::Stopped); + } + #[test] fn create_worker_rejects_mismatched_submission_acknowledgement() { let (runtime, backend) = runtime_and_backend(); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 09b93f923..52b2bc5f1 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -67,6 +67,22 @@ enum NotificationAcceptance { Rejected(String), } +// Submit and read-only receipt reconciliation must use identical payload and +// authenticated namespace inputs. Neither the catalog nor a receipt cache owns them. +fn runtime_submission_payload( + input: &WorkerInput, + submission_request_id: &str, +) -> (Vec, protocol::AuthenticatedInputSource) { + let segments = input + .segments + .clone() + .unwrap_or_else(|| vec![Segment::text(input.content.trim().to_string())]); + let source = protocol::AuthenticatedInputSource::Backend { + operation_id: submission_request_id.to_string(), + }; + (segments, source) +} + fn rollback_materialized_spawn_workdirs( materializer: &dyn WorkingDirectoryMaterializer, bindings: &BTreeMap, @@ -570,6 +586,13 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { Err(session_store::RetainedAttachmentReadError::RetentionMissing) } + async fn cleanup_failed_spawn( + &self, + _worker_ref: &WorkerRef, + ) -> Result<(), RuntimeRestoreCleanupFailure> { + Err(RuntimeRestoreCleanupFailure::Unsupported) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -931,10 +954,19 @@ pub struct ProfileRuntimeWorkerFactory { workspace_request_clients: Arc>, embedded_worker_mutation_dispatcher: Option>, failed_restore_resources: Arc>>, + failed_spawn_resources: Arc>>, #[cfg(test)] restore_faults: Arc, } +#[derive(Clone)] +struct FailedSpawnResources { + run_dir: PathBuf, + sessions: Vec>, + allocation: Option, + feature_tasks: Option, +} + #[derive(Clone)] struct FailedRestoreResources { operation_id: crate::execution::WorkerLifecycleOperationId, @@ -996,6 +1028,41 @@ impl RestoreActivationGate { } impl ProfileRuntimeWorkerFactory { + fn retain_spawn_construction_failure(&self, worker_ref: &WorkerRef, error: &WorkerError) { + match error { + WorkerError::StartupFailure { source, .. } + | WorkerError::RestoreActivationEffects { source } => { + self.retain_spawn_construction_failure(worker_ref, source) + } + WorkerError::ConstructionCleanup { + allocation, + sessions, + .. + } => { + if let Some(owner) = self + .failed_spawn_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(worker_ref) + { + owner.allocation = allocation.clone(); + owner.sessions.push(sessions.clone()); + } + } + WorkerError::ScopeLock(error) => { + if let Some(owner) = self + .failed_spawn_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(worker_ref) + { + owner.allocation = error.allocation_cleanup_handle(); + } + } + _ => {} + } + } + fn retain_construction_failure( &self, request: &WorkerExecutionRestoreRequest, @@ -1058,6 +1125,7 @@ impl ProfileRuntimeWorkerFactory { workspace_request_clients: Arc::new(HashMap::new()), embedded_worker_mutation_dispatcher: None, failed_restore_resources: Arc::new(Mutex::new(HashMap::new())), + failed_spawn_resources: Arc::new(Mutex::new(HashMap::new())), #[cfg(test)] restore_faults: Arc::new(RestoreFaults::default()), } @@ -2101,6 +2169,43 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { self.prompt_projection_cache.observe(projection).map(|_| ()) } + async fn cleanup_failed_spawn( + &self, + worker_ref: &WorkerRef, + ) -> Result<(), RuntimeRestoreCleanupFailure> { + let owner = self + .failed_spawn_resources + .lock() + .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? + .get(worker_ref) + .cloned() + .ok_or(RuntimeRestoreCleanupFailure::Ownership)?; + if let Some(features) = owner.feature_tasks.as_ref() { + features + .shutdown() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Feature)?; + } + for sessions in &owner.sessions { + sessions + .close_all() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } + if let Some(allocation) = owner.allocation.as_ref() { + allocation + .release() + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + } + Self::remove_restore_run_artifacts(&owner.run_dir) + .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; + self.failed_spawn_resources + .lock() + .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? + .remove(worker_ref); + Ok(()) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -2224,10 +2329,46 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { Some(factory) => bootstrap.with_model_client(factory()), None => bootstrap, }; + { + let mut resources = self + .failed_spawn_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if resources.contains_key(&request.worker_ref) { + return Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Reconstruction, + "fresh startup resources already retained", + )); + } + resources.insert( + request.worker_ref.clone(), + FailedSpawnResources { + run_dir: run_dir.clone(), + sessions: Vec::new(), + allocation: None, + feature_tasks: None, + }, + ); + } let mut prepared = bootstrap.prepare().await.map_err(|error| { + if let WorkerBootstrapError::Worker(source) = &error { + self.retain_spawn_construction_failure(&request.worker_ref, source); + } RuntimeRestoreFailure::from_bootstrap(RuntimeRestoreStage::Reconstruction, &error) })?; let worker = prepared.worker_mut(); + { + let mut resources = self + .failed_spawn_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let owner = resources + .get_mut(&request.worker_ref) + .expect("fresh startup owner"); + owner.allocation = worker.restore_allocation_cleanup_handle(); + owner.feature_tasks = Some(worker.retain_startup_feature_cleanup()); + owner.sessions.push(worker.workdir_sessions()); + } if let Some(snapshot) = &request.request.plugin_execution { if self.runtime_id.as_deref() != Some(snapshot.plan.execution().runtime_id.as_str()) { return Err("Plugin plan Runtime identity mismatch".into()); @@ -2265,11 +2406,19 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { ) }) .collect::>(); - worker.bind_workdir_sessions(runtime_local_workdir_router( + let sessions = runtime_local_workdir_router( &request.workdir_attachments, &workdir_capabilities, Some(worker.workspace_client_handle()), - )?); + )?; + self.failed_spawn_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(&request.worker_ref) + .expect("fresh startup owner") + .sessions + .push(sessions.clone()); + worker.bind_workdir_sessions(sessions); if let (Some(runtime_id), Some(workspace_id)) = (observation_runtime_id, observation_workspace_id.clone()) && observation_enabled @@ -2299,8 +2448,15 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { let workspace_client = worker.workspace_client_handle(); let started = prepared.start().await.map_err(|error| { + if let WorkerBootstrapError::Worker(source) = &error { + self.retain_spawn_construction_failure(&request.worker_ref, source); + } RuntimeRestoreFailure::from_bootstrap(RuntimeRestoreStage::Controller, &error) })?; + self.failed_spawn_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&request.worker_ref); let (handle, shutdown_rx, controller_task) = (started.handle, started.shutdown, started.controller_task); if flow_transition_enabled { @@ -3131,6 +3287,9 @@ impl ProfileRuntimeWorkerFactory { #[derive(Clone)] struct RuntimeExecutionTaskScope { + execution_id: uuid::Uuid, + shutdown_report: Arc>>>, + submissions: Arc>>, #[cfg(feature = "ws-server")] protocol_bridge: Arc>>, tasks: Arc>>>, @@ -3230,6 +3389,9 @@ struct RuntimeExecutionTaskCompletion { impl RuntimeExecutionTaskScope { fn new(controller_task: tokio::task::JoinHandle<()>) -> Self { let scope = Self { + execution_id: uuid::Uuid::now_v7(), + shutdown_report: Arc::default(), + submissions: Arc::default(), #[cfg(feature = "ws-server")] protocol_bridge: Arc::default(), tasks: Arc::new(Mutex::new(Vec::new())), @@ -3352,7 +3514,19 @@ impl RuntimeExecutionTaskScope { let mut guard = shutdown.lock().await; if let Some(receiver) = guard.as_mut() { match tokio::time::timeout(Duration::from_secs(5), receiver).await { - Ok(Ok(())) => { + Ok(Ok(report)) => { + *self.shutdown_report.lock().map_err(|_| { + "Controller shutdown report lock is poisoned".to_string() + })? = Some( + report + .feature_failures + .into_iter() + .map(|failure| crate::execution::WorkerFeatureShutdownFailure { + category: failure.category, + diagnostic: failure.diagnostic, + }) + .collect(), + ); guard.take(); Ok(()) } @@ -3376,7 +3550,49 @@ impl RuntimeExecutionTaskScope { // Drain bridges/relays even after an abnormal completion, while keeping // the failed completion and all timed-out handles in their owning scope. self.join().await?; - confirmation + confirmation?; + match self + .shutdown_report + .lock() + .map_err(|_| "Controller shutdown report lock is poisoned".to_string())? + .as_ref() + { + Some(failures) if failures.is_empty() => Ok(()), + Some(_) => Err( + "Controller joined with Feature shutdown failures; cleanup is insufficient".into(), + ), + None => { + Err("Controller shutdown report is missing; scope termination is unproven".into()) + } + } + } + + fn termination( + &self, + worker_ref: &WorkerRef, + operation_id: crate::execution::WorkerLifecycleOperationId, + ) -> Option { + // Only the original receiver and actual joined task handles establish + // this fact. Closed endpoints, empty backend maps and causes do not. + if !self.tasks.lock().ok()?.is_empty() || self.has_cleanup_candidates().ok()? { + return None; + } + let feature_failures = self.shutdown_report.lock().ok()?.clone()?; + let shutdown_command_id = self + .shutdown_admission + .try_lock() + .ok()? + .as_ref() + .map(|admission| admission.command_id); + Some(crate::execution::WorkerExecutionTermination { + scope: crate::execution::WorkerExecutionTerminationScope::Controller, + worker_ref: worker_ref.clone(), + stop_operation_id: operation_id, + execution_id: self.execution_id, + shutdown_command_id, + submissions: self.submissions.lock().ok()?.clone(), + feature_failures, + }) } async fn join(&self) -> Result<(), String> { @@ -3453,9 +3669,21 @@ struct RuntimeWorkerExecution { workdir_attachments: Vec, } +// The fresh factory future and its late result stay with the admitting backend. +// A bounded caller wait never cancels construction or drops a late Controller. +struct PendingRuntimeSpawn { + execution_id: uuid::Uuid, + task: tokio::sync::Mutex>>, + result: Mutex>>, + workdir_attachments: BTreeMap, + rollback_workdirs: Vec, + failure: Mutex>, +} + // Actual operation-owned resources, not a second Worker catalog or presence // marker. Timeout never drops the factory future or a late Controller result. struct PendingRuntimeRestore { + execution_id: uuid::Uuid, request: WorkerExecutionRestoreRequest, // Transient ownership mode, not a durable phase. Cold restart reclamation // must never enter preparation, canonical repair, or activation. @@ -3486,6 +3714,12 @@ impl Drop for RestoreTaskCompletion { } impl PendingRuntimeRestore { + fn stop_failure_result(&self, message: String) -> WorkerExecutionResult { + let mut result = self.failure_result(message); + result.operation = WorkerExecutionOperation::Stop; + result + } + fn task_completion(self: &Arc) -> RestoreTaskCompletion { self.task_in_flight.store(true, Ordering::Release); RestoreTaskCompletion(self.clone()) @@ -3564,6 +3798,7 @@ pub struct WorkerRuntimeExecutionBackend { working_directory_materializer: Option>, runtime: Mutex>, workers: Mutex>, + pending_spawns: Mutex>>, pending_restores: Mutex>>, // One completed execution receipt per Worker, separate from live resources. // Runtime owns durable request replay; retain this until the next admitted @@ -3577,6 +3812,7 @@ pub struct WorkerRuntimeExecutionBackend { ), >, >, + completed_stops: Mutex>, worker_locks: Mutex>>>, admissions: Mutex>, spawn_restore_timeout: Duration, @@ -3602,8 +3838,10 @@ where working_directory_materializer: None, runtime: Mutex::new(Some(runtime)), workers: Mutex::new(HashMap::new()), + pending_spawns: Mutex::new(HashMap::new()), pending_restores: Mutex::new(HashMap::new()), completed_restore_failures: Mutex::new(HashMap::new()), + completed_stops: Mutex::new(HashMap::new()), worker_locks: Mutex::new(HashMap::new()), admissions: Mutex::new(HashMap::new()), spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT, @@ -3619,6 +3857,17 @@ where workspace: Option<&str>, fingerprint: &str, ) -> Result<(), String> { + if self + .pending_spawns + .lock() + .map_err(|_| "fresh spawn owner lock is poisoned")? + .contains_key(worker) + { + return Err( + "fresh spawn still owns pending resources; Stop must reclaim its original owner" + .into(), + ); + } let mut admissions = self .admissions .lock() @@ -3817,6 +4066,150 @@ where .map_err(|err| format!("worker adapter task did not complete: {err}"))? } + fn pending_spawn_exists(&self, worker_ref: &WorkerRef) -> Result { + self.pending_spawns + .lock() + .map(|owners| owners.contains_key(worker_ref)) + .map_err(|_| "fresh spawn owner lock is poisoned".to_string()) + } + + fn wait_pending_spawn( + &self, + pending: &Arc, + timeout: Duration, + ) -> Result<(), String> { + let owner = pending.clone(); + self.run_joined_on_adapter_runtime(async move { + let mut task = owner.task.lock().await; + if let Some(task_handle) = task.as_mut() { + match tokio::time::timeout(timeout, task_handle).await { + Ok(Ok(())) => { + task.take(); + } + Ok(Err(error)) => { + let message = format!( + "fresh spawn task failed: {error}; resource cleanup is unproven" + ); + *owner + .failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(message.clone()); + task.take(); + return Err(message); + } + Err(_) => return Err( + "fresh spawn is still owned and pending; late Controller retained for Stop" + .into(), + ), + } + } + if let Some(failure) = owner + .failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + { + return Err(failure.clone()); + } + Ok(()) + }) + } + + fn spawn_controller_retained( + &self, + request: WorkerExecutionSpawnRequest, + rollback_workdirs: Vec, + ) -> Result, WorkerExecutionSpawnResult> + { + let worker_ref = request.worker_ref.clone(); + let owner = Arc::new(PendingRuntimeSpawn { + execution_id: uuid::Uuid::now_v7(), + task: tokio::sync::Mutex::new(None), + result: Mutex::new(None), + workdir_attachments: request.workdir_attachments.clone(), + rollback_workdirs, + failure: Mutex::new(None), + }); + let uncertain = |message| WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Spawn, message), + worker_state: None, + workdir_attachments: workdir_attachment_statuses(&owner.workdir_attachments), + }; + let mut owners = self + .pending_spawns + .lock() + .map_err(|_| uncertain("fresh spawn owner lock is poisoned".to_string()))?; + if owners.contains_key(&worker_ref) { + return Err(uncertain( + "fresh spawn already owns resources; reconcile with Stop".into(), + )); + } + self.completed_stops + .lock() + .map_err(|_| uncertain("Worker termination receipt registry poisoned".into()))? + .remove(&worker_ref); + owners.insert(worker_ref.clone(), owner.clone()); + drop(owners); + let factory = self.factory.clone(); + let pending = owner.clone(); + let task = self + .spawn_on_adapter_runtime(async move { + let worker_ref = request.worker_ref.clone(); + let mut result = factory.spawn_controller(request).await; + if let Err(failure) = &mut result + && failure.needs_cleanup() + { + failure.cleanup = match factory.cleanup_failed_spawn(&worker_ref).await { + Ok(()) => RuntimeRestoreCleanup::Confirmed, + Err(RuntimeRestoreCleanupFailure::Unsupported) => failure.cleanup, + Err(error) => { + failure.cleanup_failure.get_or_insert(error); + RuntimeRestoreCleanup::Failed(error) + } + }; + } + *pending + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); + }) + .map_err(|message| { + *owner + .failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(message.clone()); + uncertain(message) + })?; + *owner + .task + .try_lock() + .expect("fresh spawn task not yet awaited") = Some(task); + self.wait_pending_spawn(&owner, self.spawn_restore_timeout) + .map_err(uncertain)?; + let mut result = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(Err(failure)) = result.as_ref() + && failure.needs_cleanup() + { + return Err(WorkerExecutionSpawnResult::ReconciliationRequired { + result: failure.execution_result(WorkerExecutionOperation::Spawn), + worker_state: None, + workdir_attachments: workdir_attachment_statuses(&owner.workdir_attachments), + }); + } + let completed = result + .take() + .ok_or_else(|| uncertain("fresh spawn completion is missing".into()))?; + self.pending_spawns + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&worker_ref); + Ok(completed) + } + fn preflight_restore_inputs( &self, request: &WorkerExecutionRestoreRequest, @@ -4103,7 +4496,7 @@ where Method::Shutdown { .. } => { // Identity was checked against the captured Controller while // this same operation lock is held. Never unlock/rebind here. - let result = self.stop_worker_under_lock(worker_ref); + let result = self.stop_worker_under_lock(worker_ref, None); if result.is_accepted() { Ok(Vec::new()) } else { @@ -4718,8 +5111,202 @@ where } } + fn cleanup_pending_spawn_bindings( + &self, + worker_ref: &WorkerRef, + pending: &PendingRuntimeSpawn, + ) -> Result<(), String> { + if !pending.rollback_workdirs.is_empty() { + let materializer = self + .working_directory_materializer + .as_ref() + .ok_or("fresh spawn Workdir cleanup owner unavailable")?; + for id in pending.rollback_workdirs.iter().rev() { + materializer + .cleanup_working_directory(id) + .map_err(|error| error.to_string())?; + } + } + self.pending_spawns + .lock() + .map_err(|_| "fresh spawn owner lock is poisoned")? + .remove(worker_ref); + Ok(()) + } + + fn stop_worker_operation_under_lock( + &self, + request: WorkerExecutionStopRequest, + ) -> WorkerExecutionResult { + let execution = match self.workers.lock() { + Ok(workers) => workers.get(&request.worker_ref).cloned(), + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "worker adapter registry lock is poisoned", + ); + } + }; + { + let receipts = match self.completed_stops.lock() { + Ok(receipts) => receipts, + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "stop completion receipts lock is poisoned", + ); + } + }; + if let Some(receipt) = receipts.get(&request.worker_ref) + && receipt.confirms_scope_shutdown(&request.worker_ref, request.operation_id) + { + if execution.is_some() + || self + .pending_spawn_exists(&request.worker_ref) + .unwrap_or(true) + || !matches!(self.pending_restore(&request.worker_ref), Ok(None)) + { + return WorkerExecutionResult::rejected( + WorkerExecutionOperation::Stop, + "historical stop operation cannot target a replacement execution or candidate", + ); + } + let mut result = WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop); + result.execution_id = Some(receipt.execution_id); + result.termination = Some(receipt.clone()); + return result; + } + } + let mut result = + self.stop_worker_under_lock(&request.worker_ref, Some(request.operation_id)); + if result.termination.is_none() + && let Some(execution) = execution + { + result.termination = execution + .tasks + .termination(&request.worker_ref, request.operation_id); + } + result.execution_id = result + .termination + .as_ref() + .map(|termination| termination.execution_id); + if result.is_accepted() { + match result.termination.as_ref() { + Some(termination) + if termination + .confirms_scope_shutdown(&request.worker_ref, request.operation_id) => + { + self.completed_stops + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(request.worker_ref, termination.clone()); + } + _ => { + result.outcome = crate::execution::WorkerExecutionOutcome::Errored; + result.message = + Some("actual scope termination is unproven; stop remains pending".into()); + } + } + } + result + } + // Caller owns the per-Worker operation lock for this entire cleanup. - fn stop_worker_under_lock(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + fn stop_worker_under_lock( + &self, + worker_ref: &WorkerRef, + operation_id: Option, + ) -> WorkerExecutionResult { + let mut candidate_termination = None; + let mut startup_stop_result = None; + let pending_spawn = match self.pending_spawns.lock() { + Ok(owners) => owners.get(worker_ref).cloned(), + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "fresh spawn owner lock is poisoned", + ); + } + }; + if let Some(pending) = pending_spawn.as_ref() { + if let Err(message) = self.wait_pending_spawn(pending, Duration::from_secs(5)) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + let failure = pending + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .and_then(|result| result.as_ref().err()) + .filter(|failure| failure.needs_cleanup()) + .cloned(); + if let Some(mut failure) = failure { + let factory = self.factory.clone(); + let worker_ref = worker_ref.clone(); + let owner = pending.clone(); + if let Err(message) = self.run_joined_on_adapter_runtime(async move { + failure.cleanup = match factory.cleanup_failed_spawn(&worker_ref).await { + Ok(()) => RuntimeRestoreCleanup::Confirmed, + Err(RuntimeRestoreCleanupFailure::Unsupported) => failure.cleanup, + Err(error) => { + failure.cleanup_failure.get_or_insert(error); + RuntimeRestoreCleanup::Failed(error) + } + }; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Err(failure)); + Ok(()) + }) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + } + let mut result = pending + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match result.as_ref() { + Some(Ok(_)) => { + let controller = result.take().unwrap().unwrap_or_else(|_| unreachable!()); + let state = Arc::new(RwLock::new(controller.handle.shared_state.snapshot())); + self.retain_uncertain_unconnected_controller( + worker_ref, + controller.handle, + controller.shutdown, + Arc::new(AtomicBool::new(false)), + RuntimeExecutionTaskScope::new(controller.controller_task), + state, + Some(controller.workspace_client), + None, + workdir_attachment_statuses(&pending.workdir_attachments), + ); + } + Some(Err(failure)) if failure.needs_cleanup() => { + return failure.execution_result(WorkerExecutionOperation::Stop); + } + Some(Err(failure)) => { + startup_stop_result = + Some(failure.execution_result(WorkerExecutionOperation::Stop)); + if failure.cleanup == RuntimeRestoreCleanup::Confirmed + && let Some(operation_id) = operation_id + { + // The original factory task is joined and its resource + // owner explicitly completed cleanup. Never derive this + // fact from a projected startup attachment. + candidate_termination = Some( + crate::execution::WorkerExecutionTermination::joined_startup_candidate( + worker_ref.clone(), + operation_id, + pending.execution_id, + ), + ); + } + } + // A previous Stop transferred the Controller to its execution scope. + None => {} + } + } let pending = match self.pending_restore(worker_ref) { Ok(pending) => pending, Err(message) => { @@ -4728,13 +5315,13 @@ where }; if let Some(pending) = pending { if let Err(message) = self.wait_pending_restore(&pending, Duration::from_secs(5)) { - return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + return pending.stop_failure_result(message); } if let Err(message) = self .retry_pending_restore_cleanup(&pending, false) .and_then(|()| self.wait_pending_restore(&pending, Duration::from_secs(5))) { - return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + return pending.stop_failure_result(message); } if let Some(Err(failure)) = pending .result @@ -4743,10 +5330,7 @@ where .as_ref() && failure.needs_cleanup() { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - failure.to_string(), - ); + return failure.execution_result(WorkerExecutionOperation::Stop); } if pending.activation_started.load(Ordering::Acquire) && !pending.owns_prepared_candidate() @@ -4784,10 +5368,7 @@ where RuntimeRestoreCleanup::Confirmed | RuntimeRestoreCleanup::NotNeeded ) => {} Some(Err(failure)) => { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - failure.to_string(), - ); + return failure.execution_result(WorkerExecutionOperation::Stop); } None => { return WorkerExecutionResult::errored( @@ -4852,7 +5433,7 @@ where *pending.task.try_lock().expect("joined candidate task") = Some(task); } if let Err(message) = self.wait_pending_restore(&pending, Duration::from_secs(5)) { - return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + return pending.stop_failure_result(message); } let cleanup = *pending .discard_result @@ -4862,10 +5443,7 @@ where Some(Ok(())) => {} Some(Err(error)) => { pending.discard_started.store(false, Ordering::Release); - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - error.to_string(), - ); + return pending.stop_failure_result(error.to_string()); } None => { return WorkerExecutionResult::errored( @@ -4875,8 +5453,52 @@ where } } } - if let Err(message) = self.remove_pending_restore(worker_ref) { - return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + let discarded = *pending + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + == Some(Ok(())); + let preparation = pending + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let prepared_cleanup = discarded + && match preparation.as_ref() { + None => true, // The actual candidate's discard completed and joined. + Some(Err(failure)) => failure.cleanup == RuntimeRestoreCleanup::Confirmed, + Some(Ok(_)) => false, + }; + if let Some(Err(failure)) = preparation.as_ref() { + startup_stop_result = + Some(failure.execution_result(WorkerExecutionOperation::Stop)); + } + drop(preparation); + let activation = pending + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let activated_cleanup = matches!(activation.as_ref(), Some(Err(failure)) if failure.cleanup == RuntimeRestoreCleanup::Confirmed); + if let Some(Err(failure)) = activation.as_ref() { + startup_stop_result = + Some(failure.execution_result(WorkerExecutionOperation::Stop)); + } + drop(activation); + if !pending.cleanup_only + && (prepared_cleanup || activated_cleanup) + && let Some(operation_id) = operation_id + { + candidate_termination = Some( + crate::execution::WorkerExecutionTermination::joined_startup_candidate( + worker_ref.clone(), + operation_id, + pending.execution_id, + ), + ); + } + if candidate_termination.is_none() { + if let Err(message) = self.remove_pending_restore(worker_ref) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } } } let execution = match self.workers.lock() { @@ -4889,23 +5511,58 @@ where } }; let Some(execution) = execution else { - // The execution backend cleanup may have committed before the - // Runtime catalog commit failed. Treat the retry as converged so - // the Runtime can durably finish its Stopped transition. + let mut result = startup_stop_result.unwrap_or_else(|| WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "no retained actual execution termination evidence; map absence is not shutdown proof", + )); + // Reconciliation alone cannot issue evidence: the original candidate + // task must already be joined and its resource owner cleanup confirmed. let factory = Arc::clone(&self.factory); - let worker_ref = worker_ref.clone(); - return match self.run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move { - factory.reconcile_stopped_worker(&worker_ref).await - }) { - Ok(()) => WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop), - Err(message) => { - WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message) + let stopped_ref = worker_ref.clone(); + if let Err(message) = self + .run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move { + factory.reconcile_stopped_worker(&stopped_ref).await + }) + { + result.message = Some(message); + return result; + } + if let Some(termination) = candidate_termination { + if let Some(pending) = pending_spawn.as_ref() { + if let Err(message) = self.cleanup_pending_spawn_bindings(worker_ref, pending) { + result.message = Some(message); + return result; + } } - }; + if let Err(message) = self.remove_pending_restore(worker_ref) { + result.message = Some(message); + return result; + } + result.outcome = crate::execution::WorkerExecutionOutcome::Accepted; + result.termination = Some(termination); + } + return result; }; if let Err(message) = self.cleanup_execution_resources(&execution) { - return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + let mut result = startup_stop_result.clone().unwrap_or_else(|| { + WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message.clone()) + }); + result.outcome = crate::execution::WorkerExecutionOutcome::Errored; + result.message = Some(message); + result.termination = operation_id + .and_then(|operation_id| execution.tasks.termination(worker_ref, operation_id)); + return result; + } + if let Some(pending) = pending_spawn { + if let Err(message) = self.cleanup_pending_spawn_bindings(worker_ref, &pending) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + } + if candidate_termination.is_some() { + if let Err(message) = self.remove_pending_restore(worker_ref) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } } // External consumers may retain a closed handle indefinitely. Weak // shared-state liveness is not evidence of a published execution. @@ -4927,7 +5584,12 @@ where ); } } - WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) + let mut result = startup_stop_result + .unwrap_or_else(|| WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)); + result.outcome = crate::execution::WorkerExecutionOutcome::Accepted; + result.termination = operation_id + .and_then(|operation_id| execution.tasks.termination(worker_ref, operation_id)); + result } } @@ -4995,7 +5657,7 @@ where deletion: Some(request.clone()), }, ); - let result = self.stop_worker_under_lock(&worker); + let result = self.stop_worker_under_lock(&worker, None); if !result.is_accepted() { return Err(result.message_or_default()); } @@ -5033,6 +5695,9 @@ where { return Err("execution backend is unavailable".into()); } + if self.pending_spawn_exists(&worker_ref)? { + return Err("Subject create fresh spawn still owns resources".into()); + } if self .pending_restores .lock() @@ -5513,13 +6178,9 @@ where && request.request.workdir_attachments.is_empty() { // No provider is needed for a Workdir-less Worker. - let factory = self.factory.clone(); let bridge_context = request.context.clone(); let worker_ref = request.worker_ref.clone(); - let spawn_result = self - .run_cancellable_on_adapter_runtime(self.spawn_restore_timeout, async move { - Ok(factory.spawn_controller(request).await) - }); + let spawn_result = self.spawn_controller_retained(request, Vec::new()); let controller = match spawn_result { Ok(Ok(controller)) => controller, Ok(Err(failure)) => { @@ -5527,14 +6188,7 @@ where failure.execution_result(WorkerExecutionOperation::Spawn), ); } - Err(message) => { - return WorkerExecutionSpawnResult::Errored( - WorkerExecutionResult::errored( - WorkerExecutionOperation::Spawn, - message, - ), - ); - } + Err(result) => return result, }; return self.connect_handle( WorkerExecutionOperation::Spawn, @@ -5651,13 +6305,9 @@ where } let workdir_attachments = request.workdir_attachments.clone(); - let factory = self.factory.clone(); let bridge_context = request.context.clone(); let worker_ref = request.worker_ref.clone(); - let spawn_result = self - .run_cancellable_on_adapter_runtime(self.spawn_restore_timeout, async move { - Ok(factory.spawn_controller(request).await) - }); + let spawn_result = self.spawn_controller_retained(request, rollback_workdirs.clone()); let controller = match spawn_result { Ok(Ok(controller)) => controller, @@ -5673,23 +6323,8 @@ where } return WorkerExecutionSpawnResult::Errored(result); } - Err(message) => { - if let Err(result) = rollback_materialized_spawn_workdirs( - materializer.as_ref(), - &workdir_attachments, - &rollback_workdirs, - &WorkerExecutionResult::rejected( - WorkerExecutionOperation::Spawn, - message.clone(), - ), - ) { - return result; - } - return WorkerExecutionSpawnResult::Errored(WorkerExecutionResult::errored( - WorkerExecutionOperation::Spawn, - message, - )); - } + // The retained owner, not the timed-out caller, owns these bindings. + Err(result) => return result, }; self.connect_handle( @@ -5989,6 +6624,7 @@ where .unwrap_or_else(std::sync::PoisonError::into_inner) .remove(&worker_ref); let pending = Arc::new(PendingRuntimeRestore { + execution_id: uuid::Uuid::now_v7(), request: request.clone(), cleanup_only: false, task: tokio::sync::Mutex::new(None), @@ -6011,6 +6647,19 @@ where ); } }; + match self.completed_stops.lock() { + Ok(mut receipts) => { + receipts.remove(&worker_ref); + } + Err(_) => { + return WorkerExecutionSpawnResult::Rejected( + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "Worker termination receipt registry poisoned", + ), + ); + } + } resources.insert(worker_ref.clone(), Arc::clone(&pending)); let factory = Arc::clone(&self.factory); let result_owner = Arc::clone(&pending); @@ -6313,6 +6962,7 @@ where ); failure.durable_effects_possible = request.source_repair_possible; let pending = Arc::new(PendingRuntimeRestore { + execution_id: uuid::Uuid::now_v7(), request: request.clone(), cleanup_only: true, task: tokio::sync::Mutex::new(None), @@ -6666,6 +7316,139 @@ where Ok(()) } + fn termination_receipt( + &self, + worker_ref: &WorkerRef, + ) -> Result, String> { + let operation = self.worker_lock(worker_ref)?; + let _guard = operation + .lock() + .map_err(|_| "Worker operation lock poisoned".to_string())?; + if self + .pending_spawns + .lock() + .map_err(|_| "Worker spawn registry poisoned".to_string())? + .contains_key(worker_ref) + || self + .pending_restores + .lock() + .map_err(|_| "Worker restore registry poisoned".to_string())? + .contains_key(worker_ref) + || self + .subject_reservations + .lock() + .map_err(|_| "Subject reservation registry poisoned".to_string())? + .keys() + .any(|(worker, _)| worker == worker_ref) + { + return Ok(None); + } + let current_execution = self + .workers + .lock() + .map_err(|_| "Worker execution registry poisoned".to_string())? + .get(worker_ref) + .map(|execution| execution.tasks.execution_id); + let receipt = self + .completed_stops + .lock() + .map_err(|_| "Worker termination receipts poisoned".to_string())? + .get(worker_ref) + .cloned(); + // The positive receipt comes only from the joined scope, never from + // absence. A Restore's new Controller cannot reuse its predecessor's end. + Ok(receipt.filter(|receipt| current_execution.is_none_or(|id| id == receipt.execution_id))) + } + + fn reconcile_submission( + &self, + worker_ref: &WorkerRef, + input: WorkerInput, + ) -> WorkerExecutionResult { + let unknown = + |message| WorkerExecutionResult::errored(WorkerExecutionOperation::Input, message); + if !matches!( + input.kind, + WorkerInputKind::User | WorkerInputKind::UserIfIdle + ) { + return unknown( + "receipt reconciliation requires a Submit input; acceptance remains unknown", + ); + } + let Some(request_id) = input + .submission_request_id + .as_deref() + .filter(|id| !id.trim().is_empty()) + else { + return unknown( + "receipt reconciliation has no submission request id; acceptance remains unknown", + ); + }; + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(_) => { + return unknown( + "Worker operation lock unavailable; submission acceptance remains unknown", + ); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + return unknown( + "Worker operation lock is poisoned; submission acceptance remains unknown", + ); + } + }; + let execution = match self.workers.lock() { + Ok(workers) => match workers.get(worker_ref) { + Some(execution) => execution.clone(), + None => { + return unknown( + "no retained Controller for receipt lookup; submission acceptance remains unknown", + ); + } + }, + Err(_) => { + return unknown( + "Worker registry is unavailable; submission acceptance remains unknown", + ); + } + }; + let (segments, source) = runtime_submission_payload(&input, request_id); + // This reads the Worker's existing namespace+digest receipt. It must not + // dispatch a Method, reopen admission, or infer nonacceptance from None. + let mut result = match execution + .handle + .reconcile_submission(request_id, &segments, &source) + { + Ok(Some(receipt)) + if receipt.submission_request_id == request_id + && !receipt.submission_id.trim().is_empty() => + { + WorkerExecutionResult::accepted_submission( + WorkerExecutionOperation::Input, + receipt.submission_request_id, + receipt.submission_id, + receipt.disposition, + ) + } + Ok(Some(_)) => unknown( + "Worker returned a mismatched receipt; submission acceptance remains unknown", + ), + Ok(None) => unknown( + "exact submission receipt is missing; submission acceptance remains unknown", + ), + Err(_) => unknown( + "exact submission receipt lookup failed; submission acceptance remains unknown", + ), + }; + // Identify the Controller observed, not the originating run of a receipt + // retained across Controller replacement. This is not termination evidence. + result.execution_id = Some(execution.tasks.execution_id); + result + } + fn dispatch_input(&self, worker_ref: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult { let operation_lock = match self.worker_lock(worker_ref) { Ok(lock) => lock, @@ -6733,19 +7516,16 @@ where WorkerInputKind::User | WorkerInputKind::UserIfIdle => { let Some(submission_id) = input .submission_request_id + .as_ref() .filter(|submission_id| !submission_id.trim().is_empty()) + .cloned() else { return WorkerExecutionResult::rejected( WorkerExecutionOperation::Input, "Runtime user input is missing its internal submission id", ); }; - let segments = input - .segments - .unwrap_or_else(|| vec![Segment::text(input.content.trim().to_string())]); - let source = protocol::AuthenticatedInputSource::Backend { - operation_id: submission_id.clone(), - }; + let (segments, source) = runtime_submission_payload(&input, &submission_id); let method = if input.kind == WorkerInputKind::UserIfIdle { Method::SubmitIfIdle { submission_request_id: submission_id.clone(), @@ -6774,16 +7554,40 @@ where }; let waits_for_submission_acceptance = submission_request_id.is_some(); - if waits_for_submission_acceptance { + let mut result = if waits_for_submission_acceptance { self.send_submit_and_wait_for_acceptance( WorkerExecutionOperation::Input, worker, method, - submission_request_id.expect("Submit must have a submission request id"), + submission_request_id + .clone() + .expect("Submit must have a submission request id"), ) } else { self.send_method(WorkerExecutionOperation::Input, worker, method) + }; + if let Some(execution) = self + .workers + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(worker_ref) + { + result.execution_id = Some(execution.tasks.execution_id); + if result.is_accepted() + && let Some(ack) = result.submission.as_ref() + && Some(&ack.submission_request_id) == submission_request_id.as_ref() + { + let mut submissions = execution + .tasks + .submissions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !submissions.contains(ack) { + submissions.push(ack.clone()); + } + } } + result } fn upload_file( @@ -6903,7 +7707,22 @@ where } fn stop_worker_operation(&self, request: WorkerExecutionStopRequest) -> WorkerExecutionResult { - self.stop_worker(&request.worker_ref) + let lock = match self.worker_lock(&request.worker_ref) { + Ok(lock) => lock, + Err(message) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + }; + let _guard = match lock.lock() { + Ok(guard) => guard, + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "worker operation lock is poisoned", + ); + } + }; + self.stop_worker_operation_under_lock(request) } fn stop_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { @@ -6920,7 +7739,7 @@ where return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } }; - self.stop_worker_under_lock(worker_ref) + self.stop_worker_under_lock(worker_ref, None) } fn cancel_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { @@ -7055,6 +7874,33 @@ where method, ) }); + let stop_backend = Arc::clone(&self); + let stop_lock = Arc::clone(&operation_lock); + let stop_execution = execution.clone(); + let stop_worker_ref = worker_ref.clone(); + let stop = Arc::new(move |request: WorkerExecutionStopRequest| { + let _guard = match stop_lock.lock() { + Ok(guard) => guard, + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "worker operation lock is poisoned", + ); + } + }; + if request.worker_ref != stop_worker_ref { + return WorkerExecutionResult::rejected( + WorkerExecutionOperation::Stop, + "bound stop target mismatch", + ); + } + if let Err(result) = stop_backend + .validate_protocol_execution_under_lock(&stop_worker_ref, &stop_execution) + { + return result; + } + stop_backend.stop_worker_operation_under_lock(request) + }); let validator_backend = Arc::clone(&self); let validator_lock = Arc::clone(&operation_lock); let validator_worker_ref = worker_ref.clone(); @@ -7068,13 +7914,10 @@ where validator_backend .validate_protocol_execution_under_lock(&validator_worker_ref, &execution) }); - Ok(WorkerProtocolTransport::new( - worker_ref.clone(), - snapshot, - events, - dispatch, - validate, - )) + Ok( + WorkerProtocolTransport::new(worker_ref.clone(), snapshot, events, dispatch, validate) + .with_stop_dispatch(stop), + ) } #[cfg(feature = "ws-server")] @@ -8608,6 +9451,7 @@ mod tests { backend.pending_restores.lock().unwrap().insert( worker_ref.clone(), Arc::new(PendingRuntimeRestore { + execution_id: uuid::Uuid::now_v7(), request: direct_restore_request(worker_ref.clone()), cleanup_only: false, task: tokio::sync::Mutex::new(None), @@ -8653,6 +9497,7 @@ mod tests { }) .unwrap(); let pending = Arc::new(PendingRuntimeRestore { + execution_id: uuid::Uuid::now_v7(), request: request.clone(), cleanup_only: false, task: tokio::sync::Mutex::new(Some(failed)), @@ -9757,6 +10602,63 @@ mod tests { ); } + #[tokio::test] + async fn controller_feature_failure_is_retained_after_real_scope_join_without_cleanup_success() + { + let scope = RuntimeExecutionTaskScope::new(tokio::spawn(async {})); + scope.push( + "protocol bridge", + tokio::spawn(std::future::pending()), + true, + ); + let (sender, receiver) = tokio::sync::oneshot::channel(); + let failure = worker::hook::HookError::new( + worker::hook::HookErrorCategory::Timeout, + "Feature cleanup timed out", + ); + sender + .send(worker::controller::ControllerShutdownReport { + feature_failures: vec![failure.clone()], + }) + .unwrap(); + let shutdown = Arc::new(tokio::sync::Mutex::new(Some(receiver))); + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let operation_id = crate::execution::WorkerLifecycleOperationId::new(); + for _ in 0..2 { + assert!( + scope + .confirm_shutdown_and_join(&shutdown) + .await + .unwrap_err() + .contains("Feature shutdown failures") + ); + assert!(scope.tasks.lock().unwrap().is_empty()); + let termination = scope.termination(&worker_ref, operation_id).unwrap(); + assert!(!termination.confirms_scope_shutdown(&worker_ref, operation_id)); + assert_eq!(termination.feature_failures[0].category, failure.category); + assert_eq!( + termination.feature_failures[0].diagnostic, + failure.diagnostic + ); + } + } + + #[tokio::test] + async fn joined_tasks_without_controller_report_do_not_issue_termination() { + let scope = RuntimeExecutionTaskScope::new(tokio::spawn(async {})); + scope.join().await.unwrap(); + let shutdown = Arc::new(tokio::sync::Mutex::new(None)); + assert!(scope.confirm_shutdown_and_join(&shutdown).await.is_err()); + assert!( + scope + .termination( + &WorkerRef::new(WorkerId::now_v7()), + crate::execution::WorkerLifecycleOperationId::new() + ) + .is_none() + ); + } + #[tokio::test] async fn execution_scope_drains_remaining_tasks_and_retains_abnormal_completion() { let failed = tokio::spawn(async { panic!("injected controller failure") }); @@ -10880,7 +11782,7 @@ mod tests { #[cfg(feature = "fs-store")] #[test] - fn fs_restart_stop_cleans_prior_restore_artifacts_and_retains_journal_on_failure() { + fn fs_restart_artifact_cleanup_does_not_replace_lost_execution_termination_evidence() { require_restore_test_environment(); let root = tempfile::tempdir_in(".").unwrap(); let socket_base = Path::new(root.path().file_name().unwrap()).join("controllers"); @@ -11000,18 +11902,14 @@ mod tests { fs::remove_file(&artifact).unwrap(); fs::create_dir(&artifact).unwrap(); fs::write(artifact.join("worker.err.log"), "stale restore diagnostics").unwrap(); - restarted.stop_worker(&worker.worker_ref, None).unwrap(); + let unresolved = restarted.stop_worker(&worker.worker_ref, None).unwrap_err(); + assert!( + unresolved + .to_string() + .contains("map absence is not shutdown proof") + ); assert!(!artifact.exists()); - assert_eq!( - restarted.worker_detail(&worker.worker_ref).unwrap().status, - crate::catalog::WorkerStatus::Stopped - ); - let journal: serde_json::Value = - serde_json::from_slice(&fs::read(&journal_path).unwrap()).unwrap(); - assert_ne!( - journal["execution_state"]["state"], - "reconciliation_required" - ); + assert_eq!(fs::read(&journal_path).unwrap(), journal_before); assert_eq!( fs::read_to_string(aggregate.join("session").join("keep-authority")).unwrap(), "saved Session authority" @@ -11020,7 +11918,7 @@ mod tests { assert!(another_worker.join("keep.log").is_file()); assert!(!profile_base.exists()); assert!(backend.workers.lock().unwrap().is_empty()); - restarted.stop_worker(&worker.worker_ref, None).unwrap(); + assert!(restarted.stop_worker(&worker.worker_ref, None).is_err()); } #[tokio::test] @@ -11482,6 +12380,7 @@ mod tests { .unwrap(); let request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); let owner = Arc::new(PendingRuntimeRestore { + execution_id: uuid::Uuid::now_v7(), request: request.clone(), cleanup_only: false, task: tokio::sync::Mutex::new(None), @@ -11575,7 +12474,21 @@ mod tests { operation_id ); cleanup_allowed.store(true, Ordering::SeqCst); - assert!(backend.stop_worker_operation(stop.clone()).is_accepted()); + let cleaned = backend.stop_worker_operation(stop.clone()); + assert!(cleaned.is_accepted(), "{cleaned:?}"); + assert_eq!(cleaned.startup_failure, failed.startup_failure); + assert_eq!(cleaned.secondary_failures, failed.secondary_failures); + assert_eq!( + cleaned.startup_failure_persistence, + failed.startup_failure_persistence + ); + assert!(cleaned.startup_resources.is_some()); + let termination = cleaned.termination.as_ref().unwrap(); + assert_eq!( + termination.scope, + crate::execution::WorkerExecutionTerminationScope::UnpublishedStartupCandidate + ); + assert!(termination.confirms_scope_shutdown(&worker_ref, stop.operation_id)); assert!(backend.pending_restore(&worker_ref).unwrap().is_none()); assert!( backend.stop_worker_operation(stop).is_accepted(), @@ -11585,6 +12498,99 @@ mod tests { assert_eq!(cleanup_calls.load(Ordering::SeqCst), 3); } + struct LateSpawnFactory { + inner: MockFactory, + spawns: Arc, + release: Arc, + } + + #[async_trait] + impl RuntimeWorkerFactory for LateSpawnFactory { + async fn spawn_controller( + &self, + request: WorkerExecutionSpawnRequest, + ) -> Result { + self.spawns.fetch_add(1, Ordering::SeqCst); + self.release.notified().await; + self.inner.spawn_controller(request).await + } + + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.inner.prepare_restore(request).await + } + } + + #[test] + fn fresh_spawn_timeout_retains_factory_and_late_controller_until_bound_stop_joins() { + let root = tempfile::tempdir().unwrap(); + let spawns = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let backend = WorkerRuntimeExecutionBackend::new(LateSpawnFactory { + inner: MockFactory { + client: MockClient::sequential(vec![]), + runtime_base: root.path().join("runtime"), + cwd: root.path().to_path_buf(), + store_dir: root.path().join("sessions"), + worker_metadata_dir: root.path().join("workers"), + observed_cwds: Arc::new(Mutex::new(Vec::new())), + observed_workspace_clients: Arc::new(Mutex::new(Vec::new())), + }, + spawns: spawns.clone(), + release: release.clone(), + }) + .unwrap() + .with_spawn_restore_timeout(Duration::from_millis(1)); + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let request = WorkerExecutionSpawnRequest { + worker_ref: worker_ref.clone(), + request: create_request("late fresh Controller"), + workspace_scope: None, + context: test_execution_context(worker_ref.clone()), + workdir_attachments: BTreeMap::new(), + config_bundle: None, + }; + assert!(matches!( + backend.spawn_worker(request), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!( + backend + .pending_spawns + .lock() + .unwrap() + .contains_key(&worker_ref) + ); + assert!(backend.workers.lock().unwrap().is_empty()); + assert!(backend.termination_receipt(&worker_ref).unwrap().is_none()); + release.notify_one(); + let stop = WorkerExecutionStopRequest { + worker_ref: worker_ref.clone(), + operation_id: crate::execution::WorkerLifecycleOperationId::new(), + }; + let result = backend.stop_worker_operation(stop.clone()); + assert!(result.is_accepted(), "{result:?}"); + let termination = result.termination.unwrap(); + assert!(termination.confirms_scope_shutdown(&worker_ref, stop.operation_id)); + assert_eq!( + backend.termination_receipt(&worker_ref).unwrap(), + Some(termination.clone()) + ); + assert_eq!(spawns.load(Ordering::SeqCst), 1); + assert!(backend.pending_spawns.lock().unwrap().is_empty()); + assert!(backend.workers.lock().unwrap().is_empty()); + let replay = backend.stop_worker_operation(stop); + assert_eq!(replay.termination, Some(termination)); + let unrelated = backend.stop_worker_operation(WorkerExecutionStopRequest { + worker_ref, + operation_id: crate::execution::WorkerLifecycleOperationId::new(), + }); + assert!(!unrelated.is_accepted()); + assert!(unrelated.termination.is_none()); + } + struct LateControllerFactory { inner: MockFactory, restores: Arc, @@ -11655,7 +12661,12 @@ mod tests { )); assert!(backend.workers.lock().unwrap().is_empty()); release.notify_one(); - assert!(backend.stop_worker(&worker_ref).is_accepted()); + let stopped = backend.stop_worker_operation(WorkerExecutionStopRequest { + worker_ref: worker_ref.clone(), + operation_id: crate::execution::WorkerLifecycleOperationId::new(), + }); + assert!(stopped.is_accepted(), "{stopped:?}"); + assert!(backend.termination_receipt(&worker_ref).unwrap().is_some()); assert_eq!( restores.load(Ordering::SeqCst), 1, @@ -11674,6 +12685,10 @@ mod tests { backend.restore_worker(direct_restore_request(worker_ref.clone())), WorkerExecutionSpawnResult::Connected { .. } )); + assert!( + backend.termination_receipt(&worker_ref).unwrap().is_none(), + "the replacement Controller must not reuse its predecessor's termination" + ); let old_send = backend.run_on_adapter_runtime(async move { old_handle .send(Method::ListWorkers) @@ -11685,7 +12700,16 @@ mod tests { "old protocol transport must stay closed after restore" ); assert_eq!(restores.load(Ordering::SeqCst), 2); - assert!(backend.stop_worker(&worker_ref).is_accepted()); + let second = backend.stop_worker_operation(WorkerExecutionStopRequest { + worker_ref: worker_ref.clone(), + operation_id: crate::execution::WorkerLifecycleOperationId::new(), + }); + assert!(second.is_accepted(), "{second:?}"); + let retained = backend.termination_receipt(&worker_ref).unwrap().unwrap(); + assert_ne!( + retained.execution_id, + stopped.termination.unwrap().execution_id + ); #[cfg(feature = "ws-server")] assert!(backend.worker_snapshot(&worker_ref).is_none()); } @@ -11733,7 +12757,102 @@ mod tests { } #[test] - fn absent_execution_has_typed_input_rejection_and_safe_stop() { + #[cfg(feature = "ws-server")] + #[serial_test::serial(worker_allocation)] + fn receipt_reconciliation_reads_worker_receipts_without_appending_or_activating_input() { + // Real Runtime adapter + Controller + pending receipt storage. Only the + // model stream is gated so queries cannot race unrelated history appends. + for explicit_segments in [false, true] { + let root = tempfile::tempdir().unwrap(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let client = MockClient::sequential(vec![MockResponse::Gated { + started: started.clone(), + release: release.clone(), + events: simple_text_events(), + }]); + let calls = client.call_count.clone(); + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new(MockFactory { + client, + runtime_base: root.path().join("runtime"), + cwd: root.path().to_path_buf(), + store_dir: root.path().join("sessions"), + worker_metadata_dir: root.path().join("workers"), + observed_cwds: Arc::new(Mutex::new(Vec::new())), + observed_workspace_clients: Arc::new(Mutex::new(Vec::new())), + }) + .unwrap(), + ); + let runtime = + EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend.clone()) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let detail = runtime + .create_worker(create_request("read-only receipt")) + .unwrap(); + let handle = backend.get_execution(&detail.worker_ref).unwrap().0; + let session = || match handle.snapshot_event() { + Event::Snapshot { session, .. } => session, + _ => unreachable!("Controller full snapshot"), + }; + let mut input = WorkerInput::user(" fallback text "); + input.kind = WorkerInputKind::UserIfIdle; + input.submission_request_id = Some("receipt-query".into()); + if explicit_segments { + input.segments = Some(vec![Segment::text(" explicit text ")]); + } + let before = session(); + let missing = backend.reconcile_submission(&detail.worker_ref, input.clone()); + assert_eq!( + missing.outcome, + crate::execution::WorkerExecutionOutcome::Errored + ); + assert!(missing.submission.is_none()); + assert!(missing.termination.is_none()); + assert_eq!(session(), before); + assert_eq!(calls.load(Ordering::SeqCst), 0); + + let accepted = backend.dispatch_input(&detail.worker_ref, input.clone()); + assert!(accepted.is_accepted(), "{accepted:?}"); + backend + .run_on_adapter_runtime(async move { + tokio::time::timeout(Duration::from_secs(5), started.notified()) + .await + .map_err(|_| "model stream did not reach query gate".to_string()) + }) + .unwrap(); + let before = session(); + let found = backend.reconcile_submission(&detail.worker_ref, input.clone()); + assert!(found.is_accepted(), "{found:?}"); + assert_eq!(found.submission, accepted.submission); + assert_eq!(found.execution_id, accepted.execution_id); + assert!(found.termination.is_none()); + + let mut conflicting = input.clone(); + conflicting.segments = Some(vec![Segment::text("different payload")]); + let conflict = backend.reconcile_submission(&detail.worker_ref, conflicting); + assert_eq!( + conflict.outcome, + crate::execution::WorkerExecutionOutcome::Errored + ); + assert!(conflict.submission.is_none()); + let mut missing_input = input; + missing_input.submission_request_id = Some("absent-request".into()); + let missing = backend.reconcile_submission(&detail.worker_ref, missing_input); + assert_eq!( + missing.outcome, + crate::execution::WorkerExecutionOutcome::Errored + ); + assert!(missing.submission.is_none()); + assert_eq!(session(), before); + assert_eq!(calls.load(Ordering::SeqCst), 1); + runtime.stop_worker(&detail.worker_ref, None).unwrap(); + } + } + + #[test] + fn absent_execution_has_typed_input_rejection_but_no_stop_evidence() { let backend = WorkerRuntimeExecutionBackend::new(FailingFactory).unwrap(); let worker_ref = WorkerRef::new(WorkerId::now_v7()); let rejected = backend.dispatch_input(&worker_ref, WorkerInput::user("input")); @@ -11743,7 +12862,9 @@ mod tests { crate::execution::WorkerExecutionOutcome::Rejected ); assert!(rejected.message.unwrap().contains("live Worker execution")); - assert!(backend.stop_worker(&worker_ref).is_accepted()); + let stopped = backend.stop_worker(&worker_ref); + assert!(!stopped.is_accepted()); + assert!(stopped.termination.is_none()); assert!(backend.workers.lock().unwrap().is_empty()); } diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index eb4a0c3e1..21d98ebbc 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -42,6 +42,14 @@ use workdir::{ // WorkerHandle — client-facing, Clone-able // --------------------------------------------------------------------------- +/// Saved acceptance only; neither processing nor execution cleanup is implied. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SubmissionReceipt { + pub submission_request_id: String, + pub submission_id: String, + pub disposition: protocol::SubmissionDisposition, +} + #[derive(Clone)] pub struct WorkerHandle { method_tx: mpsc::Sender, @@ -61,6 +69,25 @@ pub struct WorkerHandle { } impl WorkerHandle { + /// Read the existing receipt without resending input or activating a run. + /// Missing evidence is not proof of rejection and must not authorize replay. + pub fn reconcile_submission( + &self, + submission_request_id: &str, + input: &[protocol::Segment], + source: &protocol::AuthenticatedInputSource, + ) -> Result, String> { + let namespace = if matches!(source, protocol::AuthenticatedInputSource::UntrustedWire) { + format!("direct:{}", self.session_id) + } else { + source.namespace() + }; + self.pending_activations + .lock() + .map_err(|_| "pending activation receipt state is poisoned".to_string())? + .reconcile_submission(submission_request_id, input, &namespace) + } + /// Transport closure belongs to this Controller endpoint, not a WorkerRef. pub fn protocol_is_closed(&self) -> bool { self.method_tx.is_closed() @@ -651,7 +678,15 @@ fn prepare_pending_run( // WorkerController — actor that owns a Worker // --------------------------------------------------------------------------- -pub type ShutdownReceiver = oneshot::Receiver<()>; +/// Sent only after Controller admission, child tasks, Workdir sessions, and observer tasks +/// have closed. Feature cleanup failures are retained separately; this is not Backend Host +/// operation drain, domain writer revocation, or a resource-removal receipt. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ControllerShutdownReport { + pub feature_failures: Vec, +} + +pub type ShutdownReceiver = oneshot::Receiver; pub struct WorkerController; @@ -664,7 +699,7 @@ pub(crate) struct InitializedWorkerController { pub(crate) worker: Worker, pub(crate) handle: WorkerHandle, method_rx: mpsc::Receiver, - shutdown_tx: oneshot::Sender<()>, + shutdown_tx: oneshot::Sender, shutdown_rx: ShutdownReceiver, registry_reclaim: PreparedSpawnedWorkerRegistryLoad, shutdown_after_idle: ShutdownAfterIdleRequest, @@ -858,7 +893,7 @@ impl WorkerController { // === 1. Initialization (channels / RuntimeDir / worker-immutable // snapshots / SpawnedWorkerRegistry / alerter attach) === - let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let (shutdown_tx, shutdown_rx) = oneshot::channel::(); let (method_tx, method_rx) = mpsc::channel::(32); let (working_event_tx, _) = broadcast::channel::(256); let alerter = Alerter::new(working_event_tx.clone()); @@ -2380,7 +2415,7 @@ async fn controller_loop( pause_tx: mpsc::Sender<()>, notify_buffer: NotifyBuffer, spawned_registry: Arc, - shutdown_tx: oneshot::Sender<()>, + shutdown_tx: oneshot::Sender, shutdown_after_idle: ShutdownAfterIdleRequest, mut command_observer: Option>, ) where @@ -3245,7 +3280,11 @@ async fn controller_loop( drop(method_rx); // Feature callbacks and tasks share the Worker scope. Stop them before // Memory/Workdir teardown so they cannot observe a partially closed Worker. - worker.stop_feature_runtime("controller shutdown").await; + let feature_failures = worker + .stop_feature_runtime_report("controller shutdown") + .await + .err() + .unwrap_or_default(); loop { match spawned_registry.shutdown_internal().await { @@ -3275,7 +3314,7 @@ async fn controller_loop( let _ = observer.await; } - let _ = shutdown_tx.send(()); + let _ = shutdown_tx.send(ControllerShutdownReport { feature_failures }); } /// Stage an agent-visible Internal SubWorker lifecycle event. @@ -5524,7 +5563,7 @@ mod tests { fn controller_shutdown_orders_child_cleanup_before_workdir_close() { let source = include_str!("controller.rs"); let shutdown_start = source - .rfind("worker.stop_feature_runtime(\"controller shutdown\")") + .rfind(".stop_feature_runtime_report(\"controller shutdown\")") .expect("controller shutdown block"); let shutdown = &source[shutdown_start..]; let children = shutdown diff --git a/crates/worker/src/hook.rs b/crates/worker/src/hook.rs index c480471dc..f88f0969d 100644 --- a/crates/worker/src/hook.rs +++ b/crates/worker/src/hook.rs @@ -947,19 +947,36 @@ impl HookRegistry { &self, context: &WorkerStoppingContext, ) -> Result<(), HookExecutionError> { + let mut failures = Vec::new(); + self.on_worker_stopping_report(context, &mut failures).await; + failures + .into_iter() + .find(|error| error.policy == HookFailurePolicy::FailClosed) + .map_or(Ok(()), Err) + } + + /// Run the same policy-governed chain once, retaining even attention/fail-open errors. + /// Continuing a lifecycle chain is not evidence that its cleanup succeeded. This report + /// does not infer failures from the bounded, shared diagnostics buffer. The caller owns + /// collected failures even if its chain deadline cancels the current callback. + pub(crate) async fn on_worker_stopping_report( + &self, + context: &WorkerStoppingContext, + failures: &mut Vec, + ) { for registration in &self.worker_stopping { if let Err(error) = registration.call(context).await { self.record_diagnostic(error.clone()); - match error.policy { - HookFailurePolicy::FailOpenWithDiagnostic - | HookFailurePolicy::AttentionRequired => { - tracing::warn!(owner = %error.owner, error = %error.source, "worker-stopping hook requires attention"); - } - HookFailurePolicy::FailClosed => return Err(error), + let fail_closed = error.policy == HookFailurePolicy::FailClosed; + if !fail_closed { + tracing::warn!(owner = %error.owner, error = %error.source, "worker-stopping hook requires attention"); + } + failures.push(error); + if fail_closed { + break; } } } - Ok(()) } } @@ -1208,6 +1225,137 @@ mod tests { assert_eq!(error.category, HookErrorCategory::InvalidInput); } + #[derive(Clone)] + struct FailingStoppingHook { + calls: Arc>>, + name: &'static str, + entered: Option>, + } + + #[async_trait] + impl Hook for FailingStoppingHook { + async fn call(&self, _: &WorkerStoppingContext) -> Result<(), HookError> { + self.calls.lock().unwrap().push(self.name); + if let Some(entered) = &self.entered { + entered.notify_one(); + return std::future::pending().await; + } + Err(HookError::new(HookErrorCategory::Dependency, self.name)) + } + } + + fn stopping_context() -> WorkerStoppingContext { + WorkerStoppingContext { + invocation: rewrite_context().invocation, + reason: "owned shutdown".into(), + } + } + + #[tokio::test] + async fn stopping_report_retains_all_policies_once_and_preserves_fail_closed_order() { + let calls = Arc::new(Mutex::new(Vec::new())); + let mut builder = HookRegistryBuilder::new(); + for (name, failure) in [ + ("attention", HookFailurePolicy::AttentionRequired), + ("open", HookFailurePolicy::FailOpenWithDiagnostic), + ("closed", HookFailurePolicy::FailClosed), + ("unreached", HookFailurePolicy::AttentionRequired), + ] { + builder + .add_named_worker_stopping( + name, + HookExecutionPolicy::new(failure, 1_000), + FailingStoppingHook { + calls: calls.clone(), + name, + entered: None, + }, + ) + .unwrap(); + } + let registry = builder.build(); + let mut failures = Vec::new(); + registry + .on_worker_stopping_report(&stopping_context(), &mut failures) + .await; + assert_eq!(*calls.lock().unwrap(), ["attention", "open", "closed"]); + assert_eq!( + failures + .iter() + .map(|error| error.source.diagnostic.as_str()) + .collect::>(), + ["attention", "open", "closed"] + ); + assert!( + failures + .iter() + .all(|error| error.source.category == HookErrorCategory::Dependency) + ); + assert_eq!(registry.diagnostics().len(), 3); + } + + #[tokio::test] + async fn stopping_compatibility_result_keeps_fail_open_policy_without_repeating_callback() { + for failure in [ + HookFailurePolicy::AttentionRequired, + HookFailurePolicy::FailOpenWithDiagnostic, + HookFailurePolicy::FailClosed, + ] { + let calls = Arc::new(Mutex::new(Vec::new())); + let mut builder = HookRegistryBuilder::new(); + builder + .add_named_worker_stopping( + "stop", + HookExecutionPolicy::new(failure, 1_000), + FailingStoppingHook { + calls: calls.clone(), + name: "stop", + entered: None, + }, + ) + .unwrap(); + let registry = builder.build(); + let result = registry.on_worker_stopping(&stopping_context()).await; + assert_eq!(result.is_err(), failure == HookFailurePolicy::FailClosed); + assert_eq!(*calls.lock().unwrap(), ["stop"]); + assert_eq!(registry.diagnostics().len(), 1); + } + } + + #[tokio::test] + async fn cancelling_stopping_chain_preserves_failures_already_collected_by_owner() { + let calls = Arc::new(Mutex::new(Vec::new())); + let entered = Arc::new(tokio::sync::Notify::new()); + let mut builder = HookRegistryBuilder::new(); + for (name, pending) in [("completed-failure", false), ("in-flight", true)] { + builder + .add_named_worker_stopping( + name, + HookExecutionPolicy::new(HookFailurePolicy::AttentionRequired, 1_000), + FailingStoppingHook { + calls: calls.clone(), + name, + entered: pending.then(|| entered.clone()), + }, + ) + .unwrap(); + } + let registry = builder.build(); + let context = stopping_context(); + let mut failures = Vec::new(); + { + let chain = registry.on_worker_stopping_report(&context, &mut failures); + tokio::pin!(chain); + tokio::select! { + _ = &mut chain => panic!("second callback must remain in flight"), + _ = entered.notified() => {}, + } + } + assert_eq!(*calls.lock().unwrap(), ["completed-failure", "in-flight"]); + assert_eq!(failures.len(), 1); + assert_eq!(failures[0].source.diagnostic, "completed-failure"); + } + #[test] fn hook_diagnostics_are_utf8_bounded() { let error = HookError::new(HookErrorCategory::Internal, "界".repeat(1_000)); diff --git a/crates/worker/src/internal_worker.rs b/crates/worker/src/internal_worker.rs index 2c6a52a13..3bc4e32d9 100644 --- a/crates/worker/src/internal_worker.rs +++ b/crates/worker/src/internal_worker.rs @@ -107,6 +107,40 @@ pub(crate) async fn run_internal_worker_with_cancel_sender( spec: InternalWorkerSpec, on_cancel_sender: F, ) -> Result +where + F: FnOnce(tokio::sync::mpsc::Sender<()>), +{ + run_internal_worker_with_cancel_sender_impl(spec, on_cancel_sender, &mut None).await +} + +pub(crate) struct InternalWorkerExecution { + pub result: Result, + // None means construction did not return a Worker whose Feature runtime could be joined. + pub feature_shutdown: Option>>, +} + +pub(crate) async fn run_internal_worker_owned( + spec: InternalWorkerSpec, + on_cancel_sender: F, +) -> InternalWorkerExecution +where + F: FnOnce(tokio::sync::mpsc::Sender<()>), +{ + let mut feature_shutdown = None; + let result = + run_internal_worker_with_cancel_sender_impl(spec, on_cancel_sender, &mut feature_shutdown) + .await; + InternalWorkerExecution { + result, + feature_shutdown, + } +} + +async fn run_internal_worker_with_cancel_sender_impl( + spec: InternalWorkerSpec, + on_cancel_sender: F, + feature_shutdown: &mut Option>>, +) -> Result where F: FnOnce(tokio::sync::mpsc::Sender<()>), { @@ -208,9 +242,11 @@ where .copied() .collect::>() .join(", "); - worker - .stop_feature_runtime("internal Worker feature installation failed") - .await; + *feature_shutdown = Some( + worker + .stop_feature_runtime_report("internal Worker feature installation failed") + .await, + ); return Err(InternalWorkerError { source: WorkerError::FeatureInstall(format!( "internal Worker feature installation failed: {diagnostics}; missing tools: {missing}" @@ -255,9 +291,11 @@ where history_entries: store.entries_count(session_id, segment_id), }), }; - worker - .stop_feature_runtime("internal Worker terminal outcome") - .await; + *feature_shutdown = Some( + worker + .stop_feature_runtime_report("internal Worker terminal outcome") + .await, + ); outcome } diff --git a/crates/worker/src/job.rs b/crates/worker/src/job.rs index d94454141..961954630 100644 --- a/crates/worker/src/job.rs +++ b/crates/worker/src/job.rs @@ -6,9 +6,10 @@ //! not a final answer or completion of execution. //! //! The Host manages concurrency, timeouts, and retries, and awaits `run` and resource release even -//! after cancellation. Emergency drop or abort cannot confirm asynchronous cleanup, so leases and -//! unknown-outcome state remain. The result Feature has no background work, but dropping a tool -//! future cannot interrupt a running synchronous sink. The Host must also bound the sink's work. +//! after cancellation. Emergency drop or abort cannot confirm asynchronous cleanup, so the Host +//! retains the exact execution obligations and their diagnosis. The result Feature has no background +//! work, but dropping a tool future cannot interrupt a running synchronous sink. The Host must also +//! bound the sink's work. mod result; pub use result::{JobResultFeature, JobResultFinalizer, JobResultSink, SUBMIT_JOB_RESULT_TOOL}; @@ -18,11 +19,11 @@ use std::sync::Arc; use agen::llm_client::LlmClient; use manifest::{ProfileDiscovery, ProfileResolveOptions, ProfileResolver, Scope, WorkerManifest}; +use sha2::{Digest, Sha256}; use crate::feature::FeatureRegistryBuilder; use crate::internal_worker::{ - InternalWorkerAuthority, InternalWorkerIdentity, InternalWorkerSpec, - run_internal_worker_with_cancel_sender, + InternalWorkerAuthority, InternalWorkerIdentity, InternalWorkerSpec, run_internal_worker_owned, }; use crate::{ PromptCatalog, PromptCatalogSource, SystemPromptContext, SystemPromptTemplate, @@ -48,7 +49,7 @@ pub enum JobExecutionError { #[error("Job model configuration failed: {0}")] Model(#[from] crate::ProviderError), #[error("Job execution failed: {0}")] - Worker(String), + Worker(#[source] Box), /// A sink error can follow durable acceptance. Hosts must reconcile their store and must /// not classify this as a known retryable failure unless they can prove non-acceptance. #[error( @@ -68,6 +69,76 @@ pub struct InternalJobResult { pub usage: Option, } +/// Result acceptance and owned Feature shutdown from one exact internal execution. +/// +/// Feature shutdown covers the Worker-managed stopping hooks and background registry only. +/// Hosts must separately join/revoke their domain callbacks and persistence operations before +/// releasing an attempt. Neither a dropped future nor this value's absence is shutdown evidence. +#[derive(Debug)] +pub struct OwnedInternalJobExecution { + pub job_id: String, + pub attempt_id: String, + pub input_digest: String, + pub run_id: uuid::Uuid, + pub result: Result, + unconfirmed_submission: Option<::job::JobResultSubmission>, + post_result_failure: Option>, + feature_shutdown: Option>>, +} + +impl OwnedInternalJobExecution { + /// The exact sealed submission whose sink acknowledgement failed, after finalization and + /// result validation. The Host can reconcile/retry only this payload against its saved + /// attempt; this is not accepted output or authority to rerun the finalizer/model. + pub fn unconfirmed_submission(&self) -> Option<&::job::JobResultSubmission> { + self.unconfirmed_submission.as_ref() + } + + pub fn feature_shutdown_complete(&self) -> bool { + matches!(self.feature_shutdown, Some(Ok(()))) + } + + pub fn feature_shutdown_failures(&self) -> &[crate::hook::HookError] { + match &self.feature_shutdown { + Some(Err(errors)) => errors, + _ => &[], + } + } + + /// Existing typed cleanup ownership is distinct from a failed model/Run diagnosis. False + /// is not termination proof: callers still require the independently joined Feature scope + /// and their exact Host effects. Construction and compaction cleanup must retain their owner. + pub fn requires_worker_cleanup_reconciliation(&self) -> bool { + fn retained(error: &crate::WorkerError) -> bool { + use crate::WorkerError; + match error { + WorkerError::ConstructionCleanup { .. } + | WorkerError::CompactionCleanupPending { .. } => true, + WorkerError::StartupFailure { source, .. } + | WorkerError::RunFailureAlreadyDelivered { source } + | WorkerError::RestoreActivationEffects { source } => retained(source), + WorkerError::RunTerminalPersistence { + run_failure, + persist_error, + } => retained(run_failure) || retained(persist_error), + _ => false, + } + } + self.worker_failure().is_some_and(retained) + } + + /// A later Worker failure does not invalidate durable result acceptance. Keep its typed + /// cause (including startup attachments and retained cleanup handles) for the Host owner. + pub fn worker_failure(&self) -> Option<&crate::WorkerError> { + self.post_result_failure + .as_deref() + .or_else(|| match &self.result { + Err(JobExecutionError::Worker(error)) => Some(error.as_ref()), + _ => None, + }) + } +} + /// Resolved policy/model/prompt, with no running resources and no inherited authority. pub struct PreparedInternalJob { request: ::job::JobRequest, @@ -196,11 +267,32 @@ fn prepare_job_from_registry_with_grant( }) } +/// Stable internal execution identity for the immutable attempt. Hosts can persist this binding +/// before dispatch; replay of a receipt must never start a second execution with the same identity. +pub fn execution_run_id( + request: &::job::JobRequest, + attempt_id: &str, +) -> Result { + request.validate()?; + ::job::validate_identifier("attempt_id", attempt_id, 512)?; + let identity = ::job::fingerprint(&( + "internal-job-execution", + &request.job_id, + attempt_id, + request.input_digest()?, + ))?; + let digest = Sha256::digest(identity.as_bytes()); + let mut bytes = [0; 16]; + bytes.copy_from_slice(&digest[..16]); + // UUIDv8 reserves application-defined identity bits; this is not a random run token. + bytes[6] = (bytes[6] & 0x0f) | 0x80; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + Ok(uuid::Uuid::from_bytes(bytes)) +} + impl PreparedInternalJob { - /// Run one immutable attempt. The sink must perform fenced, durable acceptance before - /// returning Ok. Timeout is Host-owned: cancel and then await this future for cleanup. - /// A result already accepted by the sink remains success even if the model subsequently - /// fails, reaches its turn limit or is cancelled; callers never rerun an accepted result. + /// Return result acceptance only. This convenience API does not prove Feature or Host + /// shutdown; resource-owning callers must use [Self::run_owned] and their domain fence. pub async fn run( self, attempt_id: String, @@ -210,6 +302,26 @@ impl PreparedInternalJob { where F: FnOnce(tokio::sync::mpsc::Sender<()>), { + self.run_owned(attempt_id, submit, on_cancel_sender) + .await? + .result + } + + /// Await the actual run and Feature shutdown, independently of accepted model output. + /// Timeout is Host-owned: cancel and keep owning this future and domain operations until + /// their separate termination/write-revocation evidence has been observed. + pub async fn run_owned( + self, + attempt_id: String, + submit: Arc, + on_cancel_sender: F, + ) -> Result + where + F: FnOnce(tokio::sync::mpsc::Sender<()>), + { + let run_id = execution_run_id(&self.request, &attempt_id)?; + let job_id = self.request.job_id.clone(); + let input_digest = self.request.input_digest()?; let mut feature = JobResultFeature::new(&self.request, attempt_id.clone(), submit)?; if let Some(finalizer) = self.finalizer { feature = @@ -220,7 +332,7 @@ impl PreparedInternalJob { let spec = InternalWorkerSpec { identity: InternalWorkerIdentity { kind: "job", - run_id: uuid::Uuid::now_v7(), + run_id, }, max_turns: engine_policy.max_turns.map(|n| n.get()), compaction_budget: None, @@ -241,29 +353,40 @@ impl PreparedInternalJob { workdir_session: None, }, }; - let outcome = run_internal_worker_with_cancel_sender(spec, on_cancel_sender).await; + let execution = run_internal_worker_owned(spec, on_cancel_sender).await; + feature.revoke_and_join().await; + let outcome = execution.result; let usage = match &outcome { Ok(run) => run.usage.clone(), Err(error) => error.usage.clone(), }; - if let Some(submission) = accepted.lock().expect("Job result state poisoned").clone() { - return Ok(InternalJobResult { submission, usage }); - } - if let Some(error) = feature.unconfirmed() { - return Err(JobExecutionError::ResultUnconfirmed(error)); - } - match outcome { - Err(error) => Err(JobExecutionError::Worker(error.source.to_string())), - Ok(run) - if matches!( - run.lifecycle, - WorkerRunResult::Cancelled | WorkerRunResult::RolledBack - ) => - { - Err(JobExecutionError::Cancelled) - } - Ok(_) => Err(JobExecutionError::MissingResult), - } + let lifecycle = outcome.as_ref().ok().map(|run| run.lifecycle.clone()); + let mut worker_failure = outcome.err().map(|error| Box::new(error.source)); + let submission = accepted.lock().expect("Job result state poisoned").clone(); + let result = if let Some(submission) = submission { + Ok(InternalJobResult { submission, usage }) + } else if let Some(error) = feature.unconfirmed() { + Err(JobExecutionError::ResultUnconfirmed(error)) + } else if let Some(error) = worker_failure.take() { + Err(JobExecutionError::Worker(error)) + } else if matches!( + lifecycle, + Some(WorkerRunResult::Cancelled | WorkerRunResult::RolledBack) + ) { + Err(JobExecutionError::Cancelled) + } else { + Err(JobExecutionError::MissingResult) + }; + Ok(OwnedInternalJobExecution { + job_id, + attempt_id, + input_digest, + run_id, + result, + unconfirmed_submission: feature.unconfirmed_submission(), + post_result_failure: worker_failure, + feature_shutdown: execution.feature_shutdown, + }) } } diff --git a/crates/worker/src/job/result.rs b/crates/worker/src/job/result.rs index 525bba9e0..e479427ac 100644 --- a/crates/worker/src/job/result.rs +++ b/crates/worker/src/job/result.rs @@ -124,6 +124,24 @@ impl JobResultFeature { .clone() } + pub(super) fn unconfirmed_submission(&self) -> Option<::job::JobResultSubmission> { + let state = self.state.lock().expect("Job submission state poisoned"); + state.unconfirmed.as_ref()?; + Some(::job::JobResultSubmission { + job_id: self.job_id.clone(), + attempt_id: self.attempt_id.clone(), + input_digest: self.input_digest.clone(), + result: state.prepared.clone()?, + }) + } + + /// Revoke every clone and join the exact serialized finalizer/sink operation. Cancellation + /// alone does not prove a synchronous Host sink or an asynchronous finalizer has returned. + pub(super) async fn revoke_and_join(&self) { + self.cancellation.cancel(); + let _operation = self.serial.lock().await; + } + fn definition(&self) -> ToolDefinition { let feature = self.clone(); Arc::new(move || { @@ -190,11 +208,12 @@ impl Tool for JobResultFeature { ::job::result_digest(&prepared, self.max_result_bytes) .map_err(|e| ToolError::InvalidArgument(e.to_string()))?; let mut state = self.state.lock().expect("Job submission state poisoned"); - if self - .cancelled - .lock() - .expect("Job cancellation state poisoned") - .contains(&context.execution_id()) + if self.cancellation.is_cancelled() + || self + .cancelled + .lock() + .expect("Job cancellation state poisoned") + .contains(&context.execution_id()) { return Err(ToolError::Cancelled(ToolOutput { summary: "Job result submission cancelled".into(), diff --git a/crates/worker/src/job/tests.rs b/crates/worker/src/job/tests.rs index 4c867bd61..b21ec7c9e 100644 --- a/crates/worker/src/job/tests.rs +++ b/crates/worker/src/job/tests.rs @@ -565,7 +565,7 @@ async fn accepted_result_survives_later_provider_error_and_cancellation() { let (tx, rx) = tokio::sync::oneshot::channel(); let recording = sink.clone(); let run = tokio::spawn(async move { - job.run("attempt".into(), recording, move |sender| { + job.run_owned("attempt".into(), recording, move |sender| { let _ = tx.send(sender); }) .await @@ -585,7 +585,22 @@ async fn accepted_result_survives_later_provider_error_and_cancellation() { .unwrap() .unwrap() .unwrap(); - assert_eq!(result.submission.result, json!(true)); + assert_eq!(result.job_id, "job-a"); + assert_eq!(result.attempt_id, "attempt"); + assert_eq!( + result.input_digest, + request("user:a").input_digest().unwrap() + ); + assert!(result.feature_shutdown_complete()); + assert!(result.feature_shutdown_failures().is_empty()); + assert!(!result.requires_worker_cleanup_reconciliation()); + if !cancel { + assert!(matches!( + result.worker_failure(), + Some(crate::WorkerError::Engine(_)) + )); + } + assert_eq!(result.result.unwrap().submission.result, json!(true)); assert_eq!(sink.0.lock().unwrap().len(), 1); assert!(sender.is_closed()); } @@ -1237,6 +1252,39 @@ async fn unconfirmed_sink_acceptance_is_not_classified_as_missing_result_or_know assert_eq!(sink.0.lock().unwrap().len(), 1); } +#[tokio::test] +async fn unconfirmed_sink_retains_exact_finalized_payload_without_claiming_acceptance() { + let root = tempfile::tempdir().unwrap(); + let finalizer = Arc::new(ControlledFinalizer::default()); + let sink = Arc::new(UnknownSink(Mutex::new(vec![]))); + let mut job = prepared( + root.path(), + &request("user:a"), + ScriptedClient::new(vec![ + call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{"claim":true}})), + prose(), + ]), + ); + job.finalizer = Some(finalizer.clone()); + let execution = job + .run_owned("attempt".into(), sink.clone(), |_| {}) + .await + .unwrap(); + assert!(matches!( + &execution.result, + Err(JobExecutionError::ResultUnconfirmed(_)) + )); + let observed = sink.0.lock().unwrap(); + assert_eq!(observed.len(), 1); + assert_eq!(execution.unconfirmed_submission(), observed.first()); + assert_eq!( + execution.unconfirmed_submission().unwrap().result, + json!({"prepared":{"claim":true},"completion":1}) + ); + assert_eq!(finalizer.calls.load(Ordering::SeqCst), 1); + assert!(execution.feature_shutdown_complete()); +} + #[tokio::test] async fn feature_registration_rejects_forged_fences_seals_unknown_outcomes_and_honors_cancel() { let sink = Arc::new(UnknownSink(Mutex::new(vec![]))); @@ -1282,3 +1330,224 @@ async fn feature_registration_rejects_forged_fences_seals_unknown_outcomes_and_h )); assert_eq!(sink.0.lock().unwrap().len(), 2); } + +#[tokio::test] +async fn revocation_joins_finalizer_cleanup_and_disables_every_result_clone() { + let root = tempfile::tempdir().unwrap(); + let prepared = prepared(root.path(), &request("user:a"), ScriptedClient::new(vec![])); + let finalizer = Arc::new(ControlledFinalizer { + gated: true, + ..Default::default() + }); + let sink = Arc::new(RecordingSink::default()); + let feature = JobResultFeature::new(&request("user:a"), "attempt".into(), sink.clone()) + .unwrap() + .with_finalizer(finalizer.clone(), prepared.manifest, prepared.client); + let executing = feature.clone(); + let operation = tokio::spawn(async move { + executing + .execute(r#"{"result":true}"#, ToolExecutionContext::direct()) + .await + }); + signalled(&finalizer.entered).await; + let stopping = feature.clone(); + let shutdown = tokio::spawn(async move { stopping.revoke_and_join().await }); + signalled(&finalizer.cancellation_seen).await; + assert!( + !shutdown.is_finished(), + "cancellation is not cleanup completion" + ); + assert_eq!(finalizer.active.load(Ordering::SeqCst), 1); + assert!(sink.0.lock().unwrap().is_empty()); + finalizer.cleanup.notify_one(); + assert!(matches!( + operation.await.unwrap(), + Err(ToolError::Cancelled(_)) + )); + tokio::time::timeout(std::time::Duration::from_secs(2), shutdown) + .await + .unwrap() + .unwrap(); + assert_eq!(finalizer.active.load(Ordering::SeqCst), 0); + assert!(matches!( + feature + .execute(r#"{"result":true}"#, ToolExecutionContext::direct()) + .await, + Err(ToolError::Cancelled(_)) + )); + assert_eq!(finalizer.calls.load(Ordering::SeqCst), 1); + assert!(sink.0.lock().unwrap().is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn revocation_does_not_claim_a_running_synchronous_sink_has_ended() { + struct BlockingSink { + entered: tokio::sync::Notify, + release: Mutex>, + accepted: AtomicUsize, + } + impl JobResultSink for BlockingSink { + fn submit(&self, _: ::job::JobResultSubmission) -> Result<(), String> { + self.entered.notify_one(); + self.release + .lock() + .unwrap() + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("test must release the synchronous sink within its finite budget"); + self.accepted.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + let (release, receiver) = std::sync::mpsc::channel(); + let sink = Arc::new(BlockingSink { + entered: Default::default(), + release: Mutex::new(receiver), + accepted: AtomicUsize::new(0), + }); + let feature = + JobResultFeature::new(&request("user:a"), "attempt".into(), sink.clone()).unwrap(); + let executing = feature.clone(); + let operation = tokio::spawn(async move { + executing + .execute(r#"{"result":true}"#, ToolExecutionContext::direct()) + .await + }); + signalled(&sink.entered).await; + let stopping = feature.clone(); + let started = Arc::new(tokio::sync::Notify::new()); + let observed = started.clone(); + let shutdown = tokio::spawn(async move { + observed.notify_one(); + stopping.revoke_and_join().await; + }); + signalled(&started).await; + assert!( + !shutdown.is_finished(), + "a live sink still owns its Host effect" + ); + assert_eq!(sink.accepted.load(Ordering::SeqCst), 0); + release.send(()).unwrap(); + operation.await.unwrap().unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), shutdown) + .await + .unwrap() + .unwrap(); + assert_eq!(sink.accepted.load(Ordering::SeqCst), 1); + assert!(matches!( + feature + .execute(r#"{"result":true}"#, ToolExecutionContext::direct()) + .await, + Err(ToolError::Cancelled(_)) + )); + assert_eq!(sink.accepted.load(Ordering::SeqCst), 1); +} + +#[derive(Clone)] +struct StoppingFeature { + entered: Arc, + release: Arc, + policy: crate::hook::HookFailurePolicy, + failure: Option, +} + +#[async_trait] +impl crate::hook::Hook for StoppingFeature { + async fn call( + &self, + _: &crate::hook::WorkerStoppingContext, + ) -> Result<(), crate::hook::HookError> { + self.entered.notify_one(); + self.release.notified().await; + self.failure.clone().map_or(Ok(()), Err) + } +} + +impl FeatureModule for StoppingFeature { + fn descriptor(&self) -> FeatureDescriptor { + FeatureDescriptor::builtin("job-stop-observation", "Owned stopping observation").with_hook( + crate::feature::HookDeclaration::new( + "stop", + crate::feature::FeatureHookPoint::WorkerStopping, + ), + ) + } + + fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> { + context.hooks().add_worker_stopping( + "stop", + crate::hook::HookExecutionPolicy::new(self.policy, 1_000), + self.clone(), + ) + } +} + +// Production Internal Worker/Feature lifecycle and result capability, with only the model +// transport and one stopping callback replaced. This is not a process-termination test. +#[tokio::test] +async fn accepted_result_waits_for_stopping_hook_and_retains_every_failure_policy() { + use crate::hook::{HookError, HookErrorCategory, HookFailurePolicy}; + + for (policy, failure) in [ + (HookFailurePolicy::FailClosed, None), + ( + HookFailurePolicy::FailClosed, + Some(HookError::new( + HookErrorCategory::Dependency, + "stop rejected", + )), + ), + ( + HookFailurePolicy::AttentionRequired, + Some(HookError::new( + HookErrorCategory::Dependency, + "stop rejected", + )), + ), + ( + HookFailurePolicy::FailOpenWithDiagnostic, + Some(HookError::new( + HookErrorCategory::Dependency, + "stop rejected", + )), + ), + ] { + let root = tempfile::tempdir().unwrap(); + let callback = StoppingFeature { + entered: Arc::default(), + release: Arc::default(), + policy, + failure: failure.clone(), + }; + let entered = callback.entered.clone(); + let release = callback.release.clone(); + let client = ScriptedClient::new(vec![ + call(SUBMIT_JOB_RESULT_TOOL, json!({"result":true})), + prose(), + ]); + let mut job = prepared(root.path(), &request("user:a"), client); + job.features = job.features.with_module(callback); + let sink = Arc::new(RecordingSink::default()); + let recording = sink.clone(); + let run = + tokio::spawn(async move { job.run_owned("attempt".into(), recording, |_| {}).await }); + signalled(&entered).await; + assert_eq!(sink.0.lock().unwrap().len(), 1); + assert!( + !run.is_finished(), + "result acceptance cannot bypass a live stopping hook" + ); + release.notify_one(); + let execution = tokio::time::timeout(std::time::Duration::from_secs(2), run) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(execution.feature_shutdown_complete(), failure.is_none()); + match failure { + Some(error) => assert!(execution.feature_shutdown_failures().contains(&error)), + None => assert!(execution.feature_shutdown_failures().is_empty()), + } + assert_eq!(execution.result.unwrap().submission.result, json!(true)); + assert_eq!(sink.0.lock().unwrap().len(), 1); + } +} diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index a19c433dc..232278460 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -273,6 +273,36 @@ where } impl PendingActivationState { + pub(crate) fn reconcile_submission( + &self, + request_id: &str, + input: &[Segment], + namespace: &str, + ) -> Result, String> { + let Some(receipt) = self.receipts.iter().find(|receipt| { + receipt.submission_request_id == request_id && receipt.source_namespace == namespace + }) else { + return Ok(None); + }; + if receipt.payload_digest != submission_payload_digest(input) { + return Err(PendingSubmissionError::IdempotencyConflict.to_string()); + } + // Immediate admission is only durable after UserInput and its clearing + // checkpoint commit. Queued admission already persisted its receipt. + if receipt.disposition == protocol::SubmissionDisposition::Started + && self.activating.as_ref().is_some_and(|pending| { + pending.submission_id == receipt.submission_id && !pending.was_queued + }) + { + return Ok(None); + } + Ok(Some(crate::controller::SubmissionReceipt { + submission_request_id: receipt.submission_request_id.clone(), + submission_id: receipt.submission_id.clone(), + disposition: receipt.disposition, + })) + } + pub(crate) fn has_unfinished_admission(&self) -> bool { self.activating.is_some() || !self.pending.is_empty() @@ -4696,29 +4726,43 @@ impl Worker { } pub async fn stop_feature_runtime(&mut self, reason: impl Into) { + if let Err(failures) = self.stop_feature_runtime_report(reason).await { + for error in failures { + tracing::warn!(%error, "Feature runtime shutdown requires attention"); + } + } + } + + /// Await Worker-owned stopping hooks/background tasks and report actual failures separately + /// from a Run/result. Success covers only this Feature scope, not external Host callbacks, + /// Controller children, domain writer fencing or resource removal. + pub async fn stop_feature_runtime_report( + &mut self, + reason: impl Into, + ) -> Result<(), Vec> { + let mut failures = Vec::new(); if let Some(hooks) = self.hook_registry.clone() { let context = WorkerStoppingContext { invocation: self.hook_invocation_context(None), reason: reason.into(), }; - match tokio::time::timeout( + let mut hook_failures = Vec::new(); + let deadline = tokio::time::timeout( FEATURE_HOOK_CHAIN_TIMEOUT, - hooks.on_worker_stopping(&context), + hooks.on_worker_stopping_report(&context, &mut hook_failures), ) - .await - { - Ok(Ok(())) => {} - Ok(Err(error)) => { - tracing::warn!(error = %error, "worker-stopping hook requires attention"); - } - Err(_) => { - hooks.record_chain_timeout("worker-stopping"); - tracing::warn!("worker-stopping hook chain exceeded its host deadline"); - } + .await; + failures.extend(hook_failures.into_iter().map(|error| error.source)); + if deadline.is_err() { + hooks.record_chain_timeout("worker-stopping"); + failures.push(crate::hook::HookError::new( + crate::hook::HookErrorCategory::Timeout, + "worker-stopping hook chain exceeded its host deadline", + )); } } if let Err(error) = self.feature_background_tasks.shutdown().await { - tracing::warn!(error = %error, "feature background task shutdown failed"); + failures.push(error); } if let Some(plugins) = &self.plugin_tools { // The registry has joined the drivers, including bounded on-stop. @@ -4729,6 +4773,11 @@ impl Worker { tokio::time::sleep(std::time::Duration::from_millis(100)).await; } } + if failures.is_empty() { + Ok(()) + } else { + Err(failures) + } } } @@ -13152,8 +13201,24 @@ mod build_summary_prompt_tests { .unwrap(), }), AttributionOutcome::Rejected => Ok(WorkspaceResponse { - status: 503, - body: "temporarily unavailable".to_string(), + status: 403, + body: serde_json::to_string( + &server_api::RepositoryApiError::new( + 403, + "PermissionDenied", + "Session registration permission denied.", + Vec::new(), + ) + .with_failure( + protocol::session_failure::session_registration_attachment( + protocol::session_failure::SessionRegistrationCause::denied( + protocol::session_failure::SessionPrincipal::Subject, + ), + ) + .unwrap(), + ), + ) + .unwrap(), }), AttributionOutcome::Success => { let input: server_api::SubjektivRecordSessionRequest = @@ -13484,10 +13549,23 @@ mod build_summary_prompt_tests { #[tokio::test] async fn unknown_and_failed_attribution_remain_incomplete_until_lifecycle_retry() { - for (first, expected_state) in [ - (AttributionOutcome::TransportUnknown, "outcome_unknown"), - (AttributionOutcome::Rejected, "failed"), - (AttributionOutcome::Unauthorized, "failed"), + use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause}; + for (first, expected_state, expected_cause) in [ + ( + AttributionOutcome::TransportUnknown, + "outcome_unknown", + SessionRegistrationCause::outcome_unknown(), + ), + ( + AttributionOutcome::Rejected, + "failed", + SessionRegistrationCause::denied(SessionPrincipal::Subject), + ), + ( + AttributionOutcome::Unauthorized, + "outcome_unknown", + SessionRegistrationCause::outcome_unknown(), + ), ] { let client = Arc::new(AttributionLifecycleClient::new([ first, @@ -13500,10 +13578,10 @@ mod build_summary_prompt_tests { SubjektivSessionAttributionLifecycle::NewSession, ) .unwrap_err(); - assert!(error.to_string().contains(expected_state)); + assert_eq!(error.session_registration_cause(), Some(expected_cause)); if matches!(first, AttributionOutcome::Unauthorized) { - assert!(error.to_string().contains("rejected with HTTP 401")); - assert!(!error.to_string().contains("outcome_unknown")); + // A legacy HTTP status is not a domain authorization attestation. + // Attribution remains incomplete and no implicit retry is issued. assert_eq!( client.requests.lock().unwrap().len(), 1, @@ -13672,7 +13750,12 @@ mod build_summary_prompt_tests { ) .unwrap_err(); - assert!(error.to_string().contains("legacy_unknown")); + assert_eq!( + error.session_registration_cause(), + Some(protocol::session_failure::SessionRegistrationCause::denied( + protocol::session_failure::SessionPrincipal::Subject, + )), + ); let requests = client.requests.lock().unwrap(); assert_eq!(requests.len(), 1); let request: server_api::SubjektivRecordSessionRequest = @@ -17787,6 +17870,109 @@ permission = "write" assert_eq!(state.pending[1].provenance, account_b); } + #[test] + fn submission_receipt_reconciliation_reads_persisted_identity_without_replay() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let input = vec![Segment::text("accepted Job input")]; + let accepted = handle + .accept_from_source( + "job-input".into(), + input.clone(), + "backend:job-attempt".into(), + WorkerHistoryProvenance::LegacyUnknown, + false, + ) + .unwrap(); + let location = handle.writer.state.location(); + let before = handle + .writer + .store + .read_all(location.session_id, location.segment_id) + .unwrap(); + let payload = before + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN => { + Some(payload.clone()) + } + _ => None, + }) + .unwrap(); + let restored: PendingActivationState = serde_json::from_value(payload).unwrap(); + let receipt = restored + .reconcile_submission("job-input", &input, "backend:job-attempt") + .unwrap() + .unwrap(); + assert_eq!(receipt.submission_id, accepted.submission_id); + assert_eq!(receipt.disposition, protocol::SubmissionDisposition::Queued); + assert!( + restored + .reconcile_submission("job-input", &input, "backend:other-attempt") + .unwrap() + .is_none() + ); + assert!( + restored + .reconcile_submission("missing", &input, "backend:job-attempt") + .unwrap() + .is_none() + ); + assert!( + restored + .reconcile_submission( + "job-input", + &[Segment::text("changed")], + "backend:job-attempt" + ) + .is_err() + ); + assert_eq!(restored.pending.len(), 1); + assert_eq!( + serde_json::to_value( + handle + .writer + .store + .read_all(location.session_id, location.segment_id) + .unwrap() + ) + .unwrap(), + serde_json::to_value(before).unwrap() + ); + } + + #[test] + fn submission_receipt_reconciliation_never_confirms_uncommitted_start() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let input = vec![Segment::text("not committed yet")]; + let accepted = handle + .accept("job-start".into(), input.clone(), true) + .unwrap(); + assert!( + handle + .state + .lock() + .unwrap() + .reconcile_submission("job-start", &input, &handle.direct_client_namespace()) + .unwrap() + .is_none() + ); + handle.abort_activation(accepted.activation.unwrap()); + assert!( + handle + .state + .lock() + .unwrap() + .reconcile_submission("job-start", &input, &handle.direct_client_namespace()) + .unwrap() + .is_none() + ); + } + #[test] fn restore_reconciliation_clears_interrupted_acceptance_pin_for_retry() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/backend_job.rs b/crates/workspace-server/src/backend_job.rs index e49d552c5..68d8891a7 100644 --- a/crates/workspace-server/src/backend_job.rs +++ b/crates/workspace-server/src/backend_job.rs @@ -12,10 +12,10 @@ //! input is rejected. Attempts without execution slots remain durably reserved; dispatch advances //! on completion and restart. Only explicit reevaluation adds attempts. The store checks confirmed //! failure, remaining attempt budget, completed cleanup of prior attempt Workers, and no occupancy -//! by other Jobs. Unknown execution outcomes are not rerun as new attempts. +//! by other Jobs. Missing execution evidence never authorizes a new attempt. //! //! Serialization-key resource conflicts return the original immutable request as resource_reused. -//! This does not mean new input was processed. Pending/unknown Jobs and unfinished cleanup +//! This does not mean new input was processed. Pending Jobs and unfinished cleanup //! retain resource occupancy, distinct from replay of an identical request. //! //! # Structured results and notifications @@ -28,9 +28,9 @@ //! //! Notifications have delivery records separate from results. After restart, unknown deliveries //! are retried with the same tracked notification ID. Results survive stopped/missing destinations -//! without rerunning the model. Execution recovery matches observable Workers and tracked -//! submissions; lost evidence leaves the outcome unknown. Exactly-once execution of all external -//! side effects is not guaranteed. +//! without rerunning the model. Execution reconciliation and resource release are separate from +//! the accepted result; absence of a Worker or a response is not proof that its effects ended. +//! Exactly-once execution of arbitrary external side effects is not guaranteed. use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -63,7 +63,6 @@ pub enum BackendJobState { Pending, Completed, Failed, - Unknown, } impl From for job::JobState { @@ -72,7 +71,6 @@ impl From for job::JobState { BackendJobState::Pending => Self::Pending, BackendJobState::Completed => Self::Completed, BackendJobState::Failed => Self::Failed, - BackendJobState::Unknown => Self::Unknown, } } } @@ -83,7 +81,6 @@ impl BackendJobState { Self::Pending => "pending", Self::Completed => "completed", Self::Failed => "failed", - Self::Unknown => "unknown", } } @@ -92,7 +89,6 @@ impl BackendJobState { "pending" => Ok(Self::Pending), "completed" => Ok(Self::Completed), "failed" => Ok(Self::Failed), - "unknown" => Ok(Self::Unknown), other => Err(Error::Store(format!("unknown Backend Job state `{other}`"))), } } @@ -106,7 +102,6 @@ pub enum BackendJobAttemptState { Dispatched, Completed, Failed, - Unknown, } impl From for job::JobAttemptState { @@ -117,7 +112,6 @@ impl From for job::JobAttemptState { BackendJobAttemptState::Dispatched => Self::Dispatched, BackendJobAttemptState::Completed => Self::Completed, BackendJobAttemptState::Failed => Self::Failed, - BackendJobAttemptState::Unknown => Self::Unknown, } } } @@ -130,7 +124,6 @@ impl BackendJobAttemptState { Self::Dispatched => "dispatched", Self::Completed => "completed", Self::Failed => "failed", - Self::Unknown => "unknown", } } @@ -141,7 +134,6 @@ impl BackendJobAttemptState { "dispatched" => Ok(Self::Dispatched), "completed" => Ok(Self::Completed), "failed" => Ok(Self::Failed), - "unknown" => Ok(Self::Unknown), other => Err(Error::Store(format!( "unknown Backend Job attempt state `{other}`" ))), @@ -149,7 +141,7 @@ impl BackendJobAttemptState { } pub fn terminal(self) -> bool { - matches!(self, Self::Completed | Self::Failed | Self::Unknown) + matches!(self, Self::Completed | Self::Failed) } } @@ -282,8 +274,8 @@ pub struct SubjektivConsolidationGrant { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct SubjektivJournalGrant { - /// Frozen writer generation. Legacy absence preserves its original canonical wire/digest; - /// recovery must use only the immutable original Session and still-current body. + /// Frozen writer generation. Legacy absence preserves its original canonical wire/digest + /// at the read/migration boundary only; it never authorizes a fresh Session read or selection. #[serde(default, skip_serializing_if = "String::is_empty")] pub binding_id: String, pub subject_id: String, @@ -586,6 +578,9 @@ pub struct BackendJobAttemptRecord { pub worker: Option, pub runtime_run_id: Option, pub dispatched_at: Option, + /// The Reserved -> Failed transaction closed dispatch admission before any + /// allocation was allowed. Absence of a Worker or a terminal state is not this proof. + pub allocation_precluded_at: Option, pub deadline_at: String, pub result: Option, pub result_digest: Option, @@ -756,12 +751,11 @@ mod tests { } #[test] - fn backend_state_projections_preserve_existing_wire_vocabulary() { + fn backend_state_projections_use_current_execution_vocabulary() { for state in [ BackendJobState::Pending, BackendJobState::Completed, BackendJobState::Failed, - BackendJobState::Unknown, ] { let shared = job::JobState::from(state); assert_eq!( @@ -775,7 +769,6 @@ mod tests { BackendJobAttemptState::Dispatched, BackendJobAttemptState::Completed, BackendJobAttemptState::Failed, - BackendJobAttemptState::Unknown, ] { let shared = job::JobAttemptState::from(state); assert_eq!( @@ -783,6 +776,11 @@ mod tests { serde_json::to_value(shared).unwrap() ); } + // Legacy uncertain outcomes are accepted only by the migration reader. + assert!(BackendJobState::parse("unknown").is_err()); + assert!(BackendJobAttemptState::parse("unknown").is_err()); + assert!(serde_json::from_str::("\"unknown\"").is_err()); + assert!(serde_json::from_str::("\"unknown\"").is_err()); // The shared read vocabulary does not add Backend cancellation support. assert!(BackendJobState::parse("cancelled").is_err()); assert!(BackendJobAttemptState::parse("cancelled").is_err()); @@ -851,7 +849,6 @@ mod tests { ), (BackendJobState::Pending, BackendJobAttemptState::Dispatched), (BackendJobState::Failed, BackendJobAttemptState::Failed), - (BackendJobState::Unknown, BackendJobAttemptState::Unknown), ] { reservation.job.state = state; reservation.attempt.state = attempt_state; diff --git a/crates/workspace-server/src/backend_job_host_effects.rs b/crates/workspace-server/src/backend_job_host_effects.rs new file mode 100644 index 000000000..77bd25cd5 --- /dev/null +++ b/crates/workspace-server/src/backend_job_host_effects.rs @@ -0,0 +1,248 @@ +//! Live ownership of Host effects admitted for one exact Job execution. +//! +//! This supplements durable allocation/submission/domain receipts; it is not a Job state machine. +//! Create it only when initially admitting an execution, before its first Host callback. Recovery +//! must not manufacture a drain receipt by reconstructing an empty owner after a restart. A lost +//! owner requires reconciliation of the retained domain operations and writer fences instead. +//! Closing admission does not cancel accepted operations. Their existing DbExecutor lifetimes +//! retain the request lease through actual work and discarded-result cleanup, even if the HTTP +//! waiter disappears. Detached work must explicitly retain this same lease or is not covered. +use std::sync::{Arc, Mutex}; + +use tokio::sync::Notify; +use worker_runtime::identity::RuntimeWorkerRef; + +use crate::backend_job::BackendJobAttemptRecord; +use crate::db_executor::DbOperationLifetime; +use crate::{Error, Result}; + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub(crate) struct JobHostOwner { + pub workspace_id: String, + pub job_id: String, + pub attempt_id: String, + pub input_digest: String, + pub worker: RuntimeWorkerRef, + pub runtime_run_id: String, +} + +impl TryFrom<&BackendJobAttemptRecord> for JobHostOwner { + type Error = Error; + + fn try_from(attempt: &BackendJobAttemptRecord) -> Result { + Ok(Self { + workspace_id: attempt.workspace_id.clone(), + job_id: attempt.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_digest: attempt.input_digest.clone(), + worker: attempt.worker.clone().ok_or_else(|| { + Error::InvalidInput("Job Host ownership requires an exact Worker binding".into()) + })?, + runtime_run_id: attempt + .runtime_run_id + .clone() + .filter(|id| !id.is_empty()) + .ok_or_else(|| { + Error::InvalidInput( + "Job Host ownership requires a tracked Runtime submission".into(), + ) + })?, + }) + } +} + +struct Admission { + open: bool, + requests: usize, +} + +pub(crate) struct JobHostEffects { + owner: JobHostOwner, + admission: Mutex, + finished: Notify, +} + +impl JobHostEffects { + pub(crate) fn initially_admit(owner: JobHostOwner) -> Arc { + Arc::new(Self { + owner, + admission: Mutex::new(Admission { + open: true, + requests: 0, + }), + finished: Notify::new(), + }) + } + + pub(crate) fn admit(self: &Arc, expected: &JobHostOwner) -> Result { + if &self.owner != expected { + return Err(Error::InvalidInput( + "Job Host execution owner mismatch".into(), + )); + } + let mut admission = self + .admission + .lock() + .map_err(|_| Error::Store("Job Host admission poisoned".into()))?; + if !admission.open { + return Err(Error::InvalidInput( + "Job Host execution admission is closed".into(), + )); + } + admission.requests = admission + .requests + .checked_add(1) + .ok_or_else(|| Error::Store("Job Host request count overflow".into()))?; + Ok(JobHostRequestLease { + request: Arc::new(AdmittedRequest { + effects: self.clone(), + }), + }) + } + + /// The receipt proves only this live Host owner's accepted operations ended. Its constructor + /// is private so a missing map entry, Job terminal state, or timeout cannot create evidence. + pub(crate) async fn close_and_drain( + self: &Arc, + expected: &JobHostOwner, + ) -> Result { + if &self.owner != expected { + return Err(Error::InvalidInput("Job Host close owner mismatch".into())); + } + self.admission + .lock() + .map_err(|_| Error::Store("Job Host admission poisoned".into()))? + .open = false; + loop { + let notified = self.finished.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self + .admission + .lock() + .map_err(|_| Error::Store("Job Host admission poisoned".into()))? + .requests + == 0 + { + return Ok(JobHostDrain { + owner: self.owner.clone(), + }); + } + notified.await; + } + } +} + +/// Every clone refers to the same admitted request, including its synchronous child operations. +/// Only the last real owner ends it. Returning/dropping the async waiter alone is insufficient. +#[derive(Clone)] +pub(crate) struct JobHostRequestLease { + request: Arc, +} + +impl DbOperationLifetime for JobHostRequestLease { + fn retain(&self) -> Box { + Box::new(self.clone()) + } +} + +struct AdmittedRequest { + effects: Arc, +} + +impl Drop for AdmittedRequest { + fn drop(&mut self) { + let mut admission = self + .effects + .admission + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + admission.requests -= 1; + drop(admission); + self.effects.finished.notify_waiters(); + } +} + +pub(crate) struct JobHostDrain { + owner: JobHostOwner, +} + +impl JobHostDrain { + pub(crate) fn owner(&self) -> &JobHostOwner { + &self.owner + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db_executor::{DbExecutor, DbExecutorLimits, with_operation_lifetime}; + use std::{num::NonZeroUsize, time::Duration}; + + fn owner(attempt: &str) -> JobHostOwner { + JobHostOwner { + workspace_id: "workspace".into(), + job_id: "job".into(), + attempt_id: attempt.into(), + input_digest: "input".into(), + worker: RuntimeWorkerRef::new("runtime", "worker"), + runtime_run_id: format!("run:{attempt}"), + } + } + + #[tokio::test] + async fn cancelled_waiter_cannot_release_accepted_host_effect_or_admit_after_close() { + let expected = owner("attempt-1"); + let effects = JobHostEffects::initially_admit(expected.clone()); + let lease = effects.admit(&expected).unwrap(); + let executor = DbExecutor::new(DbExecutorLimits { + workers: NonZeroUsize::new(1).unwrap(), + queued: NonZeroUsize::new(1).unwrap(), + }) + .unwrap(); + let (entered, started) = tokio::sync::oneshot::channel(); + let (release, wait) = std::sync::mpsc::channel(); + let work = executor.clone(); + let waiter: tokio::task::JoinHandle> = + tokio::spawn(with_operation_lifetime(Arc::new(lease), async move { + work.run(move || { + entered.send(()).unwrap(); + wait.recv_timeout(Duration::from_secs(5)).unwrap(); + Ok(()) + }) + .await + })); + started.await.unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let draining = effects.clone(); + let binding = expected.clone(); + let drain = tokio::spawn(async move { draining.close_and_drain(&binding).await }); + // An active accepted lease remains an owner even though its waiter is gone. + assert_eq!(effects.admission.lock().unwrap().requests, 1); + release.send(()).unwrap(); + let receipt = tokio::time::timeout(Duration::from_secs(2), drain) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(receipt.owner(), &expected); + assert!(effects.admit(&expected).is_err()); + assert!(effects.admit(&owner("attempt-2")).is_err()); + } + + #[tokio::test] + async fn stale_close_cannot_revoke_a_successor_owner_or_create_its_drain_receipt() { + let current = owner("attempt-2"); + let effects = JobHostEffects::initially_admit(current.clone()); + assert!(effects.close_and_drain(&owner("attempt-1")).await.is_err()); + let request = effects.admit(¤t).unwrap(); + let child = request.clone(); + drop(request); + assert_eq!(effects.admission.lock().unwrap().requests, 1); + drop(child); + let receipt = effects.close_and_drain(¤t).await.unwrap(); + assert_eq!(receipt.owner(), ¤t); + assert!(effects.admit(¤t).is_err()); + } +} diff --git a/crates/workspace-server/src/backend_job_recovery_store.rs b/crates/workspace-server/src/backend_job_recovery_store.rs new file mode 100644 index 000000000..373cfc57a --- /dev/null +++ b/crates/workspace-server/src/backend_job_recovery_store.rs @@ -0,0 +1,1360 @@ +//! Persistence for the Server-owned reconciliation of an accepted attempt. +//! This is cause/next-action evidence, not another execution state or a restart +//! authorization. The dispatcher keeps owning Pending until a definitive result. +use super::*; + +pub(super) const SCHEMA: &str = r#" +-- An allocation must win Reserved -> Dispatching before crossing the Runtime +-- boundary. This proof is minted only by the competing Reserved -> Failed Tx. +-- Neither migration nor later terminal/cleanup updates may synthesize it. +CREATE TRIGGER backend_job_allocation_precluded_no_insert +BEFORE INSERT ON backend_job_attempts +WHEN NEW.allocation_precluded_at IS NOT NULL +BEGIN SELECT RAISE(ABORT,'allocation preclusion requires the Reserved transition'); END; +CREATE TRIGGER backend_job_allocation_precluded_transition +BEFORE UPDATE ON backend_job_attempts +WHEN NEW.allocation_precluded_at IS NOT OLD.allocation_precluded_at AND ( + OLD.allocation_precluded_at IS NOT NULL + OR OLD.state != 'reserved' OR NEW.state != 'failed' + OR OLD.runtime_id IS NOT NULL OR OLD.worker_id IS NOT NULL + OR OLD.runtime_run_id IS NOT NULL OR OLD.dispatched_at IS NOT NULL + OR NEW.runtime_id IS NOT NULL OR NEW.worker_id IS NOT NULL + OR NEW.runtime_run_id IS NOT NULL OR NEW.dispatched_at IS NOT NULL + OR NEW.allocation_precluded_at IS NOT NEW.completed_at + OR NEW.failure_category IS NULL OR length(trim(NEW.failure_category))=0 +) +BEGIN SELECT RAISE(ABORT,'immutable pre-allocation rejection proof'); END; +CREATE TRIGGER backend_job_allocation_precluded_no_rebind +BEFORE UPDATE ON backend_job_attempts +WHEN OLD.allocation_precluded_at IS NOT NULL AND ( + NEW.state != 'failed' OR NEW.runtime_id IS NOT NULL OR NEW.worker_id IS NOT NULL + OR NEW.runtime_run_id IS NOT NULL OR NEW.dispatched_at IS NOT NULL + OR NEW.completed_at IS NOT OLD.completed_at +) +BEGIN SELECT RAISE(ABORT,'allocation was conclusively precluded'); END; +CREATE TABLE backend_job_reconciliation ( + workspace_id TEXT NOT NULL, + job_id TEXT NOT NULL, + attempt_id TEXT NOT NULL, + cause TEXT NOT NULL CHECK(length(cause)>0), + detail TEXT NOT NULL, + next_action TEXT NOT NULL CHECK(length(next_action)>0), + updated_at TEXT NOT NULL, + PRIMARY KEY(workspace_id,job_id,attempt_id), + FOREIGN KEY(workspace_id,job_id,attempt_id) REFERENCES backend_job_attempts(workspace_id,job_id,attempt_id) ON DELETE CASCADE +); +CREATE TRIGGER backend_job_reconciliation_no_legacy +BEFORE INSERT ON backend_job_reconciliation +WHEN EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=NEW.workspace_id AND h.job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT,'pre-upgrade Backend Job cannot reconcile'); END; +CREATE TABLE backend_job_execution_fences ( + workspace_id TEXT NOT NULL, + job_id TEXT NOT NULL, + attempt_id TEXT NOT NULL, + owner_json TEXT NOT NULL CHECK(json_valid(owner_json)), + history_json TEXT NOT NULL CHECK(json_valid(history_json)), + controller_json TEXT NOT NULL CHECK(json_valid(controller_json)), + host_json TEXT NOT NULL CHECK(json_valid(host_json)), + recorded_at TEXT NOT NULL, + PRIMARY KEY(workspace_id,job_id,attempt_id), + FOREIGN KEY(workspace_id,job_id,attempt_id) REFERENCES backend_job_attempts(workspace_id,job_id,attempt_id) ON DELETE CASCADE +); +CREATE TRIGGER backend_job_execution_fence_immutable +BEFORE UPDATE ON backend_job_execution_fences +BEGIN SELECT RAISE(ABORT,'immutable exact execution fence'); END; +CREATE TABLE backend_job_recovery_probes ( + workspace_id TEXT NOT NULL, + job_id TEXT NOT NULL, + attempt_id TEXT NOT NULL, + successful_writes INTEGER NOT NULL CHECK(successful_writes>0), + PRIMARY KEY(workspace_id,job_id,attempt_id), + FOREIGN KEY(workspace_id,job_id,attempt_id) REFERENCES backend_job_attempts(workspace_id,job_id,attempt_id) ON DELETE CASCADE +); +"#; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackendJobReconciliation { + pub cause: String, + pub detail: String, + pub next_action: String, + pub updated_at: String, +} + +pub(super) fn require_current_contract( + conn: &Connection, + workspace: &str, + job: &str, +) -> Result<()> { + let legacy: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM backend_job_upgrade_history WHERE workspace_id=?1 AND job_id=?2)", params![workspace,job], |row|row.get(0))?; + if legacy { + return Err(Error::InvalidInput( + "pre-upgrade Backend Job cannot recover, retry or continue".into(), + )); + } + Ok(()) +} + +pub(super) fn history( + conn: &Connection, + workspace: &str, + job: &str, +) -> Result> { + let ids = conn.prepare("SELECT attempt_id FROM backend_job_attempts WHERE workspace_id=?1 AND job_id=?2 ORDER BY attempt,attempt_id")? + .query_map(params![workspace,job], |row|row.get::<_,String>(0))?.collect::>>()?; + ids.into_iter() + .map(|id| { + read_backend_job_attempt(conn, workspace, job, &id)? + .ok_or_else(|| Error::Store("Backend Job history disappeared".into())) + }) + .collect() +} + +pub(super) fn require_history( + conn: &Connection, + workspace: &str, + expected: &[BackendJobAttemptRecord], + owner: &BackendJobAttemptRecord, +) -> Result<()> { + if owner.workspace_id != workspace + || !expected.contains(owner) + || history(conn, workspace, &owner.job_id)? != expected + { + return Err(Error::InvalidInput( + "Backend Job full attempt history or exact execution owner changed".into(), + )); + } + Ok(()) +} + +pub(super) fn record( + store: &SqliteWorkspaceStore, + workspace: &str, + expected: &BackendJobAttemptRecord, + cause: &str, + detail: &str, + next_action: &str, + now: &str, +) -> Result { + if cause.trim().is_empty() + || next_action.trim().is_empty() + || cause.len() > 128 + || next_action.len() > 512 + { + return Err(Error::InvalidInput( + "reconciliation requires a bounded cause and next action".into(), + )); + } + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + require_current_contract(&tx,workspace,&expected.job_id)?; + let job = read_backend_job(&tx,workspace,&expected.job_id)?.ok_or_else(||Error::InvalidInput("Backend Job missing".into()))?; + let current = read_backend_job_attempt(&tx,workspace,&expected.job_id,&expected.attempt_id)?.ok_or_else(||Error::InvalidInput("Backend Job attempt missing".into()))?; + if workspace != expected.workspace_id || current != *expected || job.current_attempt != expected.attempt || job.input_digest != expected.input_digest || job.state != BackendJobState::Pending || expected.state.terminal() { + return Err(Error::InvalidInput("Backend Job reconciliation owner changed".into())); + } + subject_deletion_store::require_job_admission(&tx,workspace,&job.request)?; + let detail = bounded_failure_detail(detail); + tx.execute("INSERT INTO backend_job_reconciliation(workspace_id,job_id,attempt_id,cause,detail,next_action,updated_at) VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(workspace_id,job_id,attempt_id) DO UPDATE SET cause=excluded.cause,detail=excluded.detail,next_action=excluded.next_action,updated_at=excluded.updated_at", params![workspace,expected.job_id,expected.attempt_id,cause,detail,next_action,now])?; + // Do not change the attempt's last confirmed admission/dispatch state. + tx.execute("UPDATE backend_job_attempts SET failure_category=?4,failure_detail=?5,updated_at=?6 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3",params![workspace,expected.job_id,expected.attempt_id,cause,detail,now])?; + tx.execute("UPDATE backend_jobs SET failure_category=?3,failure_detail=?4,updated_at=?5 WHERE workspace_id=?1 AND job_id=?2",params![workspace,expected.job_id,cause,detail,now])?; + let result = read_backend_job_attempt(&tx,workspace,&expected.job_id,&expected.attempt_id)?.ok_or_else(||Error::Store("Backend Job reconciliation attempt disappeared".into()))?; + tx.commit()?; + Ok(result) + }) +} + +pub(super) fn claim_cleanup_owned( + store: &SqliteWorkspaceStore, + workspace: &str, + expected_history: &[BackendJobAttemptRecord], + expected: &BackendJobAttemptRecord, + now: &str, +) -> Result<(BackendJobAttemptRecord, bool)> { + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + require_current_contract(&tx,workspace,&expected.job_id)?; + require_history(&tx,workspace,expected_history,expected)?; + if expected.worker.is_none() || expected.runtime_run_id.is_none() { + return Err(Error::InvalidInput("cleanup requires the exact allocated Worker/run owner".into())); + } + if matches!(expected.worker_cleanup_state,Some(BackendJobWorkerCleanupState::Executing | BackendJobWorkerCleanupState::Completed)) { + tx.commit()?; + return Ok((expected.clone(),false)); + } + tx.execute("UPDATE backend_job_attempts SET worker_cleanup_state='executing',worker_cleanup_failure_category=NULL,worker_cleanup_failure_detail=NULL,worker_cleanup_updated_at=?4 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3",params![workspace,expected.job_id,expected.attempt_id,now])?; + let result=read_backend_job_attempt(&tx,workspace,&expected.job_id,&expected.attempt_id)?.ok_or_else(||Error::Store("cleanup owner disappeared".into()))?; + tx.commit()?; + Ok((result,true)) + }) +} + +pub(super) fn finish_cleanup_owned( + store: &SqliteWorkspaceStore, + workspace: &str, + expected_history: &[BackendJobAttemptRecord], + expected: &BackendJobAttemptRecord, + state: BackendJobWorkerCleanupState, + cause: Option<&str>, + detail: Option<&str>, + now: &str, +) -> Result { + if !matches!( + state, + BackendJobWorkerCleanupState::Completed | BackendJobWorkerCleanupState::Failed + ) { + return Err(Error::InvalidInput( + "cleanup result must be definitive".into(), + )); + } + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + require_current_contract(&tx,workspace,&expected.job_id)?; + require_history(&tx,workspace,expected_history,expected)?; + if expected.worker.is_none() || expected.runtime_run_id.is_none() || expected.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Executing) { + return Err(Error::InvalidInput("cleanup requires an executing exact Worker/run owner".into())); + } + // Caller supplies evidence from the execution owner. A terminal Job or + // missing live map alone must never call this completion boundary. + tx.execute("UPDATE backend_job_attempts SET worker_cleanup_state=?4,worker_cleanup_failure_category=?5,worker_cleanup_failure_detail=?6,worker_cleanup_updated_at=?7,worker_cleanup_completed_at=?8 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3",params![workspace,expected.job_id,expected.attempt_id,state.as_str(),cause,detail.map(bounded_failure_detail),now,(state==BackendJobWorkerCleanupState::Completed).then_some(now)])?; + let result=read_backend_job_attempt(&tx,workspace,&expected.job_id,&expected.attempt_id)?.ok_or_else(||Error::Store("cleanup owner disappeared".into()))?; + tx.commit()?; + Ok(result) + }) +} + +/// Snapshot outside the write transaction, then compare every attempt under the +/// SQLite writer lock. No diagnosis/result is rewritten merely to test recovery. +pub(super) fn probe( + store: &SqliteWorkspaceStore, + workspace: &str, + expected: &BackendJobAttemptRecord, +) -> Result { + let observed = store.with_conn(|conn| history(conn, workspace, &expected.job_id))?; + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + require_current_contract(&tx, workspace, &expected.job_id)?; + require_history(&tx, workspace, &observed, expected)?; + require_retained_owner(&tx, workspace, expected)?; + tx.execute("INSERT INTO backend_job_recovery_probes(workspace_id,job_id,attempt_id,successful_writes) VALUES(?1,?2,?3,1) ON CONFLICT(workspace_id,job_id,attempt_id) DO UPDATE SET successful_writes=successful_writes+1",params![workspace,expected.job_id,expected.attempt_id])?; + tx.commit()?; + Ok(true) + }) +} + +fn require_retained_owner( + conn: &Connection, + workspace: &str, + expected: &BackendJobAttemptRecord, +) -> Result { + let job = read_backend_job(conn, workspace, &expected.job_id)? + .ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?; + let attempt = + read_backend_job_attempt(conn, workspace, &expected.job_id, &expected.attempt_id)?; + if workspace != expected.workspace_id + || attempt.as_ref() != Some(expected) + || job.input_digest != expected.input_digest + { + return Err(Error::RepositoryConflict( + "Backend Job exact retained owner changed".into(), + )); + } + Ok(job) +} + +fn require_current_owner( + conn: &Connection, + workspace: &str, + expected: &BackendJobAttemptRecord, +) -> Result { + let job = require_retained_owner(conn, workspace, expected)?; + if job.current_attempt != expected.attempt { + return Err(Error::RepositoryConflict( + "Backend Job current owner changed".into(), + )); + } + job.load_executable_request()?; + Ok(job) +} + +fn validate_fence( + expected: &BackendJobAttemptRecord, + controller: &str, + host: &str, +) -> Result { + use crate::backend_job_host_effects::JobHostOwner; + use worker_runtime::execution::{WorkerExecutionTermination, WorkerExecutionTerminationScope}; + if controller.len() > 65536 || host.len() > 8192 { + return Err(Error::InvalidInput( + "execution evidence exceeds bound".into(), + )); + } + let owner = JobHostOwner::try_from(expected)?; + let host: JobHostOwner = serde_json::from_str(host) + .map_err(|_| Error::InvalidInput("typed Host drain evidence required".into()))?; + let ended: WorkerExecutionTermination = serde_json::from_str(controller).map_err(|_| { + Error::InvalidInput("typed Controller termination evidence required".into()) + })?; + if host != owner + || ended.scope != WorkerExecutionTerminationScope::Controller + || !ended.feature_failures.is_empty() + || ended.execution_id.is_nil() + || ended.shutdown_command_id.is_none() + || ended.worker_ref.worker_id.to_string() != owner.worker.worker_id + { + return Err(Error::InvalidInput("Controller/Feature/background and Host drain must confirm the exact owner independently".into())); + } + Ok(owner) +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct ExecutionBinding { + attempt_id: String, + attempt: u8, + input_digest: String, + worker: Option, + runtime_run_id: Option, +} + +impl From<&BackendJobAttemptRecord> for ExecutionBinding { + fn from(value: &BackendJobAttemptRecord) -> Self { + Self { + attempt_id: value.attempt_id.clone(), + attempt: value.attempt, + input_digest: value.input_digest.clone(), + worker: value.worker.clone(), + runtime_run_id: value.runtime_run_id.clone(), + } + } +} + +pub(super) fn record_fence( + store: &SqliteWorkspaceStore, + workspace: &str, + expected: &BackendJobAttemptRecord, + controller: &str, + host: &str, + now: &str, +) -> Result { + let owner = validate_fence(expected, controller, host)?; + let observed = store.with_conn(|conn| history(conn, workspace, &expected.job_id))?; + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + require_current_contract(&tx, workspace, &expected.job_id)?; + require_history(&tx, workspace, &observed, expected)?; + require_retained_owner(&tx, workspace, expected)?; + // Strings are transport, not truthy markers. Only the Host's private + // drain constructor may supply host evidence; this checks its binding. + let saved: Option = tx.query_row("SELECT owner_json FROM backend_job_execution_fences WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3",params![workspace,expected.job_id,expected.attempt_id],|row|row.get(0)).optional()?; + let owner_json = serde_json::to_string(&owner).map_err(|error|Error::Store(error.to_string()))?; + if let Some(saved) = saved { + if saved != owner_json { return Err(Error::RepositoryConflict("execution fence owner changed".into())); } + } else { + // Persist bindings, not a second copy of domain results/causes. The + // full observed rows were compared under the same writer lock above. + let bindings: Vec<_> = observed.iter().map(ExecutionBinding::from).collect(); + let history_json = serde_json::to_string(&bindings).map_err(|error|Error::Store(error.to_string()))?; + tx.execute("INSERT INTO backend_job_execution_fences(workspace_id,job_id,attempt_id,owner_json,history_json,controller_json,host_json,recorded_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8)",params![workspace,expected.job_id,expected.attempt_id,owner_json,history_json,controller,host,now])?; + } + tx.commit()?; + Ok(expected.clone()) + }) +} + +pub(super) fn has_fence( + conn: &Connection, + workspace: &str, + expected: &BackendJobAttemptRecord, +) -> Result { + require_current_contract(conn, workspace, &expected.job_id)?; + let current = + read_backend_job_attempt(conn, workspace, &expected.job_id, &expected.attempt_id)?; + if workspace != expected.workspace_id || current.as_ref() != Some(expected) { + return Ok(false); + } + // This is a different fact from an execution termination receipt. The same + // SQLite transaction conclusively rejected admission before allocation; the + // schema guards forbid later rebinding or backfilling it from a terminal row. + if expected.allocation_precluded_at.is_some() { + if expected.state != BackendJobAttemptState::Failed + || expected.worker.is_some() + || expected.runtime_run_id.is_some() + || expected.dispatched_at.is_some() + || expected.allocation_precluded_at != expected.completed_at + { + return Err(Error::Store( + "invalid pre-allocation rejection proof".into(), + )); + } + return Ok(true); + } + let saved: Option<(String,String,String)> = conn.query_row("SELECT controller_json,host_json,history_json FROM backend_job_execution_fences WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3",params![workspace,expected.job_id,expected.attempt_id],|row|Ok((row.get(0)?,row.get(1)?,row.get(2)?))).optional()?; + let Some((controller, host, bindings)) = saved else { + return Ok(false); + }; + let bindings: Vec = serde_json::from_str(&bindings) + .map_err(|_| Error::Store("invalid execution fence history".into()))?; + let current = history(conn, workspace, &expected.job_id)?; + // Later owners do not inherit this fence. Earlier binding changes invalidate + // its full-history evidence, while ordinary result/cleanup progress does not. + if bindings.is_empty() + || !bindings.iter().all(|binding| { + current + .iter() + .any(|attempt| ExecutionBinding::from(attempt) == *binding) + }) + { + return Ok(false); + } + validate_fence(expected, &controller, &host).map(|_| true) +} + +pub(super) fn reconcile_domain_result( + store: &SqliteWorkspaceStore, + workspace: &str, + expected: &BackendJobAttemptRecord, + submission: &BackendJobResultSubmission, + now: &str, +) -> Result { + let observed = store.with_conn(|conn| history(conn, workspace, &expected.job_id))?; + store.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + require_current_contract(&tx, workspace, &expected.job_id)?; + require_history(&tx, workspace, &observed, expected)?; + let job = require_current_owner(&tx, workspace, expected)?; + subject_deletion_store::require_job_admission(&tx,workspace,&job.request)?; + if job.request.grants.subjektiv_journal.is_some() || job.state != BackendJobState::Pending + || expected.state.terminal() || expected.worker.is_none() || expected.runtime_run_id.is_none() + || submission.job_id != expected.job_id || submission.attempt_id != expected.attempt_id + || submission.input_digest != expected.input_digest { + return Err(Error::InvalidInput("saved non-Journal domain result requires a nonterminal exact current execution owner".into())); + } + let (digest,json) = result_digest(&submission.result,job.request.limits.max_result_bytes)?; + tx.execute("UPDATE backend_job_attempts SET state='completed',result_json=?4,result_digest=?5,failure_category=NULL,failure_detail=NULL,worker_cleanup_state=COALESCE(worker_cleanup_state,'pending'),worker_cleanup_updated_at=CASE WHEN worker_cleanup_state IS NULL THEN ?6 ELSE worker_cleanup_updated_at END,updated_at=?6,completed_at=?6 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3",params![workspace,expected.job_id,expected.attempt_id,json,digest,now])?; + tx.execute("UPDATE backend_jobs SET state='completed',result_json=?3,result_digest=?4,failure_category=NULL,failure_detail=NULL,updated_at=?5,completed_at=?5 WHERE workspace_id=?1 AND job_id=?2",params![workspace,expected.job_id,json,digest,now])?; + notification_routing::jobs::capture(&tx,workspace,&expected.job_id,&expected.attempt_id)?; + let result = read_backend_job_attempt(&tx,workspace,&expected.job_id,&expected.attempt_id)?.ok_or_else(||Error::Store("reconciled owner disappeared".into()))?; + tx.commit()?; + Ok(result) + }) +} + +#[cfg(test)] +mod tests { + //! Real Store/SQLite component tests. Only Runtime allocation/termination is + //! outside this boundary: saved owner bindings are fixture rows, not proof + //! that a process, Controller, background action or Host operation has ended. + use super::*; + + const NOW: &str = "2026-11-04T00:00:00Z"; + + fn fixture() -> ( + tempfile::TempDir, + SqliteWorkspaceStore, + BackendJobReservation, + ) { + let dir = tempfile::tempdir().unwrap(); + let store = + SqliteWorkspaceStore::open_for_test(&dir.path().join("recovery.sqlite")).unwrap(); + store.with_conn(|conn| { + conn.execute_batch("INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) VALUES('owner','user','owner','Owner','1','1'); INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) VALUES('workspace','owner','Workspace','active','1','1');")?; + conn.execute("INSERT INTO workspace_memory_settings(workspace_id,language,created_at,updated_at) VALUES('workspace','en',?1,?1)",params![NOW])?; + Ok(()) + }).unwrap(); + let request = BackendJobRequest { + job_id: "accepted".into(), + purpose: "recovery-test".into(), + input_ref: "saved-input".into(), + input: serde_json::json!({"body":"frozen"}), + instruction: "Use saved input".into(), + profile: "builtin:backend-job".into(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits { + max_attempts: 3, + ..Default::default() + }, + }; + let reserved = store + .reserve_backend_job("workspace", &request, NOW) + .unwrap(); + (dir, store, reserved) + } + + #[test] + fn preallocation_rejection_commits_with_failure_and_cannot_be_backfilled_or_rebound() { + let (_dir, store, reserved) = fixture(); + let id = &reserved.attempt.attempt_id; + store.with_conn(|conn| { + conn.execute_batch("CREATE TRIGGER refuse_failure BEFORE UPDATE ON backend_jobs WHEN NEW.state='failed' BEGIN SELECT RAISE(ABORT,'injected failure commit'); END;")?; + Ok(()) + }).unwrap(); + assert!( + store + .finish_backend_job_attempt( + "workspace", + "accepted", + id, + BackendJobAttemptState::Failed, + "cancelled_before_dispatch", + "No execution admitted", + NOW + ) + .is_err() + ); + let rolled_back = store + .get_backend_job_attempt("workspace", "accepted", id) + .unwrap() + .unwrap(); + assert_eq!(rolled_back, reserved.attempt); + assert!( + !store + .has_backend_job_execution_fence("workspace", &rolled_back) + .unwrap() + ); + store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER refuse_failure")?; + Ok(()) + }) + .unwrap(); + let failed = store + .finish_backend_job_attempt( + "workspace", + "accepted", + id, + BackendJobAttemptState::Failed, + "cancelled_before_dispatch", + "No execution admitted", + NOW, + ) + .unwrap(); + assert_eq!(failed.allocation_precluded_at.as_deref(), Some(NOW)); + assert!( + store + .has_backend_job_execution_fence("workspace", &failed) + .unwrap() + ); + let (replay, acquired) = store + .claim_backend_job_attempt_dispatch("workspace", "accepted", id, NOW) + .unwrap(); + assert!(!acquired); + assert_eq!(replay.attempt, failed); + store.with_conn(|conn| { + assert!(conn.execute("UPDATE backend_job_attempts SET state='dispatching',worker_id='foreign',runtime_id='foreign' WHERE attempt_id=?1", [id]).is_err()); + assert!(conn.execute("UPDATE backend_job_attempts SET allocation_precluded_at=NULL WHERE attempt_id=?1", [id]).is_err()); + Ok(()) + }).unwrap(); + let retry = store + .reserve_backend_job_retry("workspace", "accepted", NOW) + .unwrap(); + assert!(retry.attempt.allocation_precluded_at.is_none()); + assert!( + !store + .has_backend_job_execution_fence("workspace", &retry.attempt) + .unwrap() + ); + assert!( + store + .has_backend_job_execution_fence("workspace", &failed) + .unwrap() + ); + + // Dispatch claimed allocation admission, even though no Worker is bound yet. + let (claimed, acquired) = store + .claim_backend_job_attempt_dispatch( + "workspace", + "accepted", + &retry.attempt.attempt_id, + NOW, + ) + .unwrap(); + assert!(acquired); + let uncertain = store + .finish_backend_job_attempt( + "workspace", + "accepted", + &claimed.attempt.attempt_id, + BackendJobAttemptState::Failed, + "allocation_response_lost", + "Must reconcile the original allocation", + NOW, + ) + .unwrap(); + assert!(uncertain.worker.is_none()); + assert!(uncertain.allocation_precluded_at.is_none()); + assert!( + !store + .has_backend_job_execution_fence("workspace", &uncertain) + .unwrap() + ); + store.with_conn(|conn| { + assert!(conn.execute("UPDATE backend_job_attempts SET allocation_precluded_at=completed_at WHERE attempt_id=?1", [&uncertain.attempt_id]).is_err()); + Ok(()) + }).unwrap(); + } + + #[test] + fn concurrent_dispatch_claim_and_preallocation_rejection_serialize_on_the_same_attempt() { + let (dir, store, reserved) = fixture(); + let other = + SqliteWorkspaceStore::open_for_test(&dir.path().join("recovery.sqlite")).unwrap(); + let start = std::sync::Barrier::new(2); + let id = &reserved.attempt.attempt_id; + let (claim, rejected) = std::thread::scope(|scope| { + let dispatch = scope.spawn(|| { + start.wait(); + store + .claim_backend_job_attempt_dispatch("workspace", "accepted", id, NOW) + .unwrap() + }); + let reject = scope.spawn(|| { + start.wait(); + other + .finish_backend_job_attempt( + "workspace", + "accepted", + id, + BackendJobAttemptState::Failed, + "cancelled", + "Concurrent admission rejection", + NOW, + ) + .unwrap() + }); + (dispatch.join().unwrap(), reject.join().unwrap()) + }); + let saved = store + .get_backend_job_attempt("workspace", "accepted", id) + .unwrap() + .unwrap(); + assert_eq!(saved, rejected); + assert_eq!(saved.state, BackendJobAttemptState::Failed); + assert_eq!(saved.allocation_precluded_at.is_some(), !claim.1); + assert_eq!( + store + .has_backend_job_execution_fence("workspace", &saved) + .unwrap(), + !claim.1 + ); + if claim.1 { + assert_eq!(claim.0.attempt.state, BackendJobAttemptState::Dispatching); + } else { + assert_eq!(claim.0.attempt, rejected); + } + } + + #[test] + fn allocation_lookup_is_read_only_matches_create_fingerprint_and_never_resurrects_removed_worker() + { + let (_dir, store, reserved) = fixture(); + let settings = WorkspaceMemorySettingsRecord { + workspace_id: "workspace".into(), + language: "en".into(), + created_at: NOW.into(), + updated_at: NOW.into(), + }; + let key = crate::backend_job::allocation_key(&reserved.attempt.attempt_id); + assert!( + store + .lookup_backend_job_allocation("workspace", &reserved.job, &reserved.attempt) + .unwrap() + .is_none() + ); + let created = store + .reserve_worker_create( + "workspace", + "runtime", + &key, + "sha256:create-input", + None, + &settings, + ) + .unwrap(); + let found = store + .lookup_backend_job_allocation("workspace", &reserved.job, &reserved.attempt) + .unwrap() + .unwrap(); + assert_eq!(found, (created.clone(), "runtime".to_string())); + store.with_conn(|conn| { + conn.execute("UPDATE worker_create_reservations SET create_fingerprint='corrupt' WHERE allocation_key=?1", params![key])?; + Ok(()) + }).unwrap(); + assert!( + store + .lookup_backend_job_allocation("workspace", &reserved.job, &reserved.attempt) + .is_err() + ); + store.with_conn(|conn| { + conn.execute("UPDATE worker_create_reservations SET create_fingerprint=?2 WHERE allocation_key=?1", params![key,created.create_fingerprint])?; + Ok(()) + }).unwrap(); + let mut wrong = reserved.job.clone(); + wrong.input_digest = "different-input".into(); + assert!( + store + .lookup_backend_job_allocation("workspace", &wrong, &reserved.attempt) + .is_err() + ); + store + .with_conn(|conn| { + conn.execute( + "UPDATE worker_create_reservations SET state='removed' WHERE allocation_key=?1", + params![key], + )?; + Ok(()) + }) + .unwrap(); + assert!( + store + .lookup_backend_job_allocation("workspace", &reserved.job, &reserved.attempt) + .is_err() + ); + store + .with_conn(|conn| { + assert_eq!( + conn.query_row( + "SELECT count(*) FROM worker_create_reservations", + [], + |row| row.get::<_, i64>(0) + )?, + 1 + ); + let state: String = conn.query_row( + "SELECT state FROM worker_create_reservations WHERE allocation_key=?1", + params![key], + |row| row.get(0), + )?; + assert_eq!(state, "removed"); + Ok(()) + }) + .unwrap(); + } + + #[test] + fn terminal_executing_cleanup_probe_commits_without_changing_failure_and_rejects_stale_owner() { + let (dir, store, reserved) = fixture(); + let owner = bind_fixture_owner(&store, &reserved); + let failed = store + .finish_backend_job_attempt( + "workspace", + "accepted", + &owner.attempt_id, + BackendJobAttemptState::Failed, + "execution_failed", + "original cause", + NOW, + ) + .unwrap(); + let history = store + .list_backend_job_attempts("workspace", "accepted") + .unwrap(); + let (executing, claimed) = store + .claim_backend_job_worker_cleanup_owned("workspace", &history, &failed, NOW) + .unwrap(); + assert!(claimed); + let job = store + .get_backend_job("workspace", "accepted") + .unwrap() + .unwrap(); + assert!( + store + .probe_backend_job_recovery("workspace", &failed) + .is_err() + ); + store.with_conn(|conn| { + conn.execute_batch("CREATE TEMP TRIGGER probe_fault BEFORE INSERT ON main.backend_job_recovery_probes BEGIN SELECT RAISE(ABORT,'same SQLite fault'); END;")?; + Ok(()) + }).unwrap(); + assert!( + store + .probe_backend_job_recovery("workspace", &executing) + .is_err() + ); + store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER probe_fault")?; + Ok(()) + }) + .unwrap(); + assert!( + store + .probe_backend_job_recovery("workspace", &executing) + .unwrap() + ); + assert!( + store + .probe_backend_job_recovery("workspace", &executing) + .unwrap() + ); + assert_eq!( + store + .get_backend_job("workspace", "accepted") + .unwrap() + .unwrap(), + job + ); + assert_eq!( + store + .get_backend_job_attempt("workspace", "accepted", &executing.attempt_id) + .unwrap() + .unwrap(), + executing + ); + drop(store); + let store = + SqliteWorkspaceStore::open_for_test(&dir.path().join("recovery.sqlite")).unwrap(); + store + .with_conn(|conn| { + let writes: i64 = conn.query_row( + "SELECT successful_writes FROM backend_job_recovery_probes", + [], + |row| row.get(0), + )?; + assert_eq!(writes, 2); + Ok(()) + }) + .unwrap(); + assert!( + !store + .has_backend_job_execution_fence("workspace", &executing) + .unwrap() + ); + } + + #[test] + fn historical_probe_and_fence_do_not_reset_cleanup_or_release_a_successor() { + let (_dir, store, reserved) = fixture(); + let bound = bind_fixture_owner(&store, &reserved); + store.with_conn(|conn| { + conn.execute("UPDATE backend_job_attempts SET worker_id='019a4ea8-0000-7000-8000-000000000003' WHERE attempt_id=?1",params![bound.attempt_id])?; + Ok(()) + }).unwrap(); + let failed = store + .finish_backend_job_attempt( + "workspace", + "accepted", + &bound.attempt_id, + BackendJobAttemptState::Failed, + "execution_failed", + "saved original failure", + NOW, + ) + .unwrap(); + let history = store + .list_backend_job_attempts("workspace", "accepted") + .unwrap(); + let (executing, acquired) = store + .claim_backend_job_worker_cleanup_owned("workspace", &history, &failed, NOW) + .unwrap(); + assert!(acquired); + assert_eq!( + store + .recover_backend_job_worker_cleanups("workspace", "later", 100) + .unwrap(), + vec![executing.clone()] + ); + assert_eq!( + store + .get_backend_job_attempt("workspace", "accepted", &executing.attempt_id) + .unwrap() + .unwrap(), + executing + ); + assert!( + store + .reserve_backend_job_retry("workspace", "accepted", NOW) + .is_err() + ); + // The fixture substitutes only the external cleanup receipt; this does + // not claim actual Controller/background/Host/process termination. + let history = store + .list_backend_job_attempts("workspace", "accepted") + .unwrap(); + let historical = store + .finish_backend_job_worker_cleanup_owned( + "workspace", + &history, + &executing, + BackendJobWorkerCleanupState::Completed, + None, + None, + NOW, + ) + .unwrap(); + let next = store + .reserve_backend_job_retry("workspace", "accepted", NOW) + .unwrap(); + let successor = bind_fixture_owner(&store, &next); + let next_job = store + .get_backend_job("workspace", "accepted") + .unwrap() + .unwrap(); + assert!( + store + .probe_backend_job_recovery("workspace", &historical) + .unwrap() + ); + let (controller, host) = evidence(&historical); + store + .record_backend_job_execution_fence("workspace", &historical, &controller, &host, NOW) + .unwrap(); + assert!( + store + .has_backend_job_execution_fence("workspace", &historical) + .unwrap() + ); + assert!( + !store + .has_backend_job_execution_fence("workspace", &successor) + .unwrap() + ); + assert_eq!( + store + .get_backend_job_attempt("workspace", "accepted", &successor.attempt_id) + .unwrap() + .unwrap(), + successor + ); + assert_eq!( + store + .get_backend_job("workspace", "accepted") + .unwrap() + .unwrap(), + next_job + ); + let mut foreign = historical.clone(); + foreign.worker = successor.worker.clone(); + assert!( + store + .probe_backend_job_recovery("workspace", &foreign) + .is_err() + ); + assert!( + store + .record_backend_job_execution_fence("workspace", &foreign, &controller, &host, NOW) + .is_err() + ); + } + + #[test] + fn completed_journal_with_available_input_is_rediscovered_without_claiming_recording_complete() + { + let (_dir, store, reserved) = fixture(); + let owner = bind_fixture_owner(&store, &reserved); + // Selection boundary only; SQL cleanup/result rows are not external + // execution, Journal checkpoint, or Purge receipts. + store.with_conn(|conn| { + conn.execute("UPDATE backend_job_attempts SET state='completed',result_json='{}',result_digest='fixture-result',worker_cleanup_state='completed' WHERE attempt_id=?1", params![owner.attempt_id])?; + conn.execute("UPDATE backend_jobs SET state='completed',result_json='{}',result_digest='fixture-result' WHERE job_id='accepted'", [])?; + Ok(()) + }).unwrap(); + assert!( + store + .list_backend_job_worker_cleanups("workspace", 100) + .unwrap() + .is_empty() + ); + let mut request = reserved.job.request.clone(); + request.job_id = "journal-retirement".into(); + request.serialization_key = Some(crate::backend_job::journal_serialization_key("journal")); + request.grants.subjektiv_journal = Some(crate::backend_job::SubjektivJournalGrant { + binding_id: "binding".into(), subject_id: "subject".into(), session_id: "session".into(), journal_id: "journal".into(), + selected_ranges: vec![serde_json::from_value(serde_json::json!({ + "source":{"session_id":"session", "pointer":{"segment_id":"019a4ea8-0000-7000-8000-000000000001", "entry_ref":"Eentry"}, + "order":0,"content_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "total_bytes":1}, + "bytes":{"start":0,"end":1} + })).unwrap()], + }); + let journal = store + .reserve_backend_job("workspace", &request, NOW) + .unwrap(); + store.with_conn(|conn| { + conn.execute("UPDATE backend_job_attempts SET state='completed',runtime_id='runtime',worker_id='journal-worker',runtime_run_id='run',result_json='{}',result_digest='fixture-result',worker_cleanup_state='completed' WHERE attempt_id=?1", params![journal.attempt.attempt_id])?; + conn.execute("UPDATE backend_jobs SET state='completed',result_json='{}',result_digest='fixture-result' WHERE job_id=?1", params![request.job_id])?; + Ok(()) + }).unwrap(); + let selected = store + .recover_backend_job_worker_cleanups("workspace", NOW, 100) + .unwrap(); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].attempt_id, journal.attempt.attempt_id); + assert_eq!( + selected[0].worker_cleanup_state, + Some(BackendJobWorkerCleanupState::Completed) + ); + assert!( + !store + .has_backend_job_execution_fence("workspace", &selected[0]) + .unwrap() + ); + store + .reclaim_backend_job_input( + "workspace", + &request.job_id, + &journal.job.input_digest, + crate::backend_job::JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert!( + store + .recover_backend_job_worker_cleanups("workspace", NOW, 100) + .unwrap() + .is_empty() + ); + } + + #[test] + fn saved_nonjournal_domain_completion_full_cas_preserves_independent_cleanup() { + let (_dir, store, reserved) = fixture(); + let owner = bind_fixture_owner(&store, &reserved); + let submission = BackendJobResultSubmission { + job_id: owner.job_id.clone(), + attempt_id: owner.attempt_id.clone(), + input_digest: owner.input_digest.clone(), + result: serde_json::json!({"domain":"saved-disposition"}), + }; + let mut wrong = submission.clone(); + wrong.input_digest = "different-input".into(); + assert!( + store + .reconcile_backend_job_domain_result("workspace", &owner, &wrong, NOW) + .is_err() + ); + let mut oversized = submission.clone(); + oversized.result = serde_json::Value::String("x".repeat(1024 * 1024)); + assert!( + store + .reconcile_backend_job_domain_result("workspace", &owner, &oversized, NOW) + .is_err() + ); + let done = store + .reconcile_backend_job_domain_result("workspace", &owner, &submission, NOW) + .unwrap(); + assert_eq!(done.state, BackendJobAttemptState::Completed); + assert_eq!(done.result, Some(submission.result.clone())); + assert_eq!( + done.worker_cleanup_state, + Some( + owner + .worker_cleanup_state + .unwrap_or(BackendJobWorkerCleanupState::Pending) + ) + ); + assert_eq!( + done.worker_cleanup_completed_at, + owner.worker_cleanup_completed_at + ); + assert!( + !store + .has_backend_job_execution_fence("workspace", &done) + .unwrap() + ); + assert!( + store + .reconcile_backend_job_domain_result("workspace", &owner, &submission, NOW) + .is_err() + ); + assert!( + store + .reconcile_backend_job_domain_result("workspace", &done, &submission, NOW) + .is_err() + ); + } + + fn evidence(owner: &BackendJobAttemptRecord) -> (String, String) { + let host = crate::backend_job_host_effects::JobHostOwner::try_from(owner).unwrap(); + let controller = serde_json::json!({ + "scope":"controller", "worker_ref":{"worker_id":owner.worker.as_ref().unwrap().worker_id}, + "stop_operation_id":"019a4ea8-0000-7000-8000-000000000001", + "execution_id":"019a4ea8-0000-7000-8000-000000000002", + "shutdown_command_id":1,"submissions":[],"feature_failures":[] + }); + ( + controller.to_string(), + serde_json::to_string(&host).unwrap(), + ) + } + + #[test] + fn fence_requires_typed_independent_exact_owner_evidence_and_survives_only_its_binding() { + let (dir, store, reserved) = fixture(); + let owner = bind_fixture_owner(&store, &reserved); + store.with_conn(|conn| {conn.execute("UPDATE backend_job_attempts SET worker_id='019a4ea8-0000-7000-8000-000000000003' WHERE attempt_id=?1",params![owner.attempt_id])?;Ok(())}).unwrap(); + let owner = store + .get_backend_job_attempt("workspace", "accepted", &owner.attempt_id) + .unwrap() + .unwrap(); + let (controller, host) = evidence(&owner); + assert!( + store + .record_backend_job_execution_fence("workspace", &owner, "true", "true", NOW) + .is_err() + ); + let wrong = host.replace( + "\"runtime_run_id\":\"run\"", + "\"runtime_run_id\":\"successor-run\"", + ); + assert_ne!(wrong, host); + assert!( + store + .record_backend_job_execution_fence("workspace", &owner, &controller, &wrong, NOW) + .is_err() + ); + let bad = controller.replace("controller", "unpublished_startup_candidate"); + assert!( + store + .record_backend_job_execution_fence("workspace", &owner, &bad, &host, NOW) + .is_err() + ); + store + .record_backend_job_execution_fence("workspace", &owner, &controller, &host, NOW) + .unwrap(); + assert!( + store + .has_backend_job_execution_fence("workspace", &owner) + .unwrap() + ); + // A fence is not domain completion or canonical removal. + assert_eq!( + store + .get_backend_job("workspace", "accepted") + .unwrap() + .unwrap() + .state, + BackendJobState::Pending + ); + assert_eq!( + store + .get_backend_job_attempt("workspace", "accepted", &owner.attempt_id) + .unwrap() + .unwrap(), + owner + ); + drop(store); + let store = + SqliteWorkspaceStore::open_for_test(&dir.path().join("recovery.sqlite")).unwrap(); + assert!( + store + .has_backend_job_execution_fence("workspace", &owner) + .unwrap() + ); + store.with_conn(|conn| {conn.execute("UPDATE backend_job_attempts SET runtime_run_id='successor-run' WHERE attempt_id=?1",params![owner.attempt_id])?;Ok(())}).unwrap(); + let successor = store + .get_backend_job_attempt("workspace", "accepted", &owner.attempt_id) + .unwrap() + .unwrap(); + assert!( + !store + .has_backend_job_execution_fence("workspace", &owner) + .unwrap() + ); + assert!( + !store + .has_backend_job_execution_fence("workspace", &successor) + .unwrap() + ); + assert!( + store + .record_backend_job_execution_fence("workspace", &owner, &controller, &host, NOW) + .is_err() + ); + } + + #[test] + fn reconciliation_keeps_the_admission_fact_and_saved_next_action_across_reopen() { + let (dir, store, reserved) = fixture(); + let (claimed, was_claimed) = store + .claim_backend_job_attempt_dispatch( + "workspace", + "accepted", + &reserved.attempt.attempt_id, + NOW, + ) + .unwrap(); + assert!(was_claimed); + let pending = store + .record_backend_job_reconciliation( + "workspace", + &claimed.attempt, + "receipt_read_failed", + "temporary Runtime lookup failure", + "read_same_allocation_receipt", + NOW, + ) + .unwrap(); + assert_eq!(pending.state, BackendJobAttemptState::Dispatching); + assert!( + store + .reserve_backend_job_retry("workspace", "accepted", NOW) + .is_err() + ); + // Stale diagnosis cannot overwrite a later saved owner observation. + assert!( + store + .record_backend_job_reconciliation( + "workspace", + &claimed.attempt, + "late", + "stale lookup", + "do_not_reallocate", + NOW + ) + .is_err() + ); + drop(store); + let store = + SqliteWorkspaceStore::open_for_test(&dir.path().join("recovery.sqlite")).unwrap(); + let recovery = store + .list_recoverable_backend_job_attempts("workspace", 100) + .unwrap(); + assert_eq!(recovery, vec![pending]); + assert_eq!( + store + .get_backend_job("workspace", "accepted") + .unwrap() + .unwrap() + .state, + BackendJobState::Pending + ); + let diagnosis = store + .get_backend_job_reconciliation("workspace", "accepted", &reserved.attempt.attempt_id) + .unwrap() + .unwrap(); + assert_eq!(diagnosis.cause, "receipt_read_failed"); + assert_eq!(diagnosis.next_action, "read_same_allocation_receipt"); + // Restart did not clear the dispatch claim for blind retransmission. + assert!( + !store + .claim_backend_job_attempt_dispatch( + "workspace", + "accepted", + &reserved.attempt.attempt_id, + NOW + ) + .unwrap() + .1 + ); + } + + fn bind_fixture_owner( + store: &SqliteWorkspaceStore, + reservation: &BackendJobReservation, + ) -> BackendJobAttemptRecord { + store.with_conn(|conn| { + conn.execute("UPDATE backend_job_attempts SET state='dispatched',runtime_id='runtime',worker_id='worker',runtime_run_id='run',dispatched_at=?2 WHERE attempt_id=?1",params![reservation.attempt.attempt_id,NOW])?; + Ok(()) + }).unwrap(); + store + .get_backend_job_attempt("workspace", "accepted", &reservation.attempt.attempt_id) + .unwrap() + .unwrap() + } + + #[test] + fn cleanup_cas_rejects_changed_history_and_wrong_worker_run_without_releasing_owner() { + let (_dir, store, reserved) = fixture(); + let owner = bind_fixture_owner(&store, &reserved); + let history = store + .list_backend_job_attempts("workspace", "accepted") + .unwrap(); + let mut wrong = owner.clone(); + wrong.runtime_run_id = Some("later-run".into()); + assert!( + store + .claim_backend_job_worker_cleanup_owned("workspace", &history, &wrong, NOW) + .is_err() + ); + assert!( + store + .claim_backend_job_worker_cleanup_owned("workspace", &[], &owner, NOW) + .is_err() + ); + let (executing, claimed) = store + .claim_backend_job_worker_cleanup_owned("workspace", &history, &owner, NOW) + .unwrap(); + assert!(claimed); + // Even identical Job/attempt IDs cannot erase an intervening cleanup fact. + assert!( + store + .finish_backend_job_worker_cleanup_owned( + "workspace", + &history, + &owner, + BackendJobWorkerCleanupState::Completed, + None, + None, + NOW + ) + .is_err() + ); + let history = store + .list_backend_job_attempts("workspace", "accepted") + .unwrap(); + let done = store + .finish_backend_job_worker_cleanup_owned( + "workspace", + &history, + &executing, + BackendJobWorkerCleanupState::Completed, + None, + None, + NOW, + ) + .unwrap(); + assert_eq!( + done.worker_cleanup_state, + Some(BackendJobWorkerCleanupState::Completed) + ); + // Cleanup and domain completion remain independent facts. + assert_eq!( + store + .get_backend_job("workspace", "accepted") + .unwrap() + .unwrap() + .state, + BackendJobState::Pending + ); + assert!( + store + .reserve_backend_job_retry("workspace", "accepted", NOW) + .is_err() + ); + } + + #[test] + fn diagnosis_and_failure_do_not_turn_an_unbound_dispatch_into_execution_end() { + let (_dir, store, reserved) = fixture(); + store + .claim_backend_job_attempt_dispatch( + "workspace", + "accepted", + &reserved.attempt.attempt_id, + NOW, + ) + .unwrap(); + let failed = store + .finish_backend_job_attempt( + "workspace", + "accepted", + &reserved.attempt.attempt_id, + BackendJobAttemptState::Failed, + "dispatch_refused", + "definitive provider refusal", + NOW, + ) + .unwrap(); + assert_eq!( + failed.worker_cleanup_state, + Some(BackendJobWorkerCleanupState::Pending) + ); + assert!(failed.worker_cleanup_completed_at.is_none()); + assert!( + store + .reserve_backend_job_retry("workspace", "accepted", NOW) + .is_err() + ); + } +} diff --git a/crates/workspace-server/src/backend_job_upgrade.rs b/crates/workspace-server/src/backend_job_upgrade.rs new file mode 100644 index 000000000..336cd8b56 --- /dev/null +++ b/crates/workspace-server/src/backend_job_upgrade.rs @@ -0,0 +1,580 @@ +//! One-shot retirement of Jobs that predate the current execution contract. +//! +//! The Workspace ordered migration owner must validate the retained storage +//! family first, disable foreign keys BEFORE its exclusive transaction, call +//! `install`, and commit the Workspace migration receipt in that same transaction. +//! It must restore foreign keys on both success and failure. This leaf neither +//! owns a version ledger nor runs from ordinary recovery/open after that receipt. +//! +//! A failed Job here means only "do not continue this old request". In particular, +//! no execution/cleanup termination, writer revocation, Journal completion, input +//! disposal or resource release is inferred. Pre-content captures preserve the +//! original cleanup values rather than the legacy cutover's inferred cleanup; +//! external Journal operations/checkpoints are untouched. Consumers must exclude +//! `backend_job_upgrade_history` Jobs from recovery/retry and consult the actual +//! execution/Journal owners before deletion. The guards also reject stale writes. +//! Notification delivery is a separate lifecycle: its saved states, notification +//! IDs and same-ID acceptance reconciliation are deliberately outside this cutover. +use super::{Connection, Error, Result}; + +/// Final history/guards. Fresh composition calls `install_fresh`, which also +/// installs EMPTY raw evidence; this SQL alone is not the full fresh schema. +pub(super) const MIGRATION_NAME: &str = "Backend Job execution retirement and owned reconciliation"; +pub(super) const SCHEMA: &str = include_str!("backend_job_upgrade.sql"); +pub(super) const FAILURE_CATEGORY: &str = "version_upgrade_execution_aborted"; +pub(super) const FAILURE_DETAIL: &str = "バージョン更新による実行打切り。旧Jobの再開・再試行は行わない。このFailedは実行停止・書込み失効・Journal完了・rollback・資源解放の証明ではない。"; + +/// Evidence, not a second migration ledger. The existing schema-86 receipt owns +/// populated captures; fresh creation and later baselines install it EMPTY. +/// Raw request bytes remain historical and cannot authorize execution. Their only +/// allowed mutation is body retirement together with the live Job's owner. +pub(super) const RAW_SCHEMA: &str = r#" +CREATE TABLE backend_job_pre_content_history ( + workspace_id TEXT NOT NULL, + job_id TEXT NOT NULL, + purpose TEXT NOT NULL, + input_revision TEXT NOT NULL, + input_ref TEXT NOT NULL, + resource_key TEXT, + request_json TEXT NOT NULL CHECK (json_valid(request_json)), + input_retired INTEGER NOT NULL CHECK (input_retired IN (0,1)), + intent_fingerprint TEXT NOT NULL, + current_attempt INTEGER NOT NULL, + state TEXT NOT NULL, + result_digest TEXT, + failure_category TEXT, + failure_detail TEXT, + updated_at TEXT NOT NULL, + completed_at TEXT, + captured_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (workspace_id,job_id), + FOREIGN KEY (workspace_id,job_id) REFERENCES backend_jobs(workspace_id,job_id) ON DELETE CASCADE +); +CREATE TABLE backend_job_attempt_pre_content_history ( + workspace_id TEXT NOT NULL, + job_id TEXT NOT NULL, + attempt_id TEXT NOT NULL, + attempt INTEGER NOT NULL, + input_revision TEXT NOT NULL, + state TEXT NOT NULL, + runtime_id TEXT, + worker_id TEXT, + runtime_run_id TEXT, + dispatched_at TEXT, + deadline_at TEXT NOT NULL, + result_digest TEXT, + failure_category TEXT, + failure_detail TEXT, + worker_cleanup_state TEXT, + worker_cleanup_failure_category TEXT, + worker_cleanup_failure_detail TEXT, + worker_cleanup_updated_at TEXT, + worker_cleanup_completed_at TEXT, + updated_at TEXT NOT NULL, + completed_at TEXT, + captured_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (workspace_id,job_id,attempt_id), + FOREIGN KEY (workspace_id,job_id,attempt_id) REFERENCES backend_job_attempts(workspace_id,job_id,attempt_id) ON DELETE CASCADE, + FOREIGN KEY (workspace_id,job_id) REFERENCES backend_job_pre_content_history(workspace_id,job_id) ON DELETE CASCADE +); +-- Delivery is captured because the retained content migration modifies it, not +-- because notification reconciliation participates in the Job execution cutover. +CREATE TABLE backend_job_delivery_pre_content_history ( + workspace_id TEXT NOT NULL, + delivery_id TEXT NOT NULL, + job_id TEXT NOT NULL, + attempt_id TEXT NOT NULL, + target_runtime_id TEXT NOT NULL, + target_worker_id TEXT NOT NULL, + state TEXT NOT NULL, + failure_category TEXT, + failure_detail TEXT, + updated_at TEXT NOT NULL, + delivered_at TEXT, + captured_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (workspace_id,delivery_id), + FOREIGN KEY (workspace_id,delivery_id) REFERENCES backend_job_deliveries(workspace_id,delivery_id) ON DELETE CASCADE +); +CREATE TRIGGER backend_job_pre_content_capture_boundary +BEFORE INSERT ON backend_job_pre_content_history +WHEN (SELECT max(version) FROM __yoi_schema_migrations) IS NOT 85 +BEGIN SELECT RAISE(ABORT,'pre-content evidence requires the schema-85 transaction'); END; +CREATE TRIGGER backend_job_attempt_pre_content_capture_boundary +BEFORE INSERT ON backend_job_attempt_pre_content_history +WHEN (SELECT max(version) FROM __yoi_schema_migrations) IS NOT 85 +BEGIN SELECT RAISE(ABORT,'pre-content evidence requires the schema-85 transaction'); END; +CREATE TRIGGER backend_job_delivery_pre_content_capture_boundary +BEFORE INSERT ON backend_job_delivery_pre_content_history +WHEN (SELECT max(version) FROM __yoi_schema_migrations) IS NOT 85 +BEGIN SELECT RAISE(ABORT,'pre-content evidence requires the schema-85 transaction'); END; +CREATE TRIGGER backend_job_pre_content_identity_no_update +BEFORE UPDATE OF workspace_id,job_id,purpose,input_revision,input_ref,resource_key,intent_fingerprint,current_attempt,state,result_digest,failure_category,failure_detail,updated_at,completed_at,captured_at ON backend_job_pre_content_history +BEGIN SELECT RAISE(ABORT,'immutable pre-content Job evidence'); END; +CREATE TRIGGER backend_job_pre_content_request_retirement +BEFORE UPDATE OF request_json,input_retired ON backend_job_pre_content_history +WHEN ( + OLD.input_retired=0 AND NEW.input_retired=1 + AND json_type(NEW.request_json,'$.input')='null' + AND json_remove(NEW.request_json,'$.input')=json_remove(OLD.request_json,'$.input') + AND EXISTS(SELECT 1 FROM backend_jobs j WHERE j.workspace_id=OLD.workspace_id AND j.job_id=OLD.job_id + AND COALESCE(json_type(j.request_json,'$.input'),'null')='null') +) IS NOT TRUE +BEGIN SELECT RAISE(ABORT,'pre-content Job body retires only with its owner'); END; +CREATE TRIGGER backend_job_pre_content_retire_with_owner +AFTER UPDATE OF request_json ON backend_jobs +WHEN COALESCE(json_type(NEW.request_json,'$.input'),'null')='null' +BEGIN + UPDATE backend_job_pre_content_history + SET request_json=json_set(request_json,'$.input',json('null')),input_retired=1 + WHERE workspace_id=NEW.workspace_id AND job_id=NEW.job_id AND input_retired=0; +END; +CREATE TRIGGER backend_job_pre_content_no_input_restore +BEFORE UPDATE OF request_json ON backend_jobs +WHEN COALESCE(json_type(NEW.request_json,'$.input'),'null')!='null' + AND EXISTS(SELECT 1 FROM backend_job_pre_content_history h + WHERE h.workspace_id=OLD.workspace_id AND h.job_id=OLD.job_id AND h.input_retired=1) +BEGIN SELECT RAISE(ABORT,'retired pre-content Job input cannot regenerate'); END; +CREATE TRIGGER backend_job_attempt_pre_content_no_update +BEFORE UPDATE ON backend_job_attempt_pre_content_history +BEGIN SELECT RAISE(ABORT,'immutable pre-content attempt evidence'); END; +CREATE TRIGGER backend_job_delivery_pre_content_no_update +BEFORE UPDATE ON backend_job_delivery_pre_content_history +BEGIN SELECT RAISE(ABORT,'immutable pre-content delivery evidence'); END; +"#; + +const RAW_CAPTURE_SQL: &str = r#" +INSERT INTO backend_job_pre_content_history + (workspace_id,job_id,purpose,input_revision,input_ref,resource_key,request_json,input_retired,intent_fingerprint,current_attempt,state,result_digest,failure_category,failure_detail,updated_at,completed_at) +SELECT workspace_id,job_id,purpose,input_revision,input_ref,resource_key,request_json, + CASE WHEN COALESCE(json_type(request_json,'$.input'),'null')='null' THEN 1 ELSE 0 END, + intent_fingerprint,current_attempt,state,result_digest,failure_category,failure_detail,updated_at,completed_at +FROM backend_jobs; +INSERT INTO backend_job_attempt_pre_content_history + (workspace_id,job_id,attempt_id,attempt,input_revision,state,runtime_id,worker_id,runtime_run_id,dispatched_at,deadline_at,result_digest,failure_category,failure_detail,worker_cleanup_state,worker_cleanup_failure_category,worker_cleanup_failure_detail,worker_cleanup_updated_at,worker_cleanup_completed_at,updated_at,completed_at) +SELECT workspace_id,job_id,attempt_id,attempt,input_revision,state,runtime_id,worker_id,runtime_run_id,dispatched_at,deadline_at,result_digest,failure_category,failure_detail,worker_cleanup_state,worker_cleanup_failure_category,worker_cleanup_failure_detail,worker_cleanup_updated_at,worker_cleanup_completed_at,updated_at,completed_at +FROM backend_job_attempts; +INSERT INTO backend_job_delivery_pre_content_history + (workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,failure_category,failure_detail,updated_at,delivered_at) +SELECT workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,failure_category,failure_detail,updated_at,delivered_at +FROM backend_job_deliveries; +"#; + +/// Called exactly once INSIDE migrate_content_state_v85_to_v86's transaction, +/// before its Job SELECT/renames/updates. On error the owner must roll back the +/// entire schema-86 step. No open/read helper may call this entrypoint. The owner +/// holds the storage lifecycle lock until the final cutover is committed; partial +/// retained prefixes are resumable migration data, never execution admission. +pub(crate) fn capture_before_content_cutover(conn: &Connection) -> Result<()> { + require_transaction(conn)?; + if super::current_schema_version(conn)? != 85 + || !super::column_exists(conn, "backend_jobs", "input_revision")? + || super::column_exists(conn, "backend_jobs", "input_digest")? + { + return Err(Error::Store( + "pre-content Job capture requires the unmodified schema-85 boundary".into(), + )); + } + conn.execute_batch(RAW_SCHEMA)?; + conn.execute_batch(RAW_CAPTURE_SQL)?; + verify_pre_content(conn) +} + +/// Reject intervening Job/attempt ownership writes rather than erase real newer +/// evidence while undoing the retained cutover's synthetic values. Delivery +/// reconciliation and input retirement remain independent and are not frozen. +fn validate_content_cutover_before_restore(conn: &Connection) -> Result<()> { + let drift: bool = conn.query_row(r#" +WITH reserved_jobs AS ( + SELECT j.workspace_id,j.job_id FROM backend_job_pre_content_history j + WHERE EXISTS(SELECT 1 FROM backend_job_attempt_pre_content_history a WHERE a.workspace_id=j.workspace_id AND a.job_id=j.job_id) + AND NOT EXISTS(SELECT 1 FROM backend_job_attempt_pre_content_history a WHERE a.workspace_id=j.workspace_id AND a.job_id=j.job_id + AND (a.state!='reserved' OR a.runtime_id IS NOT NULL OR a.worker_cleanup_state IS NOT NULL)) +) +SELECT EXISTS( + SELECT 1 FROM backend_jobs j JOIN backend_job_pre_content_history r USING(workspace_id,job_id) + LEFT JOIN reserved_jobs v USING(workspace_id,job_id) + WHERE j.state IS NOT (CASE WHEN r.state='pending' THEN CASE WHEN v.job_id IS NOT NULL THEN 'failed' ELSE 'unknown' END ELSE r.state END) + OR j.failure_category IS NOT (CASE WHEN r.state='pending' THEN 'content_contract_cutover' ELSE r.failure_category END) + OR j.failure_detail IS NOT r.failure_detail OR j.updated_at IS NOT r.updated_at + OR j.completed_at IS NOT (CASE WHEN r.state='pending' THEN COALESCE(r.completed_at,r.updated_at) ELSE r.completed_at END) +) OR EXISTS( + SELECT 1 FROM backend_job_attempts a JOIN backend_job_attempt_pre_content_history r USING(workspace_id,job_id,attempt_id) + LEFT JOIN reserved_jobs v USING(workspace_id,job_id) + WHERE a.state IS NOT (CASE WHEN r.state='reserved' AND v.job_id IS NOT NULL THEN 'failed' + WHEN r.state IN ('reserved','dispatching','dispatched') THEN 'unknown' ELSE r.state END) + OR a.failure_category IS NOT (CASE WHEN r.state IN ('reserved','dispatching','dispatched') THEN 'content_contract_cutover' ELSE r.failure_category END) + OR a.failure_detail IS NOT r.failure_detail OR a.updated_at IS NOT r.updated_at + OR a.completed_at IS NOT (CASE WHEN r.state IN ('reserved','dispatching','dispatched') THEN COALESCE(r.completed_at,r.updated_at) ELSE r.completed_at END) + OR a.worker_cleanup_state IS NOT (CASE WHEN r.state='reserved' AND v.job_id IS NOT NULL THEN 'completed' + WHEN r.runtime_id IS NOT NULL AND r.state IN ('dispatching','dispatched') AND r.worker_cleanup_state IS NULL THEN 'pending' ELSE r.worker_cleanup_state END) + OR a.worker_cleanup_failure_category IS NOT r.worker_cleanup_failure_category + OR a.worker_cleanup_failure_detail IS NOT r.worker_cleanup_failure_detail + OR a.worker_cleanup_updated_at IS NOT r.worker_cleanup_updated_at + OR a.worker_cleanup_completed_at IS NOT r.worker_cleanup_completed_at +) + "#,[],|row|row.get(0))?; + if drift { + return Err(Error::Store("pre-content Job/attempt changed after its retained cutover; preserve evidence and reconcile before final migration".into())); + } + Ok(()) +} + +fn require_transaction(conn: &Connection) -> Result<()> { + if conn.is_autocommit() { + return Err(Error::Store( + "Backend Job upgrade requires a caller-owned transaction".into(), + )); + } + Ok(()) +} + +/// Fresh creation installs empty evidence and final guards, never captures or +/// fails Jobs. The fresh canonical state CHECKs are supplied by the owner DDL. +pub(super) fn install_fresh(conn: &Connection) -> Result<()> { + require_transaction(conn)?; + conn.execute_batch(RAW_SCHEMA)?; + install_live_checks(conn)?; + conn.execute_batch(SCHEMA)?; + conn.execute_batch( + "ALTER TABLE backend_job_attempts ADD COLUMN allocation_precluded_at TEXT;", + )?; + conn.execute_batch(super::backend_job_recovery_store::SCHEMA)?; + verify(conn) +} + +pub(super) fn install_live_checks(conn: &Connection) -> Result<()> { + for (table, old, new) in STATE_CHECKS { + replace_state_check(conn, table, old, new)?; + } + Ok(()) +} + +/// The retained pre-123 reference composition must keep its original CHECKs, +/// even when the fresh owner DDL is updated to the current live contract. +pub(super) fn retained_checks(sql: &str) -> String { + STATE_CHECKS.iter().fold( + sql.replace(" allocation_precluded_at TEXT,\n", ""), + |sql, (_, old, new)| sql.replace(new, old), + ) +} + +const SNAPSHOT_SQL: &str = r#" +INSERT INTO backend_job_upgrade_history + (workspace_id,job_id,state,failure_category,failure_detail,updated_at,completed_at) +SELECT workspace_id,job_id,state,failure_category,failure_detail,updated_at,completed_at FROM backend_jobs; +INSERT INTO backend_job_attempt_upgrade_history + (workspace_id,job_id,attempt_id,state,failure_category,failure_detail,updated_at,completed_at) +SELECT workspace_id,job_id,attempt_id,state,failure_category,failure_detail,updated_at,completed_at FROM backend_job_attempts; +"#; + +// Preserve result/input/identity and every execution timestamp, including old +// completed_at. Migration time belongs to history, not execution-end evidence. +const FAIL_JOBS_SQL: &str = r#" +UPDATE backend_jobs SET state='failed',failure_category=?1,failure_detail=?2 +WHERE state NOT IN ('completed','failed'); +"#; +const FAIL_ATTEMPTS_SQL: &str = r#" +UPDATE backend_job_attempts SET state='failed',failure_category=?1,failure_detail=?2 +WHERE state NOT IN ('completed','failed'); +"#; + +// Restore only captured values, not inferred execution-end evidence from the +// retained content cutover. Live normalized input/digest/fingerprint remain in +// their current format; their original wire identity is retained in raw evidence. +const APPLY_CAPTURED_JOBS_SQL: &str = r#" +UPDATE backend_jobs AS j SET (state,failure_category,failure_detail,updated_at,completed_at) = ( + SELECT CASE WHEN r.state IN ('completed','failed') THEN r.state ELSE 'failed' END, + CASE WHEN r.state IN ('completed','failed') THEN r.failure_category ELSE ?1 END, + CASE WHEN r.state IN ('completed','failed') THEN r.failure_detail ELSE ?2 END, + r.updated_at,r.completed_at + FROM backend_job_pre_content_history r WHERE r.workspace_id=j.workspace_id AND r.job_id=j.job_id +) WHERE EXISTS(SELECT 1 FROM backend_job_pre_content_history r WHERE r.workspace_id=j.workspace_id AND r.job_id=j.job_id); +"#; +const APPLY_CAPTURED_ATTEMPTS_SQL: &str = r#" +UPDATE backend_job_attempts AS a SET + (state,failure_category,failure_detail,updated_at,completed_at, + worker_cleanup_state,worker_cleanup_failure_category,worker_cleanup_failure_detail,worker_cleanup_updated_at,worker_cleanup_completed_at) = ( + SELECT CASE WHEN r.state IN ('completed','failed') THEN r.state ELSE 'failed' END, + CASE WHEN r.state IN ('completed','failed') THEN r.failure_category ELSE ?1 END, + CASE WHEN r.state IN ('completed','failed') THEN r.failure_detail ELSE ?2 END, + r.updated_at,r.completed_at, + r.worker_cleanup_state,r.worker_cleanup_failure_category,r.worker_cleanup_failure_detail,r.worker_cleanup_updated_at,r.worker_cleanup_completed_at + FROM backend_job_attempt_pre_content_history r + WHERE r.workspace_id=a.workspace_id AND r.job_id=a.job_id AND r.attempt_id=a.attempt_id +) WHERE EXISTS(SELECT 1 FROM backend_job_attempt_pre_content_history r + WHERE r.workspace_id=a.workspace_id AND r.job_id=a.job_id AND r.attempt_id=a.attempt_id); +"#; + +// Exact retained CHECKs only. Published Jobs-family tables may already have the +// new CHECK; the physical recognizer, not this function, owns family acceptance. +const STATE_CHECKS: [(&str, &str, &str); 2] = [ + ( + "backend_jobs", + "CHECK (state IN ('pending', 'completed', 'failed', 'unknown'))", + "CHECK (state IN ('pending', 'completed', 'failed'))", + ), + ( + "backend_job_attempts", + "CHECK (state IN ('reserved', 'dispatching', 'dispatched', 'completed', 'failed', 'unknown'))", + "CHECK (state IN ('reserved', 'dispatching', 'dispatched', 'completed', 'failed'))", + ), +]; + +/// Apply once in the caller's validated Workspace migration transaction. +/// On ANY error the caller must roll back, including DDL and its version receipt. +/// A second invocation is an error, not a new sweep of post-upgrade Jobs. +pub(super) fn install(conn: &Connection) -> Result<()> { + require_transaction(conn)?; + let foreign_keys: bool = conn.query_row("PRAGMA foreign_keys", [], |row| row.get(0))?; + if foreign_keys { + return Err(Error::Store( + "Backend Job upgrade requires foreign_keys OFF before the transaction".into(), + )); + } + for table in ["backend_jobs", "backend_job_attempts"] { + if !super::column_exists(conn, table, "input_digest")? + || super::column_exists(conn, table, "input_revision")? + { + return Err(Error::Store( + "Backend Job final cutover requires the retained content normalization first" + .into(), + )); + } + } + let raw_tables = [ + "backend_job_pre_content_history", + "backend_job_attempt_pre_content_history", + "backend_job_delivery_pre_content_history", + ]; + let present = raw_tables + .into_iter() + .map(|name| super::table_exists(conn, name)) + .collect::>>()?; + if present.iter().all(|exists| !exists) { + // Sources that were already schema 86+ cannot regain lost pre-86 facts. + conn.execute_batch(RAW_SCHEMA)?; + } else if !present.iter().all(|exists| *exists) { + return Err(Error::Store( + "incomplete pre-content Job evidence schema".into(), + )); + } + verify_pre_content(conn)?; + validate_content_cutover_before_restore(conn)?; + let (history_schema, execution_guards) = SCHEMA + .split_once("-- FINAL_EXECUTION_GUARDS") + .ok_or_else(|| Error::Store("missing final Backend Job guard boundary".into()))?; + conn.execute_batch(history_schema)?; + conn.execute_batch(SNAPSHOT_SQL)?; + for sql in [ + FAIL_JOBS_SQL, + FAIL_ATTEMPTS_SQL, + APPLY_CAPTURED_JOBS_SQL, + APPLY_CAPTURED_ATTEMPTS_SQL, + ] { + conn.execute(sql, [FAILURE_CATEGORY, FAILURE_DETAIL])?; + } + for (table, old, new) in STATE_CHECKS { + replace_state_check(conn, table, old, new)?; + } + // A captured originally-unfinished row may already be Failed by schema 86. + // Install terminal immutability only after replacing that synthetic outcome. + conn.execute_batch(execution_guards)?; + // Old Failed/Reserved rows acquire no negative allocation proof from migration. + conn.execute_batch( + "ALTER TABLE backend_job_attempts ADD COLUMN allocation_precluded_at TEXT;", + )?; + conn.execute_batch(super::backend_job_recovery_store::SCHEMA)?; + verify(conn) +} + +fn quoted(identifier: &str) -> String { + format!("\"{}\"", identifier.replace('"', "\"\"")) +} + +// Recreate under the SAME table name, without ALTER RENAME or writable_schema. +// This avoids rewriting FKs/views/triggers attached to other family tables. Copy +// every physical column, including published cleanup ownership/admission fields; +// don't replace family DDL with a baseline or infer any missing termination rows. +fn replace_state_check(conn: &Connection, table: &str, old: &str, new: &str) -> Result<()> { + let ddl: String = conn.query_row( + "SELECT sql FROM sqlite_schema WHERE type='table' AND name=?1", + [table], + |row| row.get(0), + )?; + if ddl.matches(new).count() == 1 && !ddl.contains(old) { + return Ok(()); + } + if ddl.matches(old).count() != 1 || ddl.contains(new) { + return Err(Error::Store(format!( + "Backend Job upgrade: unrecognized state CHECK in {table}" + ))); + } + let programs = conn.prepare( + "SELECT sql FROM sqlite_schema WHERE tbl_name=?1 AND type IN ('index','trigger') AND sql IS NOT NULL ORDER BY type,name", + )?.query_map([table], |row| row.get::<_, String>(0))? + .collect::>>()?; + let columns = conn + .prepare(&format!("PRAGMA table_xinfo({})", quoted(table)))? + .query_map([], |row| { + Ok((row.get::<_, String>(1)?, row.get::<_, i64>(6)?)) + })? + .collect::>>()?; + if columns.iter().any(|(_, hidden)| *hidden != 0) { + return Err(Error::Store(format!( + "Backend Job upgrade: unsupported generated/hidden column in {table}" + ))); + } + let columns = columns + .iter() + .map(|(name, _)| quoted(name)) + .collect::>() + .join(","); + let temporary = quoted(&format!("__yoi_upgrade_{table}")); + let table_name = quoted(table); + conn.execute_batch(&format!( + "CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM main.{table_name}; + DROP TABLE main.{table_name};" + ))?; + conn.execute_batch(&ddl.replacen(old, new, 1))?; + conn.execute_batch(&format!( + "INSERT INTO main.{table_name} ({columns}) SELECT {columns} FROM temp.{temporary}; + DROP TABLE temp.{temporary};" + ))?; + for sql in programs { + conn.execute_batch(&sql)?; + } + Ok(()) +} + +/// Validate ownership and immutable content without normalizing or restoring any +/// body. This also runs before the schema-86 receipt, when input_revision is live. +/// Cleanup/delivery outcomes may legitimately evolve after migration; they are +/// historical evidence, not shadow operational state machines. +pub(super) fn verify_pre_content(conn: &Connection) -> Result<()> { + let invalid: bool = conn.query_row(r#" + SELECT EXISTS( + SELECT 1 FROM backend_job_pre_content_history r JOIN backend_jobs j + ON j.workspace_id=r.workspace_id AND j.job_id=r.job_id + WHERE j.purpose IS NOT r.purpose OR j.input_ref IS NOT r.input_ref + OR j.resource_key IS NOT r.resource_key OR j.current_attempt IS NOT r.current_attempt + OR j.result_digest IS NOT r.result_digest + OR (SELECT count(*) FROM backend_job_attempts a WHERE a.workspace_id=r.workspace_id AND a.job_id=r.job_id) + != (SELECT count(*) FROM backend_job_attempt_pre_content_history a WHERE a.workspace_id=r.workspace_id AND a.job_id=r.job_id) + ) OR EXISTS( + SELECT 1 FROM backend_job_attempt_pre_content_history r JOIN backend_job_attempts a + ON a.workspace_id=r.workspace_id AND a.job_id=r.job_id AND a.attempt_id=r.attempt_id + WHERE a.attempt IS NOT r.attempt OR a.runtime_id IS NOT r.runtime_id OR a.worker_id IS NOT r.worker_id + OR a.runtime_run_id IS NOT r.runtime_run_id OR a.dispatched_at IS NOT r.dispatched_at + OR a.deadline_at IS NOT r.deadline_at OR a.result_digest IS NOT r.result_digest + ) OR EXISTS( + SELECT 1 FROM backend_job_delivery_pre_content_history r JOIN backend_job_deliveries d + ON d.workspace_id=r.workspace_id AND d.delivery_id=r.delivery_id + WHERE d.job_id IS NOT r.job_id OR d.attempt_id IS NOT r.attempt_id + OR d.target_runtime_id IS NOT r.target_runtime_id OR d.target_worker_id IS NOT r.target_worker_id + ) + "#, [], |row| row.get(0))?; + if invalid { + return Err(Error::Store( + "pre-content Job identity/result/attempt coverage drift".into(), + )); + } + let mut statement = conn.prepare("SELECT r.request_json,j.request_json,r.input_retired FROM backend_job_pre_content_history r JOIN backend_jobs j ON j.workspace_id=r.workspace_id AND j.job_id=r.job_id")?; + let mut rows = statement.query([])?; + while let Some(row) = rows.next()? { + let original: serde_json::Value = serde_json::from_str(&row.get::<_, String>(0)?) + .map_err(|_| Error::Store("invalid pre-content request evidence".into()))?; + let current: serde_json::Value = serde_json::from_str(&row.get::<_, String>(1)?) + .map_err(|_| Error::Store("invalid live request beside pre-content evidence".into()))?; + let retired: bool = row.get(2)?; + let original_input = original.get("input"); + let current_input = current.get("input"); + let original_empty = original_input.is_none_or(serde_json::Value::is_null); + let current_empty = current_input.is_none_or(serde_json::Value::is_null); + if (retired && (!original_empty || !current_empty)) + || (!retired && (original_empty || original_input != current_input)) + { + return Err(Error::Store( + "pre-content immutable input lost or regenerated".into(), + )); + } + } + Ok(()) +} + +/// Read-only postcondition; never sweeps or repairs post-migration Jobs. +pub(super) fn verify(conn: &Connection) -> Result<()> { + verify_pre_content(conn)?; + for (table, old, new) in STATE_CHECKS { + let ddl: String = conn.query_row( + "SELECT sql FROM sqlite_schema WHERE type='table' AND name=?1", + [table], + |row| row.get(0), + )?; + if ddl.contains(old) || ddl.matches(new).count() != 1 { + return Err(Error::Store(format!( + "Backend Job upgrade left a legacy live-state CHECK in {table}" + ))); + } + } + for (table, history, raw_history, identity) in [ + ( + "backend_jobs", + "backend_job_upgrade_history", + "backend_job_pre_content_history", + "a.job_id=h.job_id", + ), + ( + "backend_job_attempts", + "backend_job_attempt_upgrade_history", + "backend_job_attempt_pre_content_history", + "a.job_id=h.job_id AND a.attempt_id=h.attempt_id", + ), + ] { + let raw_identity = identity.replace("h.", "r."); + let basis = "CASE WHEN r.job_id IS NULL THEN h.state ELSE r.state END"; + let category = + "CASE WHEN r.job_id IS NULL THEN h.failure_category ELSE r.failure_category END"; + let detail = "CASE WHEN r.job_id IS NULL THEN h.failure_detail ELSE r.failure_detail END"; + let completed = "CASE WHEN r.job_id IS NULL THEN h.completed_at ELSE r.completed_at END"; + let invalid: bool = conn.query_row( + &format!( + "SELECT EXISTS(SELECT 1 FROM {table} a JOIN {history} h + ON a.workspace_id=h.workspace_id AND {identity} + LEFT JOIN {raw_history} r ON a.workspace_id=r.workspace_id AND {raw_identity} + WHERE a.state NOT IN ('completed','failed') OR a.completed_at IS NOT ({completed}) + OR (({basis}) IN ('completed','failed') + AND (a.state IS NOT ({basis}) OR a.failure_category IS NOT ({category}) + OR a.failure_detail IS NOT ({detail}))) + OR (({basis}) NOT IN ('completed','failed') + AND (a.state!='failed' OR a.failure_category IS NOT ?1 OR a.failure_detail IS NOT ?2)))" + ), + [FAILURE_CATEGORY, FAILURE_DETAIL], + |row| row.get(0), + )?; + if invalid { + return Err(Error::Store(format!( + "Backend Job upgrade left an unretired row in {table}" + ))); + } + } + let violations: i64 = + conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| { + row.get(0) + })?; + if violations != 0 { + return Err(Error::Store( + "Backend Job upgrade left foreign-key violations".into(), + )); + } + Ok(()) +} + +/// Reuse the storage owner's exact published-family fixtures in its normal suite. +#[cfg(test)] +pub(super) fn assert_published_fixture_for_test(conn: &Connection) { + tests::assert_published_fixture(conn); +} + +#[cfg(test)] +#[path = "backend_job_upgrade_tests.rs"] +mod tests; diff --git a/crates/workspace-server/src/backend_job_upgrade.sql b/crates/workspace-server/src/backend_job_upgrade.sql new file mode 100644 index 000000000..9fa265f2d --- /dev/null +++ b/crates/workspace-server/src/backend_job_upgrade.sql @@ -0,0 +1,76 @@ +-- These rows mark the migration boundary, including already terminal Jobs. +-- Do not duplicate request/result bodies: their existing retention/erasure owner +-- remains authoritative. Historical state spellings are not live execution states. +CREATE TABLE backend_job_upgrade_history ( + workspace_id TEXT NOT NULL, + job_id TEXT NOT NULL, + state TEXT NOT NULL, + failure_category TEXT, + failure_detail TEXT, + updated_at TEXT NOT NULL, + completed_at TEXT, + migrated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (workspace_id, job_id), + FOREIGN KEY (workspace_id, job_id) + REFERENCES backend_jobs(workspace_id, job_id) ON DELETE CASCADE +); +CREATE TABLE backend_job_attempt_upgrade_history ( + workspace_id TEXT NOT NULL, + job_id TEXT NOT NULL, + attempt_id TEXT NOT NULL, + state TEXT NOT NULL, + failure_category TEXT, + failure_detail TEXT, + updated_at TEXT NOT NULL, + completed_at TEXT, + migrated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + PRIMARY KEY (workspace_id, job_id, attempt_id), + FOREIGN KEY (workspace_id, job_id, attempt_id) + REFERENCES backend_job_attempts(workspace_id, job_id, attempt_id) ON DELETE CASCADE +); +CREATE TRIGGER backend_job_upgrade_history_no_update +BEFORE UPDATE ON backend_job_upgrade_history +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job upgrade history'); END; +CREATE TRIGGER backend_job_attempt_upgrade_history_no_update +BEFORE UPDATE ON backend_job_attempt_upgrade_history +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job attempt upgrade history'); END; + +-- FINAL_EXECUTION_GUARDS +-- Install these only after all captured pre-content values have been composed. +-- Input retirement and independently evidenced cleanup are deliberately separate. +CREATE TRIGGER backend_job_upgrade_no_resume +BEFORE UPDATE ON backend_jobs +WHEN EXISTS (SELECT 1 FROM backend_job_upgrade_history h + WHERE h.workspace_id=OLD.workspace_id AND h.job_id=OLD.job_id) + AND OLD.state IN ('completed','failed') + AND (NEW.state IS NOT OLD.state OR NEW.current_attempt IS NOT OLD.current_attempt + OR NEW.result_json IS NOT OLD.result_json OR NEW.result_digest IS NOT OLD.result_digest + OR NEW.failure_category IS NOT OLD.failure_category OR NEW.failure_detail IS NOT OLD.failure_detail + OR NEW.completed_at IS NOT OLD.completed_at) +BEGIN SELECT RAISE(ABORT, 'pre-upgrade Backend Job cannot resume'); END; +CREATE TRIGGER backend_job_upgrade_no_attempt +BEFORE INSERT ON backend_job_attempts +WHEN EXISTS (SELECT 1 FROM backend_job_upgrade_history h + WHERE h.workspace_id=NEW.workspace_id AND h.job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT, 'pre-upgrade Backend Job cannot create attempts'); END; +CREATE TRIGGER backend_job_upgrade_no_attempt_reparent +BEFORE UPDATE OF workspace_id,job_id ON backend_job_attempts +WHEN (NEW.workspace_id IS NOT OLD.workspace_id OR NEW.job_id IS NOT OLD.job_id) + AND EXISTS (SELECT 1 FROM backend_job_upgrade_history h + WHERE h.workspace_id=NEW.workspace_id AND h.job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT, 'pre-upgrade Backend Job cannot adopt attempts'); END; +CREATE TRIGGER backend_job_attempt_upgrade_no_resume +BEFORE UPDATE ON backend_job_attempts +WHEN EXISTS (SELECT 1 FROM backend_job_attempt_upgrade_history h + WHERE h.workspace_id=OLD.workspace_id AND h.job_id=OLD.job_id AND h.attempt_id=OLD.attempt_id) + AND OLD.state IN ('completed','failed') + AND (NEW.workspace_id IS NOT OLD.workspace_id OR NEW.job_id IS NOT OLD.job_id + OR NEW.attempt_id IS NOT OLD.attempt_id OR NEW.attempt IS NOT OLD.attempt + OR NEW.input_digest IS NOT OLD.input_digest OR NEW.state IS NOT OLD.state + OR NEW.runtime_id IS NOT OLD.runtime_id OR NEW.worker_id IS NOT OLD.worker_id + OR NEW.runtime_run_id IS NOT OLD.runtime_run_id OR NEW.dispatched_at IS NOT OLD.dispatched_at + OR NEW.deadline_at IS NOT OLD.deadline_at + OR NEW.result_json IS NOT OLD.result_json OR NEW.result_digest IS NOT OLD.result_digest + OR NEW.failure_category IS NOT OLD.failure_category OR NEW.failure_detail IS NOT OLD.failure_detail + OR NEW.completed_at IS NOT OLD.completed_at) +BEGIN SELECT RAISE(ABORT, 'pre-upgrade Backend Job attempt cannot resume'); END; diff --git a/crates/workspace-server/src/backend_job_upgrade_tests.rs b/crates/workspace-server/src/backend_job_upgrade_tests.rs new file mode 100644 index 000000000..fe445887e --- /dev/null +++ b/crates/workspace-server/src/backend_job_upgrade_tests.rs @@ -0,0 +1,1671 @@ +//! Component tests use retained schema-113 DDL and real SQLite transactions. +//! Runner tests use the store's actual plan/apply/open paths, not a second ledger. +//! They run in the ordinary workspace-server lib target once the root registers +//! this module AND the migration. No Runtime/process termination or external +//! Journal checkpoint completion is simulated or claimed by these tests. +use super::*; +use crate::backend_job::BackendJobRequest; +use crate::store::{self as store_owner, ControlPlaneStore}; +use rusqlite::{TransactionBehavior, params, types::Value}; +use std::path::Path; + +const JOB_VALUES: &str = "workspace_id,job_id,purpose,input_digest,input_ref,request_json,intent_fingerprint,current_attempt,result_json,result_digest,created_at,updated_at,completed_at"; +const ATTEMPT_VALUES: &str = "workspace_id,job_id,attempt_id,attempt,input_digest,runtime_id,worker_id,runtime_run_id,dispatched_at,deadline_at,result_json,result_digest,worker_cleanup_state,worker_cleanup_failure_category,worker_cleanup_failure_detail,worker_cleanup_updated_at,worker_cleanup_completed_at,created_at,updated_at,completed_at"; + +fn rows(conn: &Connection, sql: &str) -> Vec> { + let mut statement = conn.prepare(sql).unwrap(); + let count = statement.column_count(); + statement + .query_map([], |row| { + (0..count) + .map(|i| row.get(i)) + .collect::>>() + }) + .unwrap() + .collect::>>() + .unwrap() +} + +fn request(id: &str) -> BackendJobRequest { + BackendJobRequest { + job_id: id.into(), + purpose: "upgrade_fixture".into(), + input_ref: format!("saved://{id}"), + input: serde_json::json!({"body":"immutable input","selection":[1,2]}), + instruction: "Use only the saved input".into(), + profile: "builtin:backend-job".into(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits { + max_attempts: 3, + ..Default::default() + }, + } +} + +fn seed_workspace(conn: &Connection) { + conn.execute_batch("INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) + VALUES('upgrade-owner','user','upgrade-owner','Owner','1','1'); + INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) + VALUES('upgrade-workspace','upgrade-owner','Workspace','active','1','1');").unwrap(); +} + +fn insert_job(conn: &Connection, id: &str, state: &str) { + let request = request(id); + let legacy = store_owner::column_exists(conn, "backend_jobs", "input_revision").unwrap(); + let input_column = if legacy { + "input_revision" + } else { + "input_digest" + }; + let mut encoded = serde_json::to_value(&request).unwrap(); + let input_identity = if legacy { + let legacy_token = format!("legacy-token:{id}"); + encoded["input_revision"] = serde_json::Value::String(legacy_token.clone()); + legacy_token + } else { + request.input_digest().unwrap() + }; + let fingerprint = if legacy { + format!("legacy-fingerprint:{id}") + } else { + request.fingerprint().unwrap() + }; + // Retain noncanonical whitespace too, not merely a reserialized body value. + let encoded = if legacy { + serde_json::to_string_pretty(&encoded).unwrap() + } else { + serde_json::to_string(&request).unwrap() + }; + conn.execute(&format!("INSERT INTO backend_jobs(workspace_id,job_id,purpose,{input_column},input_ref,request_json,intent_fingerprint,state,current_attempt,result_json,result_digest,failure_category,failure_detail,created_at,updated_at,completed_at) + VALUES('upgrade-workspace',?1,?2,?3,?4,?5,?6,?7,1,?8,?9,'old-cause','old detail','created','updated',?10)"), + params![id, request.purpose, input_identity, request.input_ref, + encoded, fingerprint, state, + (state=="completed").then_some("{\"saved\":true}"), + (state=="completed").then_some("result-digest"), + (state=="completed" || state=="failed" || state=="unknown").then_some("old-end")]).unwrap(); +} + +fn insert_attempt(conn: &Connection, job: &str, id: &str, number: i64, state: &str) { + let legacy = + store_owner::column_exists(conn, "backend_job_attempts", "input_revision").unwrap(); + let input_column = if legacy { + "input_revision" + } else { + "input_digest" + }; + let input_identity = if legacy { + format!("legacy-token:{job}") + } else { + "attempt-digest".into() + }; + let ddl: String = conn + .query_row( + "SELECT sql FROM sqlite_schema WHERE name='backend_job_attempts'", + [], + |row| row.get(0), + ) + .unwrap(); + let state = if state == "dispatching" && !ddl.contains("'dispatching'") { + "dispatched" + } else { + state + }; + conn.execute(&format!("INSERT INTO backend_job_attempts(workspace_id,job_id,attempt_id,attempt,{input_column},state,runtime_id,worker_id,runtime_run_id,dispatched_at,deadline_at,result_json,result_digest,failure_category,failure_detail,created_at,updated_at,completed_at) + VALUES('upgrade-workspace',?1,?2,?3,?12,?4,?5,?6,?7,?8,'deadline',?9,?10,'old-attempt-cause','old attempt detail','created','updated',?11)"), + params![job,id,number,state, + (state!="reserved").then_some(format!("runtime-{id}")), + (state!="reserved").then_some(format!("worker-{id}")), + (state=="dispatched").then_some(format!("run-{id}")), + (state=="dispatched").then_some("dispatched-at"), + (state=="completed").then_some("{\"saved_attempt\":true}"), + (state=="completed").then_some("attempt-result-digest"), + (state=="completed" || state=="failed" || state=="unknown").then_some("old-end"),input_identity]).unwrap(); +} + +fn seed_boundary(conn: &Connection) { + seed_workspace(conn); + for (job, state, attempt_states) in [ + ("unstarted", "pending", vec!["reserved"]), + ("running", "pending", vec!["dispatched", "dispatching"]), + ("uncertain", "unknown", vec!["unknown"]), + ( + "partial-journal", + "pending", + vec!["completed", "failed", "reserved"], + ), + ("checkpointed-journal", "pending", vec!["dispatched"]), + // Even inconsistent unfinished historical attempts under terminal Jobs + // are retired; a current_attempt-only sweep would miss these rows. + ("completed", "completed", vec!["completed", "reserved"]), + ("failed", "failed", vec!["failed", "dispatched"]), + ] { + insert_job(conn, job, state); + for (index, attempt_state) in attempt_states.iter().enumerate() { + insert_attempt( + conn, + job, + &format!("{job}-{index}"), + index as i64 + 1, + attempt_state, + ); + } + } + conn.execute_batch("UPDATE backend_jobs SET current_attempt=3 WHERE job_id='partial-journal';") + .unwrap(); + if store_owner::column_exists(conn, "backend_job_attempts", "worker_cleanup_state").unwrap() { + conn.execute_batch("UPDATE backend_job_attempts SET worker_cleanup_state='executing',worker_cleanup_failure_category='saved-cleanup-cause',worker_cleanup_failure_detail='saved cleanup detail',worker_cleanup_updated_at='cleanup-updated' WHERE attempt_id='running-0';").unwrap(); + } + let delivery_ddl: String = conn + .query_row( + "SELECT sql FROM sqlite_schema WHERE name='backend_job_deliveries'", + [], + |row| row.get(0), + ) + .unwrap(); + for (index, state) in ["pending", "sending", "completed", "failed", "unknown"] + .iter() + .enumerate() + { + if !delivery_ddl.contains(&format!("'{state}'")) { + continue; + } + conn.execute("INSERT INTO backend_job_deliveries(workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,failure_category,failure_detail,created_at,updated_at,delivered_at) + VALUES('upgrade-workspace',?1,'running','running-0','target-runtime','target-worker',?2,'old-delivery-cause','old delivery detail','created','updated',?3)", + params![format!("delivery-{index}"),state,(*state=="completed").then_some("delivered")]).unwrap(); + } + // Preserve explicit erasure without copying the old body into history. + if store_owner::column_exists(conn, "backend_jobs", "input_disposition").unwrap() { + conn.execute_batch("UPDATE backend_jobs SET input_disposition='erased',request_json=json_set(request_json,'$.input',json('null')) WHERE job_id='failed';").unwrap(); + } else { + // A legacy body already absent before capture must never regenerate. + conn.execute_batch("UPDATE backend_jobs SET request_json=json_set(request_json,'$.input',json('null')) WHERE job_id='failed';").unwrap(); + } +} + +const PRE_CONTENT_JOB_VALUES: &str = "workspace_id,job_id,purpose,input_revision,input_ref,resource_key,request_json,intent_fingerprint,current_attempt,state,result_digest,failure_category,failure_detail,updated_at,completed_at"; +const PRE_CONTENT_ATTEMPT_VALUES: &str = "workspace_id,job_id,attempt_id,attempt,input_revision,state,runtime_id,worker_id,runtime_run_id,dispatched_at,deadline_at,result_digest,failure_category,failure_detail,worker_cleanup_state,worker_cleanup_failure_category,worker_cleanup_failure_detail,worker_cleanup_updated_at,worker_cleanup_completed_at,updated_at,completed_at"; +const PRE_CONTENT_DELIVERY_VALUES: &str = "workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,failure_category,failure_detail,updated_at,delivered_at"; + +fn legacy_fixture_to_85(path: &Path, baseline: i64) -> Connection { + if baseline <= 51 { + store_owner::tests::prepare_schema_v50(path, None); + } else { + store_owner::tests::prepare_retained_schema(path, baseline); + } + let conn = Connection::open(path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + let current = store_owner::current_schema_version(&conn).unwrap(); + // The oldest baseline has no Job tables. Admit rows at the first REAL Job + // step (70), keeping the original 50/51/... receipt prefix intact. + for step in store_owner::MIGRATIONS + .iter() + .filter(|step| step.version > current && step.version <= 70) + { + (step.apply)(&conn).unwrap(); + } + seed_boundary(&conn); + let current = store_owner::current_schema_version(&conn).unwrap(); + for step in store_owner::MIGRATIONS + .iter() + .filter(|step| step.version > current && step.version <= 85) + { + (step.apply)(&conn).unwrap(); + } + assert_eq!(store_owner::current_schema_version(&conn).unwrap(), 85); + conn +} + +fn pre_content_values(conn: &Connection, raw: bool) -> Vec>> { + [ + ( + "backend_jobs", + "backend_job_pre_content_history", + PRE_CONTENT_JOB_VALUES, + "job_id", + ), + ( + "backend_job_attempts", + "backend_job_attempt_pre_content_history", + PRE_CONTENT_ATTEMPT_VALUES, + "attempt_id", + ), + ( + "backend_job_deliveries", + "backend_job_delivery_pre_content_history", + PRE_CONTENT_DELIVERY_VALUES, + "delivery_id", + ), + ] + .into_iter() + .map(|(live, history, columns, key)| { + rows( + conn, + &format!( + "SELECT {columns} FROM {} ORDER BY {key}", + if raw { history } else { live } + ), + ) + }) + .collect() +} + +#[test] +fn explicit_capture_is_transactional_single_call_and_retirement_scrubs_raw_body() { + let dir = tempfile::tempdir().unwrap(); + let conn = legacy_fixture_to_85(&dir.path().join("capture.sqlite"), 85); + let original = pre_content_values(&conn, false); + assert!(capture_before_content_cutover(&conn).is_err()); + { + let tx = conn.unchecked_transaction().unwrap(); + capture_before_content_cutover(&tx).unwrap(); + assert_eq!(original, pre_content_values(&tx, true)); + let live = rows( + &tx, + "SELECT request_json FROM backend_jobs WHERE job_id='unstarted'", + ); + assert!(tx.execute_batch("UPDATE backend_job_pre_content_history SET request_json=json_set(request_json,'$.input',json('null')),input_retired=1 WHERE job_id='unstarted'").is_err()); + tx.execute_batch("UPDATE backend_jobs SET request_json=json_set(request_json,'$.input',json('null')) WHERE job_id='unstarted'").unwrap(); + assert_eq!( + rows( + &tx, + "SELECT input_retired,json_type(request_json,'$.input') FROM backend_job_pre_content_history WHERE job_id='unstarted'" + ), + vec![vec![Value::Integer(1), Value::Text("null".into())]] + ); + let Value::Text(original_request) = &live[0][0] else { + unreachable!() + }; + assert!( + tx.execute( + "UPDATE backend_jobs SET request_json=?1 WHERE job_id='unstarted'", + [original_request] + ) + .is_err() + ); + assert!( + tx.execute_batch( + "UPDATE backend_job_pre_content_history SET failure_category='rewritten'" + ) + .is_err() + ); + assert!( + tx.execute_batch( + "UPDATE backend_job_delivery_pre_content_history SET state='completed'" + ) + .is_err() + ); + assert!(capture_before_content_cutover(&tx).is_err()); + // Owner rolls back the WHOLE step after any failure, not just a subcall. + } + assert!(!store_owner::table_exists(&conn, "backend_job_pre_content_history").unwrap()); + assert_eq!(original, pre_content_values(&conn, false)); +} + +#[test] +fn intervening_cleanup_evidence_is_not_overwritten_by_original_value_restoration() { + let dir = tempfile::tempdir().unwrap(); + let conn = legacy_fixture_to_85(&dir.path().join("intervening-evidence.sqlite"), 85); + store_owner::migrate_content_state_v85_to_v86(&conn).unwrap(); + // Simulate a violation of the caller's migration-only ownership window. + // The leaf must fail closed, preserving an actual later observation instead + // of replacing it with NULL from the old capture. + conn.execute_batch("UPDATE backend_job_attempts SET worker_cleanup_state='completed',worker_cleanup_completed_at='later-owner-evidence' WHERE attempt_id='running-0'").unwrap(); + let later = rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY attempt_id", + ); + assert!(apply_leaf(&conn).is_err()); + assert_eq!( + later, + rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY attempt_id" + ) + ); + assert!(!store_owner::table_exists(&conn, "backend_job_upgrade_history").unwrap()); +} + +#[test] +fn real_content_step_receipt_failure_rolls_back_raw_capture_then_retries_into_final_cutover() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("legacy-receipt.sqlite"); + let conn = legacy_fixture_to_85(&path, 85); + let original = pre_content_values(&conn, false); + let jobs = rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id"); + let attempts = rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY attempt_id", + ); + let deliveries = rows( + &conn, + "SELECT * FROM backend_job_deliveries ORDER BY delivery_id", + ); + conn.execute_batch("CREATE TRIGGER injected_content_receipt BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=86 BEGIN SELECT RAISE(ABORT,'injected content receipt'); END;").unwrap(); + assert!(store_owner::migrate_content_state_v85_to_v86(&conn).is_err()); + assert_eq!(store_owner::current_schema_version(&conn).unwrap(), 85); + assert!(!store_owner::table_exists(&conn, "backend_job_pre_content_history").unwrap()); + assert_eq!( + jobs, + rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id") + ); + assert_eq!( + attempts, + rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY attempt_id" + ) + ); + assert_eq!( + deliveries, + rows( + &conn, + "SELECT * FROM backend_job_deliveries ORDER BY delivery_id" + ) + ); + conn.execute_batch("DROP TRIGGER injected_content_receipt") + .unwrap(); + store_owner::migrate_content_state_v85_to_v86(&conn).unwrap(); + assert_eq!(original, pre_content_values(&conn, true)); + // This is precisely the legacy vacuous failure that a final-state-only + // sweep used to miss. Original cleanup/completed_at were both NULL. + assert_eq!( + rows( + &conn, + "SELECT state,failure_category FROM backend_jobs WHERE job_id='unstarted'" + ), + vec![vec![ + Value::Text("failed".into()), + Value::Text("content_contract_cutover".into()) + ]] + ); + let delivery_after_content = rows( + &conn, + "SELECT * FROM backend_job_deliveries ORDER BY delivery_id", + ); + conn.execute_batch("CREATE TEMP TRIGGER injected_final_receipt BEFORE INSERT ON main.__yoi_schema_migrations WHEN EXISTS(SELECT 1 FROM sqlite_schema WHERE name='backend_job_upgrade_history') BEGIN SELECT RAISE(ABORT,'injected final receipt'); END;").unwrap(); + assert!(store_owner::apply_migrations(&conn).is_err()); + assert!(!store_owner::table_exists(&conn, "backend_job_upgrade_history").unwrap()); + assert_eq!(original, pre_content_values(&conn, true)); + conn.execute_batch("DROP TRIGGER injected_final_receipt") + .unwrap(); + store_owner::apply_migrations(&conn).unwrap(); + assert_captured_cutover(&conn, &original, &delivery_after_content); + drop(conn); + drop(store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap()); + let conn = Connection::open(path).unwrap(); + assert_captured_cutover(&conn, &original, &delivery_after_content); +} + +fn assert_captured_cutover( + conn: &Connection, + original: &[Vec>], + deliveries: &[Vec], +) { + verify(conn).unwrap(); + assert_eq!(original, pre_content_values(conn, true)); + assert_eq!( + deliveries, + rows( + conn, + "SELECT * FROM backend_job_deliveries ORDER BY delivery_id" + ) + ); + assert_eq!( + rows( + conn, + "SELECT state,failure_category,completed_at FROM backend_jobs WHERE job_id='unstarted'" + ), + vec![vec![ + Value::Text("failed".into()), + Value::Text(FAILURE_CATEGORY.into()), + Value::Null + ]] + ); + assert_eq!( + rows( + conn, + "SELECT state,failure_category,completed_at,worker_cleanup_state,worker_cleanup_completed_at FROM backend_job_attempts WHERE attempt_id='unstarted-0'" + ), + vec![vec![ + Value::Text("failed".into()), + Value::Text(FAILURE_CATEGORY.into()), + Value::Null, + Value::Null, + Value::Null + ]] + ); + assert_eq!( + rows( + conn, + "SELECT result_json,result_digest FROM backend_jobs WHERE job_id='completed'" + ), + vec![vec![ + Value::Text("{\"saved\":true}".into()), + Value::Text("result-digest".into()) + ]] + ); + assert_eq!( + rows( + conn, + "SELECT result_json,result_digest FROM backend_job_attempts WHERE attempt_id='completed-0'" + ), + vec![vec![ + Value::Text("{\"saved_attempt\":true}".into()), + Value::Text("attempt-result-digest".into()) + ]] + ); + for table in ["backend_jobs", "backend_job_attempts"] { + let invalid: bool = conn.query_row(&format!("SELECT EXISTS(SELECT 1 FROM {table} a JOIN {} r ON a.workspace_id=r.workspace_id AND a.job_id=r.job_id {} WHERE r.state IN ('completed','failed') AND (a.state IS NOT r.state OR a.failure_category IS NOT r.failure_category OR a.failure_detail IS NOT r.failure_detail OR a.result_digest IS NOT r.result_digest))", + if table=="backend_jobs" {"backend_job_pre_content_history"} else {"backend_job_attempt_pre_content_history"}, + if table=="backend_job_attempts" {"AND a.attempt_id=r.attempt_id"} else {""}),[],|row|row.get(0)).unwrap(); + assert!(!invalid); + } +} + +#[test] +fn oldest_retained_row_fixtures_cross_the_real_content_hook_and_reopen_without_sweeping_new_jobs() { + for baseline in 50..=85 { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("oldest-with-rows.sqlite"); + let conn = legacy_fixture_to_85(&path, baseline); + let original = pre_content_values(&conn, false); + // Actual production step must call capture; this test does not call it. + store_owner::migrate_content_state_v85_to_v86(&conn).unwrap(); + assert_eq!(original, pre_content_values(&conn, true)); + let deliveries = rows( + &conn, + "SELECT * FROM backend_job_deliveries ORDER BY delivery_id", + ); + drop(conn); + drop(store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap()); + let conn = Connection::open(&path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + assert_captured_cutover(&conn, &original, &deliveries); + insert_job(&conn, "post-cutover", "pending"); + insert_attempt(&conn, "post-cutover", "post-cutover-attempt", 1, "reserved"); + let history = rows( + &conn, + "SELECT * FROM backend_job_upgrade_history ORDER BY job_id", + ); + drop(conn); + drop(store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap()); + let conn = Connection::open(&path).unwrap(); + assert_eq!( + history, + rows( + &conn, + "SELECT * FROM backend_job_upgrade_history ORDER BY job_id" + ) + ); + assert_eq!( + rows( + &conn, + "SELECT state FROM backend_jobs WHERE job_id='post-cutover'" + ), + vec![vec![Value::Text("pending".into())]] + ); + } +} + +#[test] +fn real_input_retirement_atomically_scrubs_raw_evidence_and_retries_after_failure() { + use crate::backend_job::JobInputRetirementReason; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("retire-raw.sqlite"); + let conn = legacy_fixture_to_85(&path, 85); + conn.execute_batch("UPDATE backend_jobs SET request_json=json_set(request_json,'$.input.body','unique-pre-content-body-to-erase') WHERE job_id='unstarted'").unwrap(); + store_owner::migrate_content_state_v85_to_v86(&conn).unwrap(); + drop(conn); + let store = store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let job = store + .get_backend_job("upgrade-workspace", "unstarted") + .unwrap() + .unwrap(); + let conn = Connection::open(&path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + let bodies = rows( + &conn, + "SELECT j.request_json,r.request_json FROM backend_jobs j JOIN backend_job_pre_content_history r USING(workspace_id,job_id) WHERE j.job_id='unstarted'", + ); + conn.execute_batch("CREATE TRIGGER injected_raw_retirement BEFORE UPDATE OF request_json ON backend_job_pre_content_history WHEN OLD.job_id='unstarted' BEGIN SELECT RAISE(ABORT,'injected raw retirement'); END;").unwrap(); + assert!( + store + .reclaim_backend_job_input( + "upgrade-workspace", + "unstarted", + &job.input_digest, + JobInputRetirementReason::ExplicitErasure + ) + .is_err() + ); + assert_eq!( + bodies, + rows( + &conn, + "SELECT j.request_json,r.request_json FROM backend_jobs j JOIN backend_job_pre_content_history r USING(workspace_id,job_id) WHERE j.job_id='unstarted'" + ) + ); + conn.execute_batch("DROP TRIGGER injected_raw_retirement") + .unwrap(); + store + .reclaim_backend_job_input( + "upgrade-workspace", + "unstarted", + &job.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!( + rows( + &conn, + "SELECT j.input_disposition,json_type(j.request_json,'$.input'),r.input_retired,json_type(r.request_json,'$.input') FROM backend_jobs j JOIN backend_job_pre_content_history r USING(workspace_id,job_id) WHERE j.job_id='unstarted'" + ), + vec![vec![ + Value::Text("erased".into()), + Value::Text("null".into()), + Value::Integer(1), + Value::Text("null".into()) + ]] + ); + verify(&conn).unwrap(); + drop(conn); + drop(store); + // The existing retirement owner performs secure_delete/VACUUM/checkpoint for + // the WHOLE database, including raw evidence, not only the live Job table. + let bytes = std::fs::read(&path).unwrap(); + assert!( + !bytes + .windows(b"unique-pre-content-body-to-erase".len()) + .any(|window| window == b"unique-pre-content-body-to-erase") + ); + drop(store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap()); + let conn = Connection::open(&path).unwrap(); + assert_eq!( + rows( + &conn, + "SELECT input_retired,json_type(request_json,'$.input') FROM backend_job_pre_content_history WHERE job_id='unstarted'" + ), + vec![vec![Value::Integer(1), Value::Text("null".into())]] + ); +} + +fn retained_fixture(path: &Path) -> Connection { + store_owner::tests::prepare_retained_schema(path, 113); + let conn = Connection::open(path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + seed_boundary(&conn); + conn +} + +fn apply_leaf(conn: &Connection) -> Result<()> { + conn.execute_batch("PRAGMA foreign_keys=OFF;")?; + let result = (|| { + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + install(&tx)?; + tx.commit()?; + Ok(()) + })(); + conn.execute_batch("PRAGMA foreign_keys=ON;")?; + result +} + +fn assert_retired(conn: &Connection) { + verify(conn).unwrap(); + assert_eq!( + rows(conn, "SELECT count(*) FROM backend_job_upgrade_history"), + vec![vec![Value::Integer(7)]] + ); + for (table, history, key) in [ + ( + "backend_jobs", + "backend_job_upgrade_history", + "a.job_id=h.job_id", + ), + ( + "backend_job_attempts", + "backend_job_attempt_upgrade_history", + "a.job_id=h.job_id AND a.attempt_id=h.attempt_id", + ), + ] { + let violations: i64 = conn.query_row(&format!("SELECT count(*) FROM {table} a JOIN {history} h ON a.workspace_id=h.workspace_id AND {key} + WHERE (h.state NOT IN ('completed','failed') AND (a.state!='failed' OR a.failure_category IS NOT ?1 OR a.failure_detail IS NOT ?2)) + OR (h.state IN ('completed','failed') AND (a.state!=h.state OR a.failure_category IS NOT h.failure_category OR a.failure_detail IS NOT h.failure_detail))"), + [FAILURE_CATEGORY,FAILURE_DETAIL], |row| row.get(0)).unwrap(); + assert_eq!(violations, 0, "{table}"); + } + assert_eq!( + rows( + conn, + "SELECT state,failure_category,failure_detail,completed_at FROM backend_job_attempt_upgrade_history WHERE attempt_id='uncertain-0'" + ), + vec![vec![ + Value::Text("unknown".into()), + Value::Text("old-attempt-cause".into()), + Value::Text("old attempt detail".into()), + Value::Text("old-end".into()) + ]] + ); + // Failure did not manufacture execution/cleanup end or Journal completion. + assert_eq!( + rows( + conn, + "SELECT completed_at,worker_cleanup_state,worker_cleanup_completed_at FROM backend_job_attempts WHERE attempt_id='running-0'" + ), + vec![vec![ + Value::Null, + Value::Text("executing".into()), + Value::Null + ]] + ); +} + +#[test] +fn all_old_attempts_are_retired_without_losing_terminal_results_inputs_or_execution_evidence() { + let dir = tempfile::tempdir().unwrap(); + let conn = retained_fixture(&dir.path().join("jobs.sqlite")); + let jobs = rows( + &conn, + &format!("SELECT {JOB_VALUES} FROM backend_jobs ORDER BY job_id"), + ); + let attempts = rows( + &conn, + &format!("SELECT {ATTEMPT_VALUES} FROM backend_job_attempts ORDER BY attempt_id"), + ); + let deliveries = rows( + &conn, + "SELECT * FROM backend_job_deliveries ORDER BY delivery_id", + ); + apply_leaf(&conn).unwrap(); + assert_retired(&conn); + assert_eq!( + jobs, + rows( + &conn, + &format!("SELECT {JOB_VALUES} FROM backend_jobs ORDER BY job_id") + ) + ); + assert_eq!( + attempts, + rows( + &conn, + &format!("SELECT {ATTEMPT_VALUES} FROM backend_job_attempts ORDER BY attempt_id") + ) + ); + assert_eq!( + deliveries, + rows( + &conn, + "SELECT * FROM backend_job_deliveries ORDER BY delivery_id" + ) + ); + assert_eq!( + rows( + &conn, + "SELECT input_disposition,json_type(request_json,'$.input') FROM backend_jobs WHERE job_id='failed'" + ), + vec![vec![ + Value::Text("erased".into()), + Value::Text("null".into()) + ]] + ); + assert!( + conn.execute_batch("UPDATE backend_job_upgrade_history SET state='pending'") + .is_err() + ); +} + +#[test] +fn delivery_states_and_same_id_reconciliation_survive_job_cutover() { + let dir = tempfile::tempdir().unwrap(); + let conn = retained_fixture(&dir.path().join("delivery.sqlite")); + let deliveries = rows( + &conn, + "SELECT * FROM backend_job_deliveries ORDER BY delivery_id", + ); + let ddl = rows( + &conn, + "SELECT type,name,sql FROM sqlite_schema WHERE tbl_name='backend_job_deliveries' ORDER BY type,name", + ); + apply_leaf(&conn).unwrap(); + assert_eq!( + deliveries, + rows( + &conn, + "SELECT * FROM backend_job_deliveries ORDER BY delivery_id" + ) + ); + assert_eq!( + ddl, + rows( + &conn, + "SELECT type,name,sql FROM sqlite_schema WHERE tbl_name='backend_job_deliveries' ORDER BY type,name" + ) + ); + // Same-ID notification acceptance recovery is not Job execution recovery. + conn.execute_batch("UPDATE backend_job_deliveries SET state='pending',failure_category=NULL,failure_detail=NULL,updated_at='reconcile' WHERE state IN ('sending','unknown'); + UPDATE backend_job_deliveries SET state='sending',updated_at='claim' WHERE delivery_id='delivery-4' AND state='pending'; + UPDATE backend_job_deliveries SET state='completed',delivered_at='accepted',updated_at='accepted' WHERE delivery_id='delivery-4' AND state='sending';").unwrap(); + assert_eq!( + rows( + &conn, + "SELECT delivery_id,job_id,attempt_id,state FROM backend_job_deliveries WHERE delivery_id='delivery-4'" + ), + vec![vec![ + Value::Text("delivery-4".into()), + Value::Text("running".into()), + Value::Text("running-0".into()), + Value::Text("completed".into()) + ]] + ); + // Retained completed results can still acquire their first notification row. + conn.execute_batch("INSERT INTO backend_job_deliveries(workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,created_at,updated_at) + VALUES('upgrade-workspace','later-delivery','completed','completed-0','target-runtime','target-worker','unknown','1','1');").unwrap(); + assert_retired(&conn); +} + +#[test] +fn terminal_receipts_cleanup_evidence_and_input_retirement_remain_independent() { + let dir = tempfile::tempdir().unwrap(); + let conn = retained_fixture(&dir.path().join("terminal.sqlite")); + for state in ["completed", "failed"] { + let id = format!("terminal-{state}"); + insert_job(&conn, &id, state); + insert_attempt(&conn, &id, &id, 1, state); + } + let receipts_query = "SELECT job_id,state,result_json,result_digest,failure_category,failure_detail,completed_at FROM backend_jobs WHERE job_id LIKE 'terminal-%' ORDER BY job_id"; + let attempts_query = "SELECT attempt_id,state,result_json,result_digest,failure_category,failure_detail,completed_at FROM backend_job_attempts WHERE job_id LIKE 'terminal-%' ORDER BY attempt_id"; + let receipts = rows(&conn, receipts_query); + let attempts = rows(&conn, attempts_query); + apply_leaf(&conn).unwrap(); + conn.execute_batch("UPDATE backend_jobs SET result_json=result_json,result_digest=result_digest,failure_category=failure_category,failure_detail=failure_detail,completed_at=completed_at WHERE job_id LIKE 'terminal-%'; + UPDATE backend_job_attempts SET state=state,result_json=result_json,result_digest=result_digest,failure_category=failure_category,failure_detail=failure_detail WHERE job_id LIKE 'terminal-%'; + UPDATE backend_job_attempts SET worker_cleanup_state='executing',worker_cleanup_updated_at='cleanup-claim' WHERE job_id LIKE 'terminal-%'; + UPDATE backend_job_attempts SET worker_cleanup_state='failed',worker_cleanup_failure_category='independent-cleanup-failure',worker_cleanup_failure_detail='independent evidence',worker_cleanup_updated_at='cleanup-failed' WHERE job_id LIKE 'terminal-%'; + UPDATE backend_job_attempts SET worker_cleanup_state='executing',worker_cleanup_updated_at='cleanup-reclaim' WHERE job_id LIKE 'terminal-%'; + UPDATE backend_job_attempts SET worker_cleanup_state='completed',worker_cleanup_completed_at='independently-supplied-end',worker_cleanup_updated_at='cleanup-end' WHERE job_id LIKE 'terminal-%'; + UPDATE backend_jobs SET input_disposition='reclaimed',request_json=json_set(request_json,'$.input',json('null')) WHERE job_id='terminal-completed'; + UPDATE backend_jobs SET input_disposition='invalidated',request_json=json_set(request_json,'$.input',json('null')) WHERE job_id='terminal-failed'; + UPDATE backend_jobs SET input_disposition='erased' WHERE job_id LIKE 'terminal-%';").unwrap(); + assert_eq!(receipts, rows(&conn, receipts_query)); + assert_eq!(attempts, rows(&conn, attempts_query)); + assert_eq!( + rows( + &conn, + "SELECT DISTINCT worker_cleanup_state,worker_cleanup_failure_category FROM backend_job_attempts WHERE job_id LIKE 'terminal-%'" + ), + vec![vec![ + Value::Text("completed".into()), + Value::Text("independent-cleanup-failure".into()) + ]] + ); + assert_eq!( + rows( + &conn, + "SELECT DISTINCT input_disposition,json_type(request_json,'$.input') FROM backend_jobs WHERE job_id LIKE 'terminal-%'" + ), + vec![vec![ + Value::Text("erased".into()), + Value::Text("null".into()) + ]] + ); + verify(&conn).unwrap(); + // This fixture supplies cleanup facts explicitly; migration did not derive + // them from Failed. It is not a process-termination proof or cleanup runner. +} + +#[test] +fn old_jobs_cannot_resume_retry_accept_late_results_or_allocate_continuation_attempts() { + let dir = tempfile::tempdir().unwrap(); + let conn = retained_fixture(&dir.path().join("jobs.sqlite")); + apply_leaf(&conn).unwrap(); + for sql in [ + "UPDATE backend_jobs SET state='pending' WHERE job_id='running'", + "UPDATE backend_jobs SET current_attempt=2 WHERE job_id='running'", + "UPDATE backend_jobs SET state='completed',result_json='{}',result_digest='late' WHERE job_id='running'", + "UPDATE backend_job_attempts SET state='dispatching' WHERE attempt_id='running-0'", + "UPDATE backend_job_attempts SET runtime_run_id='late-run' WHERE attempt_id='running-0'", + "INSERT INTO backend_job_attempts(workspace_id,job_id,attempt_id,attempt,input_digest,state,deadline_at,created_at,updated_at) VALUES('upgrade-workspace','running','continuation',3,'digest','reserved','deadline','1','1')", + ] { + assert!(conn.execute_batch(sql).is_err(), "accepted {sql}"); + } + // Input retirement remains its own explicit operation and does not grant retry. + conn.execute_batch("UPDATE backend_jobs SET input_disposition='erased',request_json=json_set(request_json,'$.input',json('null')) WHERE job_id='running';").unwrap(); + assert_retired(&conn); +} + +#[test] +fn failure_after_job_conversion_rolls_back_history_ddl_and_states_then_retry_succeeds() { + let dir = tempfile::tempdir().unwrap(); + let conn = retained_fixture(&dir.path().join("jobs.sqlite")); + conn.execute_batch("CREATE TRIGGER injected_attempt_failure BEFORE UPDATE OF state ON backend_job_attempts WHEN NEW.state='failed' BEGIN SELECT RAISE(ABORT,'injected migration failure'); END;").unwrap(); + let schema = rows( + &conn, + "SELECT type,name,sql FROM sqlite_schema ORDER BY type,name", + ); + let jobs = rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id"); + let attempts = rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY attempt_id", + ); + assert!(apply_leaf(&conn).is_err()); + assert_eq!( + schema, + rows( + &conn, + "SELECT type,name,sql FROM sqlite_schema ORDER BY type,name" + ) + ); + assert_eq!( + jobs, + rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id") + ); + assert_eq!( + attempts, + rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY attempt_id" + ) + ); + let enabled: bool = conn + .query_row("PRAGMA foreign_keys", [], |row| row.get(0)) + .unwrap(); + assert!(enabled); + conn.execute_batch("DROP TRIGGER injected_attempt_failure") + .unwrap(); + apply_leaf(&conn).unwrap(); + assert_retired(&conn); +} + +#[test] +fn failure_after_rebuild_rolls_back_constraints_and_retries_without_temp_table_leaks() { + let dir = tempfile::tempdir().unwrap(); + let conn = retained_fixture(&dir.path().join("jobs.sqlite")); + conn.execute_batch("PRAGMA foreign_keys=OFF;").unwrap(); + let before = rows( + &conn, + "SELECT type,name,sql FROM sqlite_schema ORDER BY type,name", + ); + { + let tx = conn.unchecked_transaction().unwrap(); + install(&tx).unwrap(); + // Represent failure persisting the owning migration receipt after DDL. + assert!( + tx.execute_batch("INSERT INTO missing_migration_receipt VALUES(1)") + .is_err() + ); + } + assert_eq!( + before, + rows( + &conn, + "SELECT type,name,sql FROM sqlite_schema ORDER BY type,name" + ) + ); + assert!( + rows( + &conn, + "SELECT name FROM sqlite_temp_schema WHERE name LIKE '__yoi_upgrade_%'" + ) + .is_empty() + ); + conn.execute_batch("PRAGMA foreign_keys=ON;").unwrap(); + apply_leaf(&conn).unwrap(); + assert_retired(&conn); +} + +#[test] +fn ordinary_reopen_and_accidental_reapply_do_not_sweep_new_jobs() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite"); + let conn = retained_fixture(&path); + apply_leaf(&conn).unwrap(); + insert_job(&conn, "post-upgrade", "pending"); + insert_attempt(&conn, "post-upgrade", "new-attempt", 1, "reserved"); + let history = rows( + &conn, + "SELECT * FROM backend_job_upgrade_history ORDER BY job_id", + ); + drop(conn); + let conn = Connection::open(path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + verify(&conn).unwrap(); + assert!(apply_leaf(&conn).is_err()); + assert_eq!( + history, + rows( + &conn, + "SELECT * FROM backend_job_upgrade_history ORDER BY job_id" + ) + ); + assert_eq!( + rows( + &conn, + "SELECT state FROM backend_jobs WHERE job_id='post-upgrade'" + ), + vec![vec![Value::Text("pending".into())]] + ); + conn.execute_batch( + "UPDATE backend_job_attempts SET state='dispatching' WHERE attempt_id='new-attempt'", + ) + .unwrap(); + assert!(conn.execute_batch("UPDATE backend_job_attempts SET job_id='unstarted',attempt=2 WHERE attempt_id='new-attempt'").is_err()); + assert!( + conn.execute_batch("UPDATE backend_jobs SET state='unknown' WHERE job_id='post-upgrade'") + .is_err() + ); +} + +#[test] +fn install_requires_an_explicit_migration_transaction_and_pretransaction_fk_window() { + let dir = tempfile::tempdir().unwrap(); + let conn = retained_fixture(&dir.path().join("jobs.sqlite")); + assert!(install(&conn).is_err()); + let tx = conn.unchecked_transaction().unwrap(); + assert!(install(&tx).is_err()); + assert!(!store_owner::table_exists(&tx, "backend_job_upgrade_history").unwrap()); +} + +// These tests intentionally fail if only module registration is added: the +// production ordered runner, fresh creation and physical recognizer must also +// be connected. No ignored feature or duplicate test-only migration is used. +#[test] +fn retained_baselines_reach_upgrade_via_the_real_runner_and_reopen_keeps_new_jobs_live() { + for baseline in 50..=113 { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("retained.sqlite"); + if baseline <= 51 { + store_owner::tests::prepare_schema_v50(&path, None); + if baseline == 51 { + let conn = Connection::open(&path).unwrap(); + store_owner::migrate_workspace_runtime_bindings_v50_to_v51(&conn).unwrap(); + } + } else { + store_owner::tests::prepare_retained_schema(&path, baseline); + } + if baseline >= 86 { + let conn = Connection::open(&path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + seed_boundary(&conn); + } + let store = store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap(); + drop(store); + let conn = Connection::open(&path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + verify(&conn).unwrap(); + if baseline >= 86 { + assert_retired(&conn); + // Already-converted source rows have no recoverable pre-86 wire. + assert_eq!( + rows( + &conn, + "SELECT count(*) FROM backend_job_pre_content_history" + ), + vec![vec![Value::Integer(0)]] + ); + } else { + seed_workspace(&conn); + } + insert_job(&conn, "post-upgrade", "pending"); + insert_attempt(&conn, "post-upgrade", "post-upgrade-attempt", 1, "reserved"); + let history = rows( + &conn, + "SELECT * FROM backend_job_upgrade_history ORDER BY job_id", + ); + let receipts = rows( + &conn, + "SELECT * FROM __yoi_schema_migrations ORDER BY version", + ); + drop(conn); + drop(store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap()); + let conn = Connection::open(&path).unwrap(); + assert_eq!( + history, + rows( + &conn, + "SELECT * FROM backend_job_upgrade_history ORDER BY job_id" + ) + ); + assert_eq!( + receipts, + rows( + &conn, + "SELECT * FROM __yoi_schema_migrations ORDER BY version" + ) + ); + assert_eq!( + rows( + &conn, + "SELECT state FROM backend_jobs WHERE job_id='post-upgrade'" + ), + vec![vec![Value::Text("pending".into())]] + ); + } +} + +#[test] +fn store_retry_and_recovery_exclude_migrated_jobs_but_admit_a_new_request() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("api.sqlite"); + store_owner::tests::prepare_retained_schema(&path, 113); + let conn = Connection::open(&path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + seed_workspace(&conn); + insert_job(&conn, "legacy-retry", "pending"); + let old_attempt = crate::backend_job::attempt_id("legacy-retry", 1); + insert_attempt(&conn, "legacy-retry", &old_attempt, 1, "reserved"); + conn.execute( + "UPDATE backend_job_attempts SET input_digest=?1 WHERE attempt_id=?2", + params![request("legacy-retry").input_digest().unwrap(), old_attempt], + ) + .unwrap(); + drop(conn); + let store = store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let old = store + .get_backend_job("upgrade-workspace", "legacy-retry") + .unwrap() + .unwrap(); + assert_eq!(old.failure_category.as_deref(), Some(FAILURE_CATEGORY)); + let retired_attempt = store + .get_backend_job_attempt("upgrade-workspace", "legacy-retry", &old_attempt) + .unwrap() + .unwrap(); + assert!( + store + .probe_backend_job_recovery("upgrade-workspace", &retired_attempt) + .is_err() + ); + let saved_domain_result = crate::backend_job::BackendJobResultSubmission { + job_id: "legacy-retry".into(), + attempt_id: old_attempt.clone(), + input_digest: retired_attempt.input_digest.clone(), + result: serde_json::json!({"saved":true}), + }; + assert!( + store + .reconcile_backend_job_domain_result( + "upgrade-workspace", + &retired_attempt, + &saved_domain_result, + "2026-11-04T00:00:00Z" + ) + .is_err() + ); + // The old attempt has no Worker and no cleanup debt. Retry must still be + // rejected, not merely hidden by an unfinished-cleanup or attempt-budget gate. + assert!( + store + .reserve_backend_job_retry("upgrade-workspace", "legacy-retry", "2026-11-04T00:00:00Z") + .is_err() + ); + assert!( + store + .list_recoverable_backend_job_attempts("upgrade-workspace", 100) + .unwrap() + .is_empty() + ); + assert!( + store + .list_reserved_backend_job_attempts("upgrade-workspace", 100) + .unwrap() + .is_empty() + ); + let next = store + .reserve_backend_job( + "upgrade-workspace", + &request("new-live"), + "2026-11-04T00:00:00Z", + ) + .unwrap(); + assert_eq!(next.job.state, crate::backend_job::BackendJobState::Pending); + drop(store); + let reopened = store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let recoverable = reopened + .list_recoverable_backend_job_attempts("upgrade-workspace", 100) + .unwrap(); + assert_eq!(recoverable.len(), 1); + assert_eq!(recoverable[0].job_id, "new-live"); + assert_eq!( + reopened + .get_backend_job("upgrade-workspace", "legacy-retry") + .unwrap() + .unwrap(), + old + ); +} + +#[test] +fn store_delivery_reconciliation_keeps_the_original_delivery_id_after_cutover() { + use crate::backend_job::BackendJobDeliveryState; + use worker_runtime::identity::RuntimeWorkerRef; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("delivery-api.sqlite"); + drop(retained_fixture(&path)); + let store = store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let target = RuntimeWorkerRef::new("target-runtime", "target-worker"); + let recovered = store + .recover_backend_job_deliveries("upgrade-workspace", "2026-11-04T00:00:00Z") + .unwrap(); + assert!( + recovered + .iter() + .any(|record| record.delivery_id == "delivery-4" + && record.state == BackendJobDeliveryState::Pending) + ); + let (claimed, was_claimed) = store + .reserve_backend_job_delivery( + "upgrade-workspace", + "delivery-4", + "running", + &target, + "2026-11-04T00:00:01Z", + ) + .unwrap(); + assert!(was_claimed); + assert_eq!(claimed.delivery_id, "delivery-4"); + let finished = store + .finish_backend_job_delivery( + "upgrade-workspace", + "delivery-4", + BackendJobDeliveryState::Completed, + None, + None, + "2026-11-04T00:00:02Z", + ) + .unwrap(); + let replay = store + .finish_backend_job_delivery( + "upgrade-workspace", + "delivery-4", + BackendJobDeliveryState::Completed, + None, + None, + "2026-11-04T00:00:03Z", + ) + .unwrap(); + assert_eq!(finished, replay); + drop(store); + let conn = Connection::open(path).unwrap(); + assert_eq!( + rows(&conn, "SELECT count(*) FROM backend_job_deliveries"), + vec![vec![Value::Integer(5)]] + ); + assert_retired(&conn); +} + +#[test] +fn store_terminal_replays_and_retirement_preserve_old_causes_and_results() { + use crate::backend_job::{ + BackendJobAttemptState, BackendJobResultSubmission, JobInputRetirementReason, + }; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("terminal-api.sqlite"); + store_owner::tests::prepare_retained_schema(&path, 113); + let conn = Connection::open(&path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + seed_workspace(&conn); + let value = serde_json::json!({"already":"accepted"}); + let (result_digest, result_json) = + job::result_digest(&value, request("x").limits.max_result_bytes).unwrap(); + for state in ["completed", "failed"] { + let id = format!("terminal-api-{state}"); + insert_job(&conn, &id, state); + let attempt = crate::backend_job::attempt_id(&id, 1); + insert_attempt(&conn, &id, &attempt, 1, state); + conn.execute("UPDATE backend_job_attempts SET input_digest=?1,runtime_run_id='retained-run',worker_cleanup_state='completed',worker_cleanup_completed_at='retained-cleanup-end' WHERE attempt_id=?2", + params![request(&id).input_digest().unwrap(),attempt]).unwrap(); + if state == "completed" { + conn.execute( + "UPDATE backend_jobs SET result_json=?1,result_digest=?2 WHERE job_id=?3", + params![result_json, result_digest, id], + ) + .unwrap(); + conn.execute( + "UPDATE backend_job_attempts SET result_json=?1,result_digest=?2 WHERE job_id=?3", + params![result_json, result_digest, id], + ) + .unwrap(); + } + } + let query = "SELECT job_id,state,result_json,result_digest,failure_category,failure_detail,completed_at FROM backend_jobs ORDER BY job_id"; + let before = rows(&conn, query); + drop(conn); + let store = store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let id = "terminal-api-completed"; + let attempt_id = crate::backend_job::attempt_id(id, 1); + let attempt = store + .get_backend_job_attempt("upgrade-workspace", id, &attempt_id) + .unwrap() + .unwrap(); + let replay = store + .accept_backend_job_result( + "upgrade-workspace", + attempt.worker.as_ref().unwrap(), + &BackendJobResultSubmission { + job_id: id.into(), + attempt_id, + input_digest: request(id).input_digest().unwrap(), + result: value, + }, + "2026-11-04T00:00:00Z", + ) + .unwrap(); + assert!(replay.replayed); + let failed_id = "terminal-api-failed"; + let failed_attempt = crate::backend_job::attempt_id(failed_id, 1); + let failed = store + .finish_backend_job_attempt( + "upgrade-workspace", + failed_id, + &failed_attempt, + BackendJobAttemptState::Failed, + "old-attempt-cause", + "old attempt detail", + "2026-11-04T00:00:01Z", + ) + .unwrap(); + assert_eq!( + failed.failure_category.as_deref(), + Some("old-attempt-cause") + ); + for (id, reason) in [ + (id, JobInputRetirementReason::Completed), + (failed_id, JobInputRetirementReason::LegacyInvalidation), + ] { + store + .reclaim_backend_job_input( + "upgrade-workspace", + id, + &request(id).input_digest().unwrap(), + reason, + ) + .unwrap(); + store + .reclaim_backend_job_input( + "upgrade-workspace", + id, + &request(id).input_digest().unwrap(), + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + } + drop(store); + let conn = Connection::open(path).unwrap(); + assert_eq!(before, rows(&conn, query)); + assert_eq!( + rows( + &conn, + "SELECT DISTINCT input_disposition,json_type(request_json,'$.input') FROM backend_jobs" + ), + vec![vec![ + Value::Text("erased".into()), + Value::Text("null".into()) + ]] + ); + verify(&conn).unwrap(); +} + +#[test] +fn owning_runner_receipt_failure_rolls_back_conversion_and_retry_commits_once() { + let dir = tempfile::tempdir().unwrap(); + let conn = retained_fixture(&dir.path().join("receipt.sqlite")); + let jobs = rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id"); + let attempts = rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY attempt_id", + ); + conn.execute_batch("CREATE TEMP TRIGGER injected_upgrade_receipt_failure BEFORE INSERT ON main.__yoi_schema_migrations + WHEN EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='backend_job_upgrade_history') + BEGIN SELECT RAISE(ABORT,'injected owning upgrade receipt failure'); END;").unwrap(); + assert!(store_owner::apply_migrations(&conn).is_err()); + assert!(!store_owner::table_exists(&conn, "backend_job_upgrade_history").unwrap()); + assert_eq!( + jobs, + rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id") + ); + assert_eq!( + attempts, + rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY attempt_id" + ) + ); + let enabled: bool = conn + .query_row("PRAGMA foreign_keys", [], |row| row.get(0)) + .unwrap(); + assert!(enabled); + conn.execute_batch("DROP TRIGGER injected_upgrade_receipt_failure") + .unwrap(); + store_owner::apply_migrations(&conn).unwrap(); + assert_retired(&conn); + let receipts = rows( + &conn, + "SELECT * FROM __yoi_schema_migrations ORDER BY version", + ); + store_owner::apply_migrations(&conn).unwrap(); + assert_eq!( + receipts, + rows( + &conn, + "SELECT * FROM __yoi_schema_migrations ORDER BY version" + ) + ); +} + +#[test] +fn both_103_receipts_and_baseline_only_identities_use_the_real_upgrade_runner() { + for job_variant in [false, true] { + for baseline_only in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("branch.sqlite"); + store_owner::tests::prepare_retained_schema(&path, 102); + let conn = Connection::open(&path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + if job_variant { + (store_owner::JOB_INPUT_RETENTION_SCHEMA_103.apply)(&conn).unwrap(); + } else { + store_owner::migrate_passkey_results_v102_to_v103(&conn).unwrap(); + } + if baseline_only { + conn.execute_batch("DELETE FROM __yoi_schema_migrations; INSERT INTO __yoi_schema_migrations(version,name) VALUES(103,'workspace schema baseline');").unwrap(); + } + seed_boundary(&conn); + let before = rows( + &conn, + "SELECT * FROM __yoi_schema_migrations WHERE version<=103 ORDER BY version", + ); + drop(conn); + drop(store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap()); + let conn = Connection::open(path).unwrap(); + assert_retired(&conn); + assert_eq!( + before, + rows( + &conn, + "SELECT * FROM __yoi_schema_migrations WHERE version<=103 ORDER BY version" + ) + ); + } + } +} + +#[test] +fn fresh_creation_and_empty_retained_upgrade_have_identical_job_ddl() { + let dir = tempfile::tempdir().unwrap(); + // Each catalog owns its sibling feature-storage identity directory. Sharing + // a parent would ask the same owner to replace an already-bound catalog. + let migrated = dir.path().join("migrated/server.sqlite"); + let fresh = dir.path().join("fresh/server.sqlite"); + std::fs::create_dir_all(migrated.parent().unwrap()).unwrap(); + std::fs::create_dir_all(fresh.parent().unwrap()).unwrap(); + store_owner::tests::prepare_retained_schema(&migrated, 113); + drop(store_owner::SqliteWorkspaceStore::open_for_test(&migrated).unwrap()); + drop(store_owner::SqliteWorkspaceStore::open_for_test(&fresh).unwrap()); + // Column declaration order can differ after retained ALTERs. Compare values, + // keys, FK actions and index columns, not SQLite's source formatting. + fn objects(path: &Path) -> Vec> { + let conn = Connection::open(path).unwrap(); + let tables = rows( + &conn, + "SELECT name FROM sqlite_schema WHERE type='table' AND name LIKE 'backend_job%' ORDER BY name", + ); + let mut signature = tables.clone(); + for table in tables { + let Value::Text(name) = &table[0] else { + unreachable!() + }; + let name = format!("'{}'", name.replace('\'', "''")); + for query in [ + format!( + "SELECT name,type,\"notnull\",dflt_value,pk FROM pragma_table_info({name}) ORDER BY name" + ), + format!( + "SELECT \"table\",\"from\",\"to\",on_update,on_delete,match,seq FROM pragma_foreign_key_list({name}) ORDER BY \"table\",\"from\",\"to\",seq" + ), + format!( + "SELECT i.\"unique\",i.partial,(SELECT group_concat(name,',') FROM (SELECT name FROM pragma_index_info(i.name) ORDER BY seqno)) FROM pragma_index_list({name}) i ORDER BY 1,2,3" + ), + ] { + signature.extend(rows(&conn, &query)); + } + } + signature.extend(rows(&conn,"SELECT type,name,replace(replace(replace(sql,char(10),''),char(13),''),' ','') FROM sqlite_schema WHERE name LIKE 'backend_job%' AND type IN ('trigger','index') AND sql IS NOT NULL ORDER BY type,name")); + verify(&conn).unwrap(); + signature + } + assert_eq!(objects(&migrated), objects(&fresh)); +} + +// The storage-format fixture owner can call this for EVERY real published stage +// and convergence predecessor. It snapshots all rows, so policy/root/erasure and +// existing T820 history/cleanup invocation evidence must survive byte-for-byte. +pub(super) fn assert_published_fixture(conn: &Connection) { + let tables = conn.prepare("SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name").unwrap() + .query_map([],|row| row.get::<_,String>(0)).unwrap() + .collect::>>().unwrap(); + let saved = tables + .iter() + .map(|name| { + let columns = store_owner::table_columns(conn, name) + .unwrap() + .into_iter() + .filter(|column| { + !["backend_jobs", "backend_job_attempts"].contains(&name.as_str()) + || !["state", "failure_category", "failure_detail"] + .contains(&column.as_str()) + }) + .map(|column| quoted(&column)) + .collect::>() + .join(","); + let query = format!("SELECT {columns} FROM {} ORDER BY {columns}", quoted(name)); + let before = rows(conn, &query); + (name.clone(), query, before) + }) + .collect::>(); + apply_leaf(conn).unwrap(); + verify(conn).unwrap(); + for (name, query, before) in saved { + assert_eq!(before, rows(conn, &query), "changed {name}"); + } +} + +#[test] +fn captured_history_is_accepted_at_each_real_retained_prefix_and_cannot_be_forged_later() { + let dir = tempfile::tempdir().unwrap(); + let conn = legacy_fixture_to_85(&dir.path().join("prefixes.sqlite"), 85); + let original = pre_content_values(&conn, false); + for migration in store_owner::MIGRATIONS + .iter() + .filter(|migration| migration.version >= 86 && migration.version <= 122) + { + (migration.apply)(&conn).unwrap(); + // In particular 103 keeps its original branch name and all later + // prefixes recognize genuine raw capture without relabeling receipts. + super::super::storage_formats::verify_history(&conn, migration.version).unwrap(); + assert_eq!(original, pre_content_values(&conn, true)); + let error=conn.execute_batch("INSERT INTO backend_job_pre_content_history SELECT * FROM backend_job_pre_content_history LIMIT 1").unwrap_err(); + assert!( + error.to_string().contains("schema-85 transaction"), + "{error}" + ); + } + store_owner::apply_migrations(&conn).unwrap(); + let receipts = rows( + &conn, + "SELECT version,name FROM __yoi_schema_migrations ORDER BY version", + ); + assert_eq!( + receipts.last().unwrap(), + &vec![Value::Integer(123), Value::Text(MIGRATION_NAME.into())] + ); + assert_eq!(original, pre_content_values(&conn, true)); + assert!(conn.execute_batch("INSERT INTO backend_job_pre_content_history SELECT * FROM backend_job_pre_content_history LIMIT 1").is_err()); +} + +#[test] +fn raw_capture_shape_drift_is_rejected_before_the_runner_writes_or_retires_jobs() { + let dir = tempfile::tempdir().unwrap(); + let conn = legacy_fixture_to_85(&dir.path().join("raw-drift.sqlite"), 85); + store_owner::migrate_content_state_v85_to_v86(&conn).unwrap(); + conn.execute_batch("CREATE INDEX forged_raw_index ON backend_job_pre_content_history(job_id)") + .unwrap(); + let jobs = rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id"); + let receipts = rows( + &conn, + "SELECT * FROM __yoi_schema_migrations ORDER BY version", + ); + assert!(store_owner::apply_migrations(&conn).is_err()); + assert_eq!( + jobs, + rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id") + ); + assert_eq!( + receipts, + rows( + &conn, + "SELECT * FROM __yoi_schema_migrations ORDER BY version" + ) + ); + assert!(!store_owner::table_exists(&conn, "backend_job_upgrade_history").unwrap()); + conn.execute_batch("DROP INDEX forged_raw_index").unwrap(); + store_owner::apply_migrations(&conn).unwrap(); + verify(&conn).unwrap(); +} + +#[test] +fn fresh_install_has_empty_histories_and_rejects_legacy_live_state_at_the_sql_boundary() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("fresh-live.sqlite"); + drop(store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap()); + let conn = Connection::open(path).unwrap(); + store_owner::configure_sqlite(&conn).unwrap(); + for table in [ + "backend_job_pre_content_history", + "backend_job_attempt_pre_content_history", + "backend_job_delivery_pre_content_history", + "backend_job_upgrade_history", + "backend_job_attempt_upgrade_history", + ] { + assert_eq!( + rows(&conn, &format!("SELECT count(*) FROM {table}")), + vec![vec![Value::Integer(0)]] + ); + } + seed_workspace(&conn); + insert_job(&conn, "new-live", "pending"); + insert_attempt(&conn, "new-live", "new-live-attempt", 1, "reserved"); + assert!( + conn.execute_batch("UPDATE backend_jobs SET state='unknown' WHERE job_id='new-live'") + .is_err() + ); + assert!( + conn.execute_batch( + "UPDATE backend_job_attempts SET state='unknown' WHERE job_id='new-live'" + ) + .is_err() + ); + assert_eq!( + rows( + &conn, + "SELECT state FROM backend_jobs WHERE job_id='new-live'" + ), + vec![vec![Value::Text("pending".into())]] + ); +} + +#[test] +fn actual_runner_conversion_fault_restores_pragma_history_and_rows_then_reopens_new_job() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("actual-runner-fault.sqlite"); + let conn = retained_fixture(&path); + for migration in store_owner::MIGRATIONS + .iter() + .filter(|m| m.version > 113 && m.version <= 122) + { + (migration.apply)(&conn).unwrap(); + } + let jobs = rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id"); + let attempts = rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY job_id,attempt_id", + ); + let receipts = rows( + &conn, + "SELECT * FROM __yoi_schema_migrations ORDER BY version", + ); + // A connection-local fault is not a forged physical storage family. It fires + // after Job conversion, in the actual ordered runner's attempt update. + conn.execute_batch("CREATE TEMP TRIGGER actual_attempt_fault BEFORE UPDATE OF state ON main.backend_job_attempts WHEN NEW.state='failed' AND OLD.state NOT IN ('completed','failed') BEGIN SELECT RAISE(ABORT,'actual attempt conversion fault'); END;").unwrap(); + let error = store_owner::apply_migrations(&conn).unwrap_err(); + assert!( + error + .to_string() + .contains("actual attempt conversion fault"), + "{error}" + ); + assert_eq!( + jobs, + rows(&conn, "SELECT * FROM backend_jobs ORDER BY job_id") + ); + assert_eq!( + attempts, + rows( + &conn, + "SELECT * FROM backend_job_attempts ORDER BY job_id,attempt_id" + ) + ); + assert_eq!( + receipts, + rows( + &conn, + "SELECT * FROM __yoi_schema_migrations ORDER BY version" + ) + ); + assert!(!store_owner::table_exists(&conn, "backend_job_upgrade_history").unwrap()); + assert!( + conn.query_row("PRAGMA foreign_keys", [], |row| row.get::<_, bool>(0)) + .unwrap() + ); + conn.execute_batch("DROP TRIGGER actual_attempt_fault") + .unwrap(); + store_owner::apply_migrations(&conn).unwrap(); + assert_retired(&conn); // includes unchanged Completed/Failed and all historical attempts + let saved = rows( + &conn, + "SELECT * FROM backend_job_attempt_upgrade_history ORDER BY job_id,attempt_id", + ); + drop(conn); + let store = store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let fresh = store + .reserve_backend_job( + "upgrade-workspace", + &request("after-fault-retry"), + "2026-11-04T00:00:00Z", + ) + .unwrap(); + drop(store); + let store = store_owner::SqliteWorkspaceStore::open_for_test(&path).unwrap(); + assert_eq!( + store + .get_backend_job_attempt( + "upgrade-workspace", + "after-fault-retry", + &fresh.attempt.attempt_id + ) + .unwrap() + .unwrap(), + fresh.attempt + ); + drop(store); + let conn = Connection::open(path).unwrap(); + assert_eq!( + saved, + rows( + &conn, + "SELECT * FROM backend_job_attempt_upgrade_history ORDER BY job_id,attempt_id" + ) + ); + assert_retired(&conn); +} diff --git a/crates/workspace-server/src/db_executor.rs b/crates/workspace-server/src/db_executor.rs index 344bf98e6..8d39e0293 100644 --- a/crates/workspace-server/src/db_executor.rs +++ b/crates/workspace-server/src/db_executor.rs @@ -53,10 +53,31 @@ tokio::task_local! { static OPERATION_LIFETIME: Arc; } +struct NestedOperationLifetime { + outer: Arc, + inner: Arc, +} + +impl DbOperationLifetime for NestedOperationLifetime { + fn retain(&self) -> Box { + Box::new((self.outer.retain(), self.inner.retain())) + } +} + pub(crate) async fn with_operation_lifetime( lifetime: Arc, future: F, ) -> F::Output { + // An inner attempt scope must not replace its Workspace admission lease. Carry both + // sources through actual SQL, synchronous continuations, and discarded-result cleanup. + let lifetime = OPERATION_LIFETIME + .try_with(|outer| { + Arc::new(NestedOperationLifetime { + outer: outer.clone(), + inner: lifetime.clone(), + }) as Arc + }) + .unwrap_or(lifetime); OPERATION_LIFETIME.scope(lifetime, future).await } @@ -591,6 +612,94 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + async fn nested_workspace_and_attempt_lifetimes_survive_sql_child_and_discarded_reply_cleanup() + { + struct ReplyCleanup { + entered: Option>, + finish: mpsc::Receiver<()>, + } + impl Drop for ReplyCleanup { + fn drop(&mut self) { + self.entered.take().unwrap().send(()).unwrap(); + self.finish.recv_timeout(Duration::from_secs(5)).unwrap(); + } + } + let (dir, owner, mut parent_conn) = database(); + let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap(); + let parent = executor(1); + let child = executor(1); + let child_owner = child.clone(); + let workspace = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let attempt = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let outer: Arc = Arc::new(Lease(workspace.clone())); + let inner: Arc = Arc::new(Lease(attempt.clone())); + let (entered, running) = oneshot::channel(); + let (release, permit) = mpsc::channel(); + let (reply_dropping, dropping) = oneshot::channel(); + let (finish_reply, finish) = mpsc::channel(); + let mut operation = Box::pin(with_operation_lifetime( + outer, + with_operation_lifetime( + inner, + parent.run(move || -> Result<()> { + entered.send(()).unwrap(); + permit.recv_timeout(Duration::from_secs(5)).unwrap(); + let tx = + parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute("INSERT INTO writes VALUES (1)", [])?; + tx.commit()?; + let reply = child_owner.submit(move || { + let tx = child_conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let committed: i64 = tx + .query_row("SELECT count(*) FROM writes", [], |r| r.get(0)) + .unwrap(); + assert_eq!(committed, 1); + tx.execute("INSERT INTO writes VALUES (2)", []).unwrap(); + tx.commit().unwrap(); + // Failed reply delivery drops this result outside the SQL transaction. + Ok::<_, Error>(ReplyCleanup { + entered: Some(reply_dropping), + finish, + }) + })?; + drop(reply); + Ok(()) + }), + ), + )); + assert!(futures::poll!(&mut operation).is_pending()); + observed(running).await.unwrap(); + drop(operation); + for count in [&workspace, &attempt] { + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + } + release.send(()).unwrap(); + observed(dropping).await.unwrap(); + observed(parent.drain()).await.unwrap(); + for count in [&workspace, &attempt] { + assert_eq!( + count.load(std::sync::atomic::Ordering::SeqCst), + 1, + "both owners must survive the parent and cancelled waiter" + ); + } + let mut drain = Box::pin(child.drain()); + assert!(futures::poll!(&mut drain).is_pending()); + assert_eq!( + owner + .query_row("SELECT count(*) FROM writes", [], |r| r.get::<_, i64>(0)) + .unwrap(), + 2 + ); + finish_reply.send(()).unwrap(); + observed(drain).await.unwrap(); + assert_eq!(workspace.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!(attempt.load(std::sync::atomic::Ordering::SeqCst), 0); + } + #[tokio::test] async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() { let (_dir, owner, mut worker) = database(); diff --git a/crates/workspace-server/src/frozen_content_state_migration.rs b/crates/workspace-server/src/frozen_content_state_migration.rs index 9fb7ae16e..fda68ad55 100644 --- a/crates/workspace-server/src/frozen_content_state_migration.rs +++ b/crates/workspace-server/src/frozen_content_state_migration.rs @@ -95,6 +95,7 @@ fn migrate_content_state_v85_to_v86(conn: &Connection) -> Result<()> { // Converge on the canonical schema for both legacy creation paths. tx.execute_batch("CREATE INDEX IF NOT EXISTS worker_create_reservations_worker ON worker_create_reservations(workspace_id, worker_id);")?; if column_exists(&tx, "backend_jobs", "input_revision")? { + backend_job_upgrade::capture_before_content_cutover(&tx)?; let jobs = { let mut stmt = tx.prepare("SELECT workspace_id,job_id,request_json FROM backend_jobs")?; diff --git a/crates/workspace-server/src/frozen_content_state_tests.rs b/crates/workspace-server/src/frozen_content_state_tests.rs index b2003d2ca..006b3619c 100644 --- a/crates/workspace-server/src/frozen_content_state_tests.rs +++ b/crates/workspace-server/src/frozen_content_state_tests.rs @@ -426,25 +426,25 @@ fn content_cutover_keeps_each_evaluation_provenance_and_deduplicates_only_identi } #[test] -fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_it() { +fn content_cutover_version_abort_retains_prior_cleanup_and_uncertain_resource_owners() { for (state, expected_job, expected_attempt, expected_cleanup) in [ ( "reserved", BackendJobState::Failed, BackendJobAttemptState::Failed, - Some(BackendJobWorkerCleanupState::Completed), + None, ), ( "dispatching", - BackendJobState::Unknown, - BackendJobAttemptState::Unknown, - Some(BackendJobWorkerCleanupState::Pending), + BackendJobState::Failed, + BackendJobAttemptState::Failed, + None, ), ( "dispatched", - BackendJobState::Unknown, - BackendJobAttemptState::Unknown, - Some(BackendJobWorkerCleanupState::Pending), + BackendJobState::Failed, + BackendJobAttemptState::Failed, + None, ), ] { let dir = tempfile::tempdir().unwrap(); @@ -490,15 +490,11 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ let next = store .reserve_backend_job("space", &next, "2026-08-15T00:00:00Z") .unwrap(); - assert_eq!(next.resource_reused, state != "reserved"); - assert_eq!( - next.job.request.job_id, - if state == "reserved" { - "next-job" - } else { - "legacy-job" - } + assert!( + next.resource_reused, + "migration failure is not termination evidence" ); + assert_eq!(next.job.request.job_id, "legacy-job"); } } diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 2c1739cba..3674e36f9 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -1437,6 +1437,45 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { Ok(None) } + /// Original allocation identity only, not execution or input acceptance. + fn lookup_job_worker_creation( + &self, + _worker_id: &str, + _create_fingerprint: &str, + _binding: &worker_runtime::catalog::BackendJobExecutionBinding, + ) -> Result, RuntimeRegistryError> { + Err(RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: self.runtime_id().to_owned(), + code: "worker_creation_receipt_unsupported".into(), + message: "Runtime does not support original Job Worker identity lookup".into(), + }) + } + + fn backend_job_termination( + &self, + _worker_id: &str, + ) -> Result, RuntimeRegistryError> + { + Err(RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: self.runtime_id().to_owned(), + code: "worker_termination_receipt_unsupported".into(), + message: "Runtime does not support actual scope-termination receipt lookup".into(), + }) + } + + /// Read-only exact receipt lookup. Unsupported lookup is not proof of rejection. + fn reconcile_worker_input( + &self, + _worker_id: &str, + _request: WorkerInputRequest, + ) -> Result { + Err(RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: self.runtime_id().to_owned(), + code: "worker_input_receipt_unsupported".into(), + message: "Runtime does not support read-only submission receipt lookup".into(), + }) + } + fn send_input(&self, worker_id: &str, _request: WorkerInputRequest) -> WorkerInputResult { WorkerInputResult { state: InternalWorkerOperationState::Unsupported, @@ -2075,6 +2114,40 @@ impl RuntimeRegistry { runtime.send_protocol_method(worker_id, method) } + pub fn lookup_job_worker_creation( + &self, + worker: &RuntimeWorkerRef, + create_fingerprint: &str, + binding: &worker_runtime::catalog::BackendJobExecutionBinding, + ) -> Result, RuntimeRegistryError> { + validate_backend_identifier("runtime_id", worker.runtime_id.as_str())?; + validate_backend_identifier("worker_id", worker.worker_id.as_str())?; + self.runtime(worker.runtime_id.as_str())? + .lookup_job_worker_creation(worker.worker_id.as_str(), create_fingerprint, binding) + } + + pub fn backend_job_termination( + &self, + worker: &RuntimeWorkerRef, + ) -> Result, RuntimeRegistryError> + { + validate_backend_identifier("runtime_id", worker.runtime_id.as_str())?; + validate_backend_identifier("worker_id", worker.worker_id.as_str())?; + self.runtime(worker.runtime_id.as_str())? + .backend_job_termination(worker.worker_id.as_str()) + } + + pub fn reconcile_worker_input( + &self, + worker: &RuntimeWorkerRef, + request: WorkerInputRequest, + ) -> Result { + validate_backend_identifier("runtime_id", worker.runtime_id.as_str())?; + validate_backend_identifier("worker_id", worker.worker_id.as_str())?; + self.runtime(worker.runtime_id.as_str())? + .reconcile_worker_input(worker.worker_id.as_str(), request) + } + pub fn send_input( &self, worker: &RuntimeWorkerRef, @@ -3792,6 +3865,111 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { }) } + fn lookup_job_worker_creation( + &self, + worker_id: &str, + create_fingerprint: &str, + binding: &worker_runtime::catalog::BackendJobExecutionBinding, + ) -> Result, RuntimeRegistryError> { + let worker_ref = + self.worker_ref(worker_id) + .ok_or_else(|| RuntimeRegistryError::UnknownWorker { + worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id), + })?; + self.runtime + .lookup_job_worker_creation( + &self.workspace_id, + &worker_ref, + create_fingerprint, + binding, + ) + .map(|worker| worker.map(|worker| self.map_worker_detail(worker))) + .map_err(|error| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: self.runtime_id.clone(), + code: "embedded_job_creation_lookup_failed".into(), + message: error.to_string(), + }) + } + + fn backend_job_termination( + &self, + worker_id: &str, + ) -> Result, RuntimeRegistryError> + { + let worker_ref = + self.worker_ref(worker_id) + .ok_or_else(|| RuntimeRegistryError::UnknownWorker { + worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id), + })?; + self.runtime + .execution_termination(&worker_ref) + .map_err(|error| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: self.runtime_id.clone(), + code: "embedded_worker_termination_lookup_failed".into(), + message: error.to_string(), + }) + } + + fn reconcile_worker_input( + &self, + worker_id: &str, + request: WorkerInputRequest, + ) -> Result { + if !self.execution_enabled { + return Err(RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: self.runtime_id.clone(), + code: "embedded_worker_execution_unavailable".into(), + message: "Runtime has no execution backend for receipt lookup".into(), + }); + } + let worker_ref = + self.worker_ref(worker_id) + .ok_or_else(|| RuntimeRegistryError::UnknownWorker { + worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id), + })?; + let input = EmbeddedWorkerInput { + kind: match request.kind { + WorkerInputKind::User => EmbeddedWorkerInputKind::User, + WorkerInputKind::UserIfIdle => EmbeddedWorkerInputKind::UserIfIdle, + WorkerInputKind::Notify => EmbeddedWorkerInputKind::Notify, + WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact, + WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer, + }, + content: request.content, + source: request.source, + submission_request_id: request.submission_request_id, + segments: request.segments, + }; + let receipt = self + .runtime + .reconcile_input(&worker_ref, input) + .map_err(|error| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: self.runtime_id.clone(), + code: "embedded_worker_receipt_lookup_failed".into(), + message: error.to_string(), + })?; + if receipt.is_accepted() { + Ok(WorkerInputResult { + state: InternalWorkerOperationState::Accepted, + disposition: WorkerInputDisposition::Accepted, + worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id), + runtime_run_id: receipt.submission.map(|ack| ack.submission_request_id), + diagnostics: Vec::new(), + }) + } else { + Ok(embedded_input_rejected( + &self.runtime_id, + worker_id, + WorkerInputDisposition::Unknown, + diagnostic( + "embedded_worker_receipt_not_confirmed", + HostDiagnosticSeverity::Warning, + receipt.message_or_default(), + ), + )) + } + } + fn send_input(&self, worker_id: &str, request: WorkerInputRequest) -> WorkerInputResult { if !self.execution_enabled { return embedded_input_rejected( @@ -8587,6 +8765,122 @@ mod tests { assert!(initial_worker_input(&[]).is_none()); } + #[test] + fn registry_receipt_lookup_uses_embedded_read_boundary_without_input_dispatch() { + use worker_runtime::execution::{ + WorkerExecutionBackend, WorkerExecutionOperation, WorkerExecutionResult, + }; + #[derive(Default)] + struct ReceiptBackend { + creation: AcceptingExecutionBackend, + receipt: Mutex>, + queries: Mutex>, + } + impl WorkerExecutionBackend for ReceiptBackend { + fn backend_id(&self) -> &str { + "receipt-only-test-backend" + } + fn spawn_worker( + &self, + request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.creation.spawn_worker(request) + } + fn dispatch_input( + &self, + _: &EmbeddedWorkerRef, + _: EmbeddedWorkerInput, + ) -> WorkerExecutionResult { + panic!("a receipt query must not dispatch input") + } + fn reconcile_submission( + &self, + _: &EmbeddedWorkerRef, + input: EmbeddedWorkerInput, + ) -> WorkerExecutionResult { + self.queries.lock().unwrap().push(input); + self.receipt.lock().unwrap().clone().unwrap_or_else(|| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::Input, + "no stored receipt", + ) + }) + } + } + let backend = Arc::new(ReceiptBackend::default()); + let runtime = EmbeddedWorkerRuntime::new_memory_with_execution_backend( + "workspace-test", + backend.clone(), + ) + .unwrap(); + let mut spawn = embedded_spawn_request(); + spawn.intent = WorkerSpawnIntent::BackendJob { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + purpose: "receipt-only test".into(), + input_digest: format!("sha256:{}", "a".repeat(64)), + subjektiv_consolidation: false, + subjektiv_journal: false, + }; + let binding = worker_runtime::catalog::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_digest: Some(format!("sha256:{}", "a".repeat(64))), + subjektiv_consolidation: false, + subjektiv_journal: false, + }; + let created = runtime.spawn_worker(test_create_binding(), spawn); + let worker = created.worker.unwrap().worker; + let registry = RuntimeRegistry::for_workspace(runtime); + let saved = registry + .lookup_job_worker_creation(&worker, "sha256:test-create", &binding) + .unwrap() + .unwrap(); + assert_eq!(saved.worker, worker); + let mut foreign = binding.clone(); + foreign.attempt_id = "attempt-foreign".into(); + assert!( + registry + .lookup_job_worker_creation(&worker, "sha256:test-create", &foreign) + .is_err() + ); + assert!( + registry + .lookup_job_worker_creation(&worker, "foreign-create", &binding) + .is_err() + ); + let request = || WorkerInputRequest { + source: None, + kind: WorkerInputKind::User, + content: "frozen input".into(), + submission_request_id: Some("stored-request".into()), + segments: None, + }; + let absent = registry.reconcile_worker_input(&worker, request()).unwrap(); + assert_eq!(absent.disposition, WorkerInputDisposition::Unknown); + *backend.receipt.lock().unwrap() = Some(WorkerExecutionResult::accepted_submission( + WorkerExecutionOperation::Input, + "stored-request", + "stored-submission", + protocol::SubmissionDisposition::Started, + )); + let accepted = registry.reconcile_worker_input(&worker, request()).unwrap(); + assert_eq!(accepted.state, InternalWorkerOperationState::Accepted); + assert_eq!(accepted.runtime_run_id.as_deref(), Some("stored-request")); + *backend.receipt.lock().unwrap() = Some(WorkerExecutionResult::accepted_submission( + WorkerExecutionOperation::Input, + "foreign-request", + "stored-submission", + protocol::SubmissionDisposition::Started, + )); + let foreign = registry.reconcile_worker_input(&worker, request()).unwrap(); + assert_eq!(foreign.disposition, WorkerInputDisposition::Unknown); + let queries = backend.queries.lock().unwrap(); + assert_eq!(queries.len(), 3); + assert!(queries.iter().all(|input| input.content == "frozen input" + && input.submission_request_id.as_deref() == Some("stored-request"))); + } + #[test] fn embedded_runtime_with_execution_backend_routes_input_and_updates_status() { let runtime = EmbeddedWorkerRuntime::new_memory_with_execution_backend( diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 1f9f2fbc7..5c5b35a96 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -799,7 +799,7 @@ CREATE TABLE backend_jobs ( resource_key TEXT, request_json TEXT NOT NULL, intent_fingerprint TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown')), + state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed')), current_attempt INTEGER NOT NULL CHECK (current_attempt > 0 AND current_attempt <= 3), result_json TEXT, result_digest TEXT, @@ -823,7 +823,7 @@ CREATE TABLE backend_job_attempts ( attempt_id TEXT NOT NULL, attempt INTEGER NOT NULL CHECK (attempt > 0 AND attempt <= 3), input_digest TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatching', 'dispatched', 'completed', 'failed', 'unknown')), + state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatching', 'dispatched', 'completed', 'failed')), runtime_id TEXT, worker_id TEXT, runtime_run_id TEXT, @@ -843,6 +843,7 @@ CREATE TABLE backend_job_attempts ( created_at TEXT NOT NULL, updated_at TEXT NOT NULL, completed_at TEXT, + allocation_precluded_at TEXT, PRIMARY KEY (workspace_id, job_id, attempt_id), UNIQUE (workspace_id, job_id, attempt), UNIQUE (workspace_id, runtime_id, worker_id), diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index e013be2f0..c7a028ec6 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -3,6 +3,7 @@ pub mod auth; pub mod authority; pub mod backend_job; +mod backend_job_host_effects; pub mod companion; pub mod config; pub mod config_adoption; diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 991472c36..fa3aee993 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -26,6 +26,7 @@ use subjektiv::api::*; mod backend_job_profile; +mod backend_job_recovery; mod chat; mod external_operation_audit; mod session_registration; @@ -2130,6 +2131,7 @@ pub struct WorkspaceApi { Arc>>>>, notification_delivery_lifecycle: Arc, backend_job_drain_requests: Arc, + backend_job_recovery: Arc, journal_lifecycle: Arc>>>>, journal_dispatch_owner: Option>, plugin_runtime_owner: Option>, @@ -2642,6 +2644,27 @@ impl WorkerRemovalService { target: &RuntimeWorkerRef, reason: &str, parent_workspace_deletion_operation_id: Option<&str>, + ) -> std::result::Result { + self.execute_target_removal_on_host_checked( + runtime, + target, + reason, + parent_workspace_deletion_operation_id, + None, + ) + .await + } + + /// Restarted Job cleanup may resume only its already-observed canonical + /// operation. Check under the same removal lock; never fall through to planning + /// a new removal when the retained operation disappeared or was superseded. + async fn execute_target_removal_on_host_checked( + &self, + runtime: &RuntimeRegistry, + target: &RuntimeWorkerRef, + reason: &str, + parent_workspace_deletion_operation_id: Option<&str>, + expected_removal: Option<(&str, &str)>, ) -> std::result::Result { let remove_lock = { let mut locks = self @@ -2670,6 +2693,20 @@ impl WorkerRemovalService { .store .recover_worker_removal_execution(&self.workspace_id, &target) .map_err(|_| "Worker removal recovery authority is unavailable".to_string())?; + if let Some((operation_id, fingerprint)) = expected_removal { + if !prepared.as_ref().is_some_and(|saved| { + saved.plan.workspace_id == self.workspace_id + && saved.plan.worker == *target + && saved.plan.operation_id == operation_id + && saved.plan.input_fingerprint == fingerprint + && saved.runtime_request.operation_id == operation_id + && saved.runtime_request.input_fingerprint == fingerprint + }) { + return Err( + "Retained Job cleanup removal operation changed or is unavailable".into(), + ); + } + } if let Some(prepared) = prepared { if prepared.plan.state == crate::retention::WorkerRemovalPlanState::Succeeded { self.worker_projection @@ -4730,6 +4767,7 @@ impl WorkspaceApi { notification_delivery_lifecycle: Arc::new(Default::default()), orchestrator_attention: Arc::new(Mutex::new(HashMap::new())), backend_job_drain_requests: Arc::new(AtomicU64::new(0)), + backend_job_recovery: Arc::default(), journal_lifecycle: Arc::new(Mutex::new(HashMap::new())), journal_dispatch_owner: None, plugin_runtime_owner: None, @@ -4845,7 +4883,16 @@ impl WorkspaceApi { &self, request: &BackendJobRequest, ) -> Result { - self.resolve_backend_job_profile(request)?; + // Existing intent replays may only observe the original admission and + // must not depend on today's Profile/default. A new dispatch still + // resolves current configuration before creating execution authority. + if self + .store + .get_backend_job(self.workspace_id(), &request.job_id)? + .is_none() + { + self.resolve_backend_job_profile(request)?; + } let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); let reservation = { let gate = subjektiv_deletion::job_subject(request) @@ -4864,7 +4911,9 @@ impl WorkspaceApi { self.store .reserve_backend_job(&self.config.workspace_id, request, &now)? }; + let owned_attempt = reservation.attempt.clone(); let result = self.dispatch_backend_job_reservation(reservation); + self.own_backend_job_recovery(&owned_attempt); if result.is_err() { self.drain_reserved_backend_jobs(); } @@ -4872,8 +4921,7 @@ impl WorkspaceApi { } /// Explicit re-evaluation creates a new bounded attempt only after a - /// definitive failure. Unknown execution outcomes require manual attention - /// and cannot be replayed because the original tracked run may still execute. + /// definitive failure and independently confirmed execution/writer cleanup. /// Transport replay must call `dispatch_backend_job` with the original /// immutable request. pub fn retry_backend_job(&self, job_id: &str) -> Result { @@ -4899,7 +4947,9 @@ impl WorkspaceApi { self.store .reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)? }; + let owned_attempt = reservation.attempt.clone(); let result = self.dispatch_backend_job_reservation(reservation); + self.own_backend_job_recovery(&owned_attempt); if result.is_err() { self.drain_reserved_backend_jobs(); } @@ -4910,6 +4960,12 @@ impl WorkspaceApi { &self, reservation: BackendJobReservation, ) -> Result { + let dispatch_gate = self + .backend_job_recovery + .dispatch_gate(&reservation.attempt.attempt_id); + let _dispatch_guard = dispatch_gate + .lock() + .map_err(|_| Error::Store("Backend Job dispatch owner poisoned".into()))?; // Reread even a terminal/stale reservation before deciding whether it can // dispatch or replay: an explicit erasure may have retired its saved input. let journal_gate = subjektiv_deletion::job_subject(&reservation.job.request) @@ -4995,67 +5051,45 @@ impl WorkspaceApi { return Ok(reservation); } let request = &reservation.job.request; - let (profile, config_bundle) = match self.resolve_backend_job_profile(request) { - Ok(resolved) => resolved, - Err(error) => { - self.finish_backend_job_attempt( - &request.job_id, - &reservation.attempt.attempt_id, - BackendJobAttemptState::Failed, - "profile_requirements", - &error.to_string(), - )?; - return Err(error); - } - }; + let spawn_request = + match self.backend_job_spawn_request(&reservation.job, &reservation.attempt) { + Ok(request) => request, + Err(error) => { + self.finish_backend_job_attempt( + &request.job_id, + &reservation.attempt.attempt_id, + BackendJobAttemptState::Failed, + "profile_requirements", + &error.to_string(), + )?; + return Err(error); + } + }; let worker_input = request.worker_input(&reservation.attempt.attempt_id)?; - let control_operation = WorkerControlOperation { - operation_id: allocation_key(&reservation.attempt.attempt_id), - input_fingerprint: reservation.job.intent_fingerprint.clone(), - }; - let spawned = match self.spawn_workspace_worker( - EMBEDDED_WORKER_RUNTIME_ID, - WorkerSpawnRequest { - requested_worker_name: Some(format!("job:{}", request.purpose)), - singleton_key: None, - intent: WorkerSpawnIntent::BackendJob { - job_id: request.job_id.clone(), - attempt_id: reservation.attempt.attempt_id.clone(), - purpose: request.purpose.clone(), - input_digest: request.input_digest()?, - subjektiv_consolidation: request.grants.subjektiv_consolidation.is_some(), - subjektiv_journal: request.grants.subjektiv_journal.is_some(), - }, - acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { - expected_segments: 0, - }, - profile, - ticket_assignment: None, - initial_submit: Vec::new(), - workdir_attachment_requests: Vec::new(), - resolved_workdir_attachment_requests: Vec::new(), - resolved_workdir_attachments: Vec::new(), - resolved_config_bundle: Some(config_bundle), - resolved_worker_observation_enabled: false, - resolved_worker_observation_grants: Vec::new(), - resolved_workspace_api: None, - resolved_memory_settings: None, - // Journal Jobs use their independent frozen Job Host grant, not the - // main-Subject/Memory attachment or its resident/extraction policy. - resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some(), - resolved_control_operation: Some(control_operation), - resolved_execution_grant: None, - }, - ) { + let spawned = match self.spawn_workspace_worker(EMBEDDED_WORKER_RUNTIME_ID, spawn_request) { Ok(spawned) => spawned, Err(error) => { let detail = error.error.to_string(); - self.finish_backend_job_attempt( - &request.job_id, - &reservation.attempt.attempt_id, - BackendJobAttemptState::Failed, - "dispatch_error", + if matches!( + &error.error, + Error::UnknownRuntime(_) + | Error::InvalidInput(_) + | Error::WorkspacePermissionDenied(_) + ) { + self.finish_backend_job_attempt( + &request.job_id, + &reservation.attempt.attempt_id, + BackendJobAttemptState::Failed, + "dispatch_rejected", + &detail, + )?; + return Err(error.error); + } + self.retain_backend_job_recovery( + &reservation.attempt, + "allocation_response_lost", &detail, + "Read the same allocation receipt; never allocate a replacement Worker", )?; return Err(error.error); } @@ -5084,6 +5118,18 @@ impl WorkspaceApi { }; let bound_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); let runtime_run_id = crate::backend_job::runtime_run_id(&reservation.attempt.attempt_id); + // Only this initial allocation path can open Host admission. Recovery must + // never reconstruct an empty owner and use its drain as historical proof. + self.backend_job_recovery.initially_admit( + crate::backend_job_host_effects::JobHostOwner { + workspace_id: self.workspace_id().to_owned(), + job_id: request.job_id.clone(), + attempt_id: reservation.attempt.attempt_id.clone(), + input_digest: reservation.attempt.input_digest.clone(), + worker: worker.worker.clone(), + runtime_run_id: runtime_run_id.clone(), + }, + )?; let newly_bound = self.worker_projection.publish_ordered(|| { self.store .bind_backend_job_attempt_worker( @@ -5123,71 +5169,15 @@ impl WorkspaceApi { &reservation.attempt.deadline_at, ); - let submitted = match self.runtime.send_input( - &worker.worker, - WorkerInputRequest { - source: None, - kind: WorkerInputKind::User, - content: worker_input, - submission_request_id: Some(runtime_run_id.clone()), - segments: None, - }, - ) { - Ok(submitted) => submitted, - Err(error) => { - let detail = error.message(); - self.finish_backend_job_attempt( - &request.job_id, - &reservation.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "input_outcome_unknown", - &detail, - )?; - return Err(error.into_error()); - } - }; - if submitted.state != InternalWorkerOperationState::Accepted { - let detail = submitted - .diagnostics - .first() - .map(|diagnostic| diagnostic.message.as_str()) - .unwrap_or("embedded Runtime rejected Backend Job input"); - let (state, category) = match submitted.disposition { - WorkerInputDisposition::Unknown => { - (BackendJobAttemptState::Unknown, "input_outcome_unknown") - } - WorkerInputDisposition::Accepted | WorkerInputDisposition::Rejected => { - (BackendJobAttemptState::Failed, "input_rejected") - } - }; - self.finish_backend_job_attempt( + let bound = self + .store + .get_backend_job_attempt( + self.workspace_id(), &request.job_id, &reservation.attempt.attempt_id, - state, - category, - detail, - )?; - return Err(Error::RuntimeOperationFailed { - runtime_id: EMBEDDED_WORKER_RUNTIME_ID.to_string(), - code: format!("backend_job_{category}"), - message: detail.to_string(), - }); - } - if submitted.runtime_run_id.as_deref() != Some(runtime_run_id.as_str()) { - self.finish_backend_job_attempt( - &request.job_id, - &reservation.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "runtime_run_mismatch", - "embedded Runtime accepted Backend Job input without matching tracked run evidence", - )?; - return Err(Error::RuntimeOperationFailed { - runtime_id: EMBEDDED_WORKER_RUNTIME_ID.to_string(), - code: "backend_job_runtime_run_mismatch".to_string(), - message: "embedded Runtime accepted Backend Job input without matching tracked run evidence" - .to_string(), - }); - } + )? + .ok_or_else(|| Error::Store("bound Backend Job attempt missing".into()))?; + self.submit_initial_backend_job_input(&reservation.job, &bound, worker_input)?; subjektiv_journal::attest_delivery(self, &reservation)?; let job = self .store @@ -5293,6 +5283,7 @@ impl WorkspaceApi { }); } } + self.own_backend_job_recovery(attempt); } if attempts.len() < page_limit || !made_progress { return; @@ -5308,18 +5299,7 @@ impl WorkspaceApi { failure_category: &str, failure_detail: &str, ) -> Result { - if let Some(job) = self - .store - .get_backend_job(&self.config.workspace_id, job_id)? - && job.request.grants.subjektiv_journal.is_some() - && state != BackendJobAttemptState::Unknown - && let Some(attempt) = - self.store - .get_backend_job_attempt(&self.config.workspace_id, job_id, attempt_id)? - { - subjektiv_journal::close(self, &job.request, &attempt)?; - } - let attempt = self.store.finish_backend_job_attempt( + self.store.finish_backend_job_attempt( &self.config.workspace_id, job_id, attempt_id, @@ -5328,15 +5308,6 @@ impl WorkspaceApi { failure_detail, &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), )?; - // A retained checkpoint can prove the Journal result even if teardown is - // blocked. Do not make outcome resolution depend on successful Session cleanup. - if attempt.state == BackendJobAttemptState::Unknown - && let Some(job) = self.store.get_backend_job(self.workspace_id(), job_id)? - && job.request.grants.subjektiv_journal.is_some() - && let Err(error) = subjektiv_journal::reconcile_unknown(self, &job, &attempt) - { - tracing::warn!(%error,%job_id,%attempt_id,"Journal Unknown outcome remains recoverable"); - } let current = self .store .get_backend_job_attempt(self.workspace_id(), job_id, attempt_id)? @@ -5345,71 +5316,33 @@ impl WorkspaceApi { Ok(current) } - /// Admit on the independent cleanup lane before claiming durable work. Rejection leaves - /// Pending/Failed recoverable, not an orphaned Executing claim. Runtime teardown is a - /// lifecycle fence, not model-processing evidence; unknown DB outcomes are not retried. + /// Cleanup has its own admitted lifetime, separate from the result callback lease. + /// Claim occurs only after lane admission; retained exact owners recover storage faults. fn schedule_backend_job_worker_cleanup(&self, attempt: &BackendJobAttemptRecord) -> bool { if !attempt.state.terminal() || attempt.worker.is_none() || !matches!( attempt.worker_cleanup_state, - Some(BackendJobWorkerCleanupState::Pending | BackendJobWorkerCleanupState::Failed) + Some( + BackendJobWorkerCleanupState::Pending + | BackendJobWorkerCleanupState::Failed + | BackendJobWorkerCleanupState::Executing + | BackendJobWorkerCleanupState::Completed + ) ) { return false; } - let Ok(runtime) = tokio::runtime::Handle::try_current() else { - tracing::warn!(job_id = %attempt.job_id, "Backend Job cleanup remains durable without a runtime"); - return false; - }; - let api = self.clone(); - let job_id = attempt.job_id.clone(); - let attempt_id = attempt.attempt_id.clone(); - // Admit before claiming: overload must leave durable Pending/Failed - // ownership recoverable, never strand a Running claim without a job. - let admission = self.host_executors.cleanup().and_then(|executor| { - executor.dispatch(move || { - let cleanup = || -> Result<()> { - let (attempt, claimed) = api.store.claim_backend_job_worker_cleanup( - &api.config.workspace_id, &job_id, &attempt_id, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; - if !claimed { return Ok(()); } - let waits_for_run_teardown = attempt.state == BackendJobAttemptState::Completed - || attempt.failure_category.as_deref() == Some("invalid_result"); - runtime.block_on(async { - if waits_for_run_teardown { - api.wait_for_backend_job_worker_run_teardown(&attempt).await?; - } - // Already on the Host owner; removal must not submit to and - // wait on its own lane. All transaction helpers return before I/O. - api.cleanup_backend_job_attempt_worker(attempt).await - }) - }; - if let Err(error) = cleanup() { - tracing::warn!(%error, %job_id, %attempt_id, "Backend Job Worker cleanup failed"); - } - }) - }); - if let Err(error) = admission { - tracing::warn!(%error, "Backend Job cleanup was not admitted; its durable ownership is unchanged"); - return false; - } - true + self.own_backend_job_cleanup(attempt) } async fn wait_for_backend_job_worker_run_teardown( &self, attempt: &BackendJobAttemptRecord, ) -> Result<()> { - let worker = attempt.worker.as_ref().ok_or_else(|| { - Error::Store("claimed Backend Job Worker cleanup lost its binding".to_string()) - })?; - let Some(runtime_run_id) = attempt.runtime_run_id.as_deref() else { - // Attempts created before tracked Runtime submissions can only be - // recovered after a Backend restart, which has already ended their - // in-process result response and Run. Canonical stop/remove remains - // the safety boundary for those legacy leftovers. + // This wait only preserves response ordering. It is NEVER termination proof; + // canonical stop plus exact Controller/Host fences are required below. + let Some(worker) = attempt.worker.as_ref() else { return Ok(()); }; let journal_request = self @@ -5418,56 +5351,40 @@ impl WorkspaceApi { .filter(|job| job.request.grants.subjektiv_journal.is_some()) .map(|job| job.request); let deadline = chrono::DateTime::parse_from_rfc3339(&attempt.deadline_at) - .map_err(|e| Error::Store(e.to_string()))?; - loop { - if let Some(request) = &journal_request { - // Preserve normal result-response teardown ordering, but an explicit - // erasure or the immutable Job deadline must not wait on a stuck Run. - // Actual stop and canonical Purge still happen below this wait. - if Utc::now() >= deadline || subjektiv_journal::session_erased(self, request)? { - return Ok(()); - } + .map_err(|error| Error::Store(error.to_string()))?; + while Utc::now() < deadline { + // Explicit erasure interrupts response ordering, not termination proof. + // The caller must still stop, join and fence this exact execution. + if let Some(request) = &journal_request + && subjektiv_journal::session_erased(self, request)? + { + break; } match self.runtime.worker(worker) { Ok(detail) - if detail.state.eq_ignore_ascii_case("stopped") - || detail.execution_reconstructed - || detail.worker_state.as_ref().is_some_and(|snapshot| { - snapshot.state == protocol::WorkerState::Idle - && snapshot.last_finished_submission_request_id.as_deref() - == Some(runtime_run_id) - }) => + if detail.worker_state.as_ref().is_some_and(|state| { + state.last_finished_submission_request_id == attempt.runtime_run_id + }) => { - return Ok(()); + break; } - Ok(_) => { - // In this process incarnation the result capability runs inside - // this tracked Submit. The controller publishes the exact request - // id only after the invocation returns. A Worker reconstructed - // from durable state is handled above instead: opening the - // Worker store exclusively proves its prior process and any - // pre-marker Run are gone, even if execution restoration has - // not yet produced a fresh live snapshot. - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - Err(RuntimeRegistryError::UnknownWorker { .. }) => return Ok(()), - Err(error) => { - self.store.finish_backend_job_worker_cleanup( - &self.config.workspace_id, - &attempt.job_id, - &attempt.attempt_id, - BackendJobWorkerCleanupState::Failed, - Some("backend_job_cleanup_run_state_unavailable"), - Some(&error.message()), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - )?; - return Err(error.into_error()); + Ok(detail) + if detail.state.eq_ignore_ascii_case("stopped") + || detail.execution_reconstructed => + { + break; } + Err(_) => break, + _ => tokio::time::sleep(std::time::Duration::from_millis(10)).await, } } + Ok(()) } fn recover_backend_job_worker_cleanups(&self) -> Result<()> { + // The same Store list also returns Completed Journal cleanup with + // Available input. Its owner takes the retirement-only step before claim; + // neither a live Worker nor a new removal operation is required. let page_limit = usize::from(ABSOLUTE_MAX_CONCURRENT_JOBS); let mut attempts = self.store.recover_backend_job_worker_cleanups( &self.config.workspace_id, @@ -5494,7 +5411,11 @@ impl WorkspaceApi { async fn cleanup_backend_job_attempt_worker( &self, attempt: BackendJobAttemptRecord, + retained_removal: Option<(String, String)>, ) -> Result<()> { + let history = self + .store + .list_backend_job_attempts(self.workspace_id(), &attempt.job_id)?; let job_id = attempt.job_id.as_str(); let attempt_id = attempt.attempt_id.as_str(); let worker = attempt.worker.clone().ok_or_else(|| { @@ -5502,10 +5423,10 @@ impl WorkspaceApi { })?; let fail = |category: &str, detail: String| -> Result<()> { - self.store.finish_backend_job_worker_cleanup( + self.store.finish_backend_job_worker_cleanup_owned( &self.config.workspace_id, - job_id, - attempt_id, + &history, + &attempt, BackendJobWorkerCleanupState::Failed, Some(category), Some(&detail), @@ -5518,11 +5439,12 @@ impl WorkspaceApi { }) }; let complete = || -> Result<()> { - let journal = self + self.require_backend_job_attempt_effects_ended(&attempt)?; + let journal_job = self .store .get_backend_job(self.workspace_id(), job_id)? - .is_some_and(|job| job.request.grants.subjektiv_journal.is_some()); - if journal { + .filter(|job| job.request.grants.subjektiv_journal.is_some()); + if journal_job.is_some() { let removal = self .config_store .recover_worker_removal_execution(self.workspace_id(), &worker) @@ -5539,95 +5461,37 @@ impl WorkspaceApi { ); } } - self.store.finish_backend_job_worker_cleanup( + if let Some(job) = &journal_job { + if !subjektiv_journal::session_erased(self, &job.request)? { + // Close the old generation before publishing cleanup completion. + // A failed close remains owned and recoverable, never hidden by + // an already-Completed cleanup marker. + subjektiv_journal::close(self, &job.request, &attempt)?; + } + } + self.store.finish_backend_job_worker_cleanup_owned( &self.config.workspace_id, - job_id, - attempt_id, + &history, + &attempt, BackendJobWorkerCleanupState::Completed, None, None, &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), )?; - if let Some(job) = self - .store - .get_backend_job(&self.config.workspace_id, job_id)? - && job.request.grants.subjektiv_journal.is_some() - { - // Cleanup completion is a durable stop/removal fence. Unknown is - // reconciled with retained domain receipts before closing its generation. - let current = self - .store - .get_backend_job_attempt(self.workspace_id(), job_id, attempt_id)? - .ok_or_else(|| Error::Store("cleaned Journal attempt missing".into()))?; - if current.state == BackendJobAttemptState::Unknown { - subjektiv_journal::reconcile_unknown(self, &job, ¤t)?; - } else if !subjektiv_journal::session_erased(self, &job.request)? { - subjektiv_journal::close(self, &job.request, ¤t)?; - if current.state == BackendJobAttemptState::Completed { - subjektiv_journal::drain_after_completion(self, &job.request); - if job.input_disposition - == crate::backend_job::JobInputDisposition::Available - { - self.store.reclaim_backend_job_input( - self.workspace_id(), - job_id, - &job.input_digest, - crate::backend_job::JobInputRetirementReason::Completed, - )?; - } - } - } - } - Ok(()) + self.finish_backend_job_cleanup_retirement(job_id, attempt_id) }; - let detail = match self.runtime.worker(&worker) { - Ok(detail) => detail, - Err(RuntimeRegistryError::UnknownWorker { .. }) => { - if self - .store - .get_worker_registry(&self.config.workspace_id, &worker)? - .is_none() - { - return complete(); - } - // A canonical removal may have deleted the Runtime object before - // its catalog commit. Let the existing durable removal operation - // recover that midpoint; without such an operation this remains - // a diagnosable retryable failure rather than a direct DB delete. - let response = WorkerRemovalService::new(self) - .execute_target_removal_on_host( - self.runtime.as_ref(), - &worker, - &format!("Terminal Backend Job attempt {attempt_id}"), - None, - ) - .await - .map_err(|message| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "backend_job_cleanup_removal_failed".to_string(), - message, - }); - return match response { - Ok(response) if response.status == StatusCode::OK.as_u16() => complete(), - Ok(response) => { - fail("backend_job_cleanup_runtime_worker_missing", response.body) - } - Err(error) => fail("backend_job_cleanup_removal_failed", error.to_string()), - }; - } - Err(error) => { - return fail("backend_job_cleanup_runtime_unavailable", error.message()); - } - }; - if !detail.state.eq_ignore_ascii_case("stopped") { + // Stop acceptance and the status projection are not scope termination receipts. + // Even a currently Stopped Worker needs retained exact execution-owner proof. + if self + .require_backend_job_attempt_effects_ended(&attempt) + .is_err() + { match self.runtime.stop_worker( &worker, WorkerLifecycleRequest { stop_intent: None, - reason: Some(format!( - "Backend Job attempt {attempt_id} reached terminal state" - )), + reason: Some(format!("Backend Job attempt {attempt_id} cleanup")), ticket_assignment: None, }, ) { @@ -5638,36 +5502,22 @@ impl WorkspaceApi { runtime_diagnostics_message(&result.diagnostics), ); } - Err(error) => { - return fail("backend_job_cleanup_stop_failed", error.message()); - } + Err(error) => return fail("backend_job_cleanup_stop_unconfirmed", error.message()), } - } - match self.runtime.worker(&worker) { - Ok(detail) if detail.state.eq_ignore_ascii_case("stopped") => {} - Ok(detail) => { - return fail( - "backend_job_cleanup_stop_unconfirmed", - format!( - "Runtime reported Worker state `{}` after accepting stop", - detail.state - ), - ); - } - Err(error) => { - return fail( - "backend_job_cleanup_stop_confirmation_failed", - error.message(), - ); + if let Err(error) = self.fence_backend_job_effects(&attempt).await { + return fail("backend_job_cleanup_effects_unconfirmed", error.to_string()); } } let response = WorkerRemovalService::new(self) - .execute_target_removal_on_host( + .execute_target_removal_on_host_checked( self.runtime.as_ref(), &worker, &format!("Terminal Backend Job attempt {attempt_id}"), None, + retained_removal + .as_ref() + .map(|(id, fingerprint)| (id.as_str(), fingerprint.as_str())), ) .await .map_err(|message| Error::RuntimeOperationFailed { @@ -5682,168 +5532,36 @@ impl WorkspaceApi { if response.status == StatusCode::OK.as_u16() { return complete(); } - let runtime_absent = matches!( - self.runtime.worker(&worker), - Err(RuntimeRegistryError::UnknownWorker { .. }) - ); - let catalog_absent = self - .store - .get_worker_registry(&self.config.workspace_id, &worker)? - .is_none(); - if runtime_absent && catalog_absent { - return complete(); - } fail("backend_job_cleanup_removal_rejected", response.body) } - /// Reconcile durable non-terminal attempts after Backend restart. Reserved - /// attempts are replay-dispatched through their deterministic allocation; - /// dispatched attempts keep running only while their bound Worker remains - /// known, and always receive a fresh deadline task. + /// Restore owners for current-contract attempts without replaying uncertain effects. pub fn recover_backend_jobs(&self) -> Result<()> { - let attempts = self.store.list_recoverable_backend_job_attempts( - &self.config.workspace_id, + for attempt in self.store.list_recoverable_backend_job_attempts( + self.workspace_id(), usize::from(ABSOLUTE_MAX_CONCURRENT_JOBS), - )?; - for attempt in attempts { - match attempt.state { - BackendJobAttemptState::Reserved => { - let job = self - .store - .get_backend_job(&self.config.workspace_id, &attempt.job_id)? - .ok_or_else(|| { - Error::Store("recoverable Backend Job disappeared".to_string()) - })?; - if let Err(error) = - self.dispatch_backend_job_reservation(BackendJobReservation { - job, - attempt, - replayed: true, - resource_reused: false, - }) - { - tracing::warn!(%error, "Backend Job reserved-attempt recovery failed"); - } - } - BackendJobAttemptState::Dispatching => { - self.finish_backend_job_attempt( - &attempt.job_id, - &attempt.attempt_id, - BackendJobAttemptState::Unknown, - "recovery_dispatch_claim_unknown", - "a Backend Job dispatch claim could not be safely replayed after restart", - )?; - } - BackendJobAttemptState::Dispatched => { - if attempt.runtime_run_id.is_none() { - self.finish_backend_job_attempt( - &attempt.job_id, - &attempt.attempt_id, - BackendJobAttemptState::Unknown, - "recovery_runtime_run_unknown", - "the Runtime accepted Worker dispatch without durable run evidence", - )?; - continue; - } - let observed = attempt - .worker - .as_ref() - .map(|worker| self.runtime.worker(worker)); - if observed.as_ref().is_some_and(|result| { - result.as_ref().is_ok_and(|detail| { - detail.state.eq_ignore_ascii_case("stopped") - || detail.execution_reconstructed - }) - }) && self - .store - .get_backend_job(self.workspace_id(), &attempt.job_id)? - .is_some_and(|job| job.request.grants.subjektiv_journal.is_some()) - { - // This is not success evidence. Move to Unknown to fence tools; - // only domain completion or confirmed canonical cleanup resolves it. - self.finish_backend_job_attempt(&attempt.job_id, &attempt.attempt_id, - BackendJobAttemptState::Unknown, "recovery_journal_execution_ended", - "Runtime reports the prior Journal execution ended; reconciling saved domain receipts")?; - continue; - } - let worker_known = observed.as_ref().is_some_and(|result| result.is_ok()); - if !worker_known { - self.finish_backend_job_attempt( - &attempt.job_id, - &attempt.attempt_id, - BackendJobAttemptState::Unknown, - "recovery_worker_unknown", - "the bound Runtime Worker was not recoverable after Backend restart", - )?; - } else { - self.schedule_backend_job_timeout( - &attempt.job_id, - &attempt.attempt_id, - &attempt.deadline_at, - ); - } - } - BackendJobAttemptState::Completed - | BackendJobAttemptState::Failed - | BackendJobAttemptState::Unknown => {} - } + )? { + self.own_backend_job_recovery(&attempt); } - subjektiv_journal::recover_unknowns(self)?; + subjektiv_journal::recover_attempts(self)?; self.recover_backend_job_worker_cleanups()?; self.drain_reserved_backend_jobs(); Ok(()) } - fn schedule_backend_job_timeout(&self, job_id: &str, attempt_id: &str, deadline_at: &str) { - let Ok(deadline) = chrono::DateTime::parse_from_rfc3339(deadline_at) else { - return; - }; - let delay = (deadline.with_timezone(&Utc) - Utc::now()) - .to_std() - .unwrap_or_default(); - let Ok(handle) = tokio::runtime::Handle::try_current() else { - return; - }; - let api = self.clone(); - let job_id = job_id.to_string(); - let attempt_id = attempt_id.to_string(); - handle.spawn(async move { - tokio::time::sleep(delay).await; - let admission = api.host_executors.host().and_then(|executor| executor.dispatch(move || { - let Ok(Some(attempt)) = - api.store - .get_backend_job_attempt(&api.config.workspace_id, &job_id, &attempt_id) - else { - return; - }; - if attempt.state != BackendJobAttemptState::Dispatched { - return; - } - // Fence result acceptance before stopping the Worker so a late - // completion cannot turn a timed-out attempt into success. - if api - .finish_backend_job_attempt( - &job_id, - &attempt_id, - BackendJobAttemptState::Failed, - "timeout", - "Backend Job attempt exceeded its durable deadline", - ) - .is_ok() - { - api.drain_reserved_backend_jobs(); - } - })); - if let Err(error) = admission { - tracing::warn!(%error, "Backend Job timeout was not admitted; its deadline remains durable"); - } - }); + fn schedule_backend_job_timeout(&self, job_id: &str, attempt_id: &str, _deadline_at: &str) { + if let Ok(Some(attempt)) = + self.store + .get_backend_job_attempt(self.workspace_id(), job_id, attempt_id) + { + self.own_backend_job_recovery(&attempt); + } } /// Accept the only authoritative success signal for a Job attempt. The /// source identity is fenced against the durable Worker binding. Result /// persistence precedes best-effort notification delivery. - pub fn accept_backend_job_result( + fn accept_backend_job_result_inner( &self, source_worker: &RuntimeWorkerRef, submission: &BackendJobResultSubmission, @@ -5995,14 +5713,8 @@ impl WorkspaceApi { &accepted_at, )?; drop(subject_guard); - if let Some(job) = self - .store - .get_backend_job(&self.config.workspace_id, &submission.job_id)? - && job.request.grants.subjektiv_journal.is_some() - { - subjektiv_journal::close(self, &job.request, &acceptance.attempt)?; - subjektiv_journal::drain_after_completion(self, &job.request); - } + // Closing a generation/releasing pins is a writer operation, not a consequence + // of accepting a result. Canonical cleanup performs it after independent fences. notification_delivery::deliver_job_result(self, &acceptance); let _ = self.schedule_backend_job_worker_cleanup(&acceptance.attempt); self.drain_reserved_backend_jobs(); @@ -15079,9 +14791,13 @@ fn build_inner_router(api: WorkspaceApi) -> Router { .route("/api/w/{workspace_id}/subjektiv/journal/job", post(subjektiv_journal::job_route)) .route("/api/w/{workspace_id}/subjektiv/journal/job/finish", post(subjektiv_journal::finish_route)) .fallback(get(static_or_spa_fallback)) - .with_state(api) + .with_state(api.clone()) .merge(generated_workspace_contract_router(contract_service)); auth.merge(workspace) + .layer(middleware::from_fn_with_state( + api, + backend_job_recovery::admit_host_request, + )) .layer(middleware::from_fn(log_failed_api_response)) } @@ -15253,6 +14969,7 @@ pub async fn serve( // Feature shutdown fences new operations and waits for synchronous connection // holders. Accepted close retains ownership after its async waiter disappears. async fn shutdown_workspace_feature_storage(api: &WorkspaceApi) -> Result<()> { + api.stop_backend_job_recovery().await; // Aborting the async hook cannot cancel an already admitted Host closure. // Fence all delivery entry points and wait for their actual I/O owners first. api.notification_delivery_lifecycle.stop_and_drain().await; @@ -51717,6 +51434,8 @@ mod tests { // The receiver belongs to one mock execution, not to a Runtime WorkerRecord. // Bound protocol senders become terminal when Stop drops this execution. struct DeterministicWorkerExecution { + execution_id: uuid::Uuid, + background: Mutex>>, subject_release: std::sync::Mutex< Option<( runtime_api::SubjectReleaseRequest, @@ -51743,6 +51462,8 @@ mod tests { let (method_tx, method_rx) = tokio::sync::mpsc::channel(32); let (events, _) = tokio::sync::broadcast::channel(32); Self { + execution_id: uuid::Uuid::now_v7(), + background: Mutex::new(Vec::new()), subject_release: std::sync::Mutex::new(None), dispatch_hook: std::sync::Mutex::new(None), context, @@ -51852,6 +51573,23 @@ mod tests { } struct DeterministicExecutionBackend { + receipts: Mutex< + HashMap< + (worker_runtime::identity::WorkerRef, String), + ( + serde_json::Value, + worker_runtime::execution::WorkerExecutionResult, + ), + >, + >, + terminations: Mutex< + HashMap< + worker_runtime::identity::WorkerRef, + worker_runtime::execution::WorkerExecutionTermination, + >, + >, + lose_input_ack: std::sync::atomic::AtomicBool, + reject_input_definitively: std::sync::atomic::AtomicBool, assignment_release_enabled: std::sync::atomic::AtomicBool, contexts: std::sync::Mutex< std::collections::HashMap< @@ -51901,6 +51639,10 @@ mod tests { .as_nanos() ); Self { + receipts: Mutex::new(HashMap::new()), + terminations: Mutex::new(HashMap::new()), + lose_input_ack: std::sync::atomic::AtomicBool::new(false), + reject_input_definitively: std::sync::atomic::AtomicBool::new(false), assignment_release_enabled: std::sync::atomic::AtomicBool::new(false), contexts: std::sync::Mutex::new(std::collections::HashMap::new()), materializer: worker_runtime::working_directory::RuntimeGitMaterializer::new( @@ -52288,6 +52030,117 @@ mod tests { Ok(()) } + fn termination_receipt( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result< + Option, + String, + > { + let receipt = self.terminations.lock().unwrap().get(worker_ref).cloned(); + if let Some(current) = self.contexts.lock().unwrap().get(worker_ref) + && receipt + .as_ref() + .is_some_and(|receipt| receipt.execution_id != current.execution_id) + { + return Err( + "Fixture execution was replaced; old termination is not current evidence" + .into(), + ); + } + Ok(receipt) + } + + fn stop_worker_operation( + &self, + request: worker_runtime::execution::WorkerExecutionStopRequest, + ) -> worker_runtime::execution::WorkerExecutionResult { + use worker_runtime::execution::*; + if let Some(receipt) = self + .terminations + .lock() + .unwrap() + .get(&request.worker_ref) + .cloned() + && receipt.stop_operation_id == request.operation_id + { + let mut result = WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop); + result.execution_id = Some(receipt.execution_id); + result.termination = Some(receipt); + return result; + } + let execution = self + .contexts + .lock() + .unwrap() + .get(&request.worker_ref) + .cloned(); + let Some(execution) = execution else { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "No fixture execution or matching stop receipt", + ); + }; + let mut result = self.stop_worker(&request.worker_ref); + // This fixture has no real Controller/Features. Join its actual echo + // threads before attesting its explicitly substituted execution boundary. + for task in std::mem::take(&mut *execution.background.lock().unwrap()) { + if task.join().is_err() { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "Fixture background join failed", + ); + } + } + let submissions = self + .receipts + .lock() + .unwrap() + .iter() + .filter(|((worker, _), (_, result))| { + worker == &request.worker_ref + && result.execution_id == Some(execution.execution_id) + }) + .filter_map(|(_, (_, result))| result.submission.clone()) + .collect(); + let receipt = WorkerExecutionTermination { + scope: WorkerExecutionTerminationScope::Controller, + worker_ref: request.worker_ref.clone(), + stop_operation_id: request.operation_id, + execution_id: execution.execution_id, + shutdown_command_id: Some(1), + submissions, + feature_failures: Vec::new(), + }; + self.terminations + .lock() + .unwrap() + .insert(request.worker_ref, receipt.clone()); + result.execution_id = Some(receipt.execution_id); + result.termination = Some(receipt); + result + } + + fn reconcile_submission( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + input: worker_runtime::interaction::WorkerInput, + ) -> worker_runtime::execution::WorkerExecutionResult { + use worker_runtime::execution::*; + let key = ( + worker_ref.clone(), + input.submission_request_id.clone().unwrap_or_default(), + ); + let payload = serde_json::to_value(input).unwrap(); + match self.receipts.lock().unwrap().get(&key) { + Some((saved, receipt)) if saved == &payload => receipt.clone(), + _ => WorkerExecutionResult::errored( + WorkerExecutionOperation::Input, + "Fixture exact input receipt missing or mismatched", + ), + } + } + fn stop_worker( &self, worker_ref: &worker_runtime::identity::WorkerRef, @@ -52339,10 +52192,15 @@ mod tests { ); }; let bound_execution = Arc::downgrade(&execution); - drop(execution); if let Some(hook) = self.input_hook.lock().unwrap().take() { hook(worker_ref); } + if self.reject_input_definitively.load(Ordering::Acquire) { + return worker_runtime::execution::WorkerExecutionResult::rejected( + worker_runtime::execution::WorkerExecutionOperation::Input, + "Fixture admission rejected before acceptance", + ); + } if let Some(message) = self.input_failure.lock().unwrap().clone() { return worker_runtime::execution::WorkerExecutionResult::errored( worker_runtime::execution::WorkerExecutionOperation::Input, @@ -52352,7 +52210,7 @@ mod tests { let submission_request_id = input.submission_request_id.clone(); let input_kind = input.kind.clone(); let content = input.content.clone(); - std::thread::spawn(move || { + let background = std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(25)); if let Some(execution) = bound_execution.upgrade() { let _ = execution.publish_event(protocol::Event::TextDone { @@ -52360,7 +52218,9 @@ mod tests { }); } }); - if input_kind == worker_runtime::interaction::WorkerInputKind::Notify { + execution.background.lock().unwrap().push(background); + let mut receipt = if input_kind == worker_runtime::interaction::WorkerInputKind::Notify + { worker_runtime::execution::WorkerExecutionResult::accepted_notification( worker_runtime::execution::WorkerExecutionOperation::Input, submission_request_id.expect("Notify test input has a Runtime request id"), @@ -52383,7 +52243,23 @@ mod tests { worker_runtime::execution::WorkerExecutionOperation::Input, ) .with_worker_state(protocol::WorkerStateSnapshot::initial()) + }; + receipt.execution_id = Some(execution.execution_id); + if let Some(id) = &input.submission_request_id { + self.receipts.lock().unwrap().insert( + (worker_ref.clone(), id.clone()), + (serde_json::to_value(&input).unwrap(), receipt.clone()), + ); } + if self.lose_input_ack.swap(false, Ordering::AcqRel) { + let mut lost = worker_runtime::execution::WorkerExecutionResult::errored( + worker_runtime::execution::WorkerExecutionOperation::Input, + "Fixture response lost after durable input acceptance", + ); + lost.execution_id = Some(execution.execution_id); + return lost; + } + receipt } } @@ -53346,1127 +53222,7 @@ mod tests { assert!(execution.take_inputs().is_empty()); } - #[tokio::test] - async fn backend_job_runner_uses_registered_embedded_worker_and_structured_result_authority() { - let temp = tempfile::tempdir().unwrap(); - let (api, execution) = test_api_with_recording_backend(temp.path()).await; - let request = BackendJobRequest { - job_id: "runner-check-1".to_string(), - purpose: "runner_contract_check".to_string(), - input_ref: "test://runner/input/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits::default(), - }; - - let stale_reservation = api - .store - .reserve_backend_job( - TEST_WORKSPACE_ID, - &request, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - .unwrap(); - let dispatched = api - .dispatch_backend_job_reservation(stale_reservation.clone()) - .unwrap(); - let concurrent_replay = api - .dispatch_backend_job_reservation(stale_reservation) - .unwrap(); - assert!(concurrent_replay.replayed); - assert_eq!( - execution.take_inputs().len(), - 1, - "a stale concurrent dispatch replay must not submit the attempt twice" - ); - assert_eq!(dispatched.attempt.state, BackendJobAttemptState::Dispatched); - assert!( - dispatched.attempt.runtime_run_id.is_some(), - "accepted Job input must be fenced to durable Runtime run evidence" - ); - let worker = dispatched.attempt.worker.clone().expect("bound Job Worker"); - assert_eq!(worker.runtime_id, EMBEDDED_WORKER_RUNTIME_ID); - assert!( - api.store - .get_worker_registry(TEST_WORKSPACE_ID, &worker) - .unwrap() - .is_some(), - "Job dispatch must use the ordinary Workspace Worker registry" - ); - assert_eq!( - api.store - .worker_registry_projection(TEST_WORKSPACE_ID, &worker) - .unwrap() - .unwrap() - .job, - Some(crate::backend_job::BackendJobWorkerBinding { - job_id: request.job_id.clone(), - attempt_id: dispatched.attempt.attempt_id.clone(), - purpose: request.purpose.clone(), - }) - ); - let input_error = send_runtime_worker_input( - State(api.clone()), - AxumPath((worker.runtime_id.clone(), worker.worker_id.clone())), - Json(server_api::RuntimeWorkerInputRequest { - submission_request_id: None, - kind: Some("user".to_string()), - content: "attempt to rerun immutable Job input".to_string(), - segments: None, - }), - ) - .await - .unwrap_err(); - assert_eq!( - input_error.into_response().status(), - StatusCode::BAD_REQUEST - ); - assert!( - execution.take_inputs().is_empty(), - "Job-owned Workers must reject interactive reruns" - ); - let replay = api.dispatch_backend_job(&request).unwrap(); - assert!(replay.replayed); - assert_eq!(replay.attempt.worker, Some(worker.clone())); - - let accepted = api - .accept_backend_job_result( - &worker, - &BackendJobResultSubmission { - job_id: request.job_id.clone(), - attempt_id: dispatched.attempt.attempt_id, - input_digest: request.input_digest().unwrap(), - result: serde_json::json!({"valid": true}), - }, - ) - .unwrap(); - assert_eq!(accepted.job.state, BackendJobState::Completed); - } - - #[tokio::test] - async fn terminal_backend_job_cleanup_removes_only_bound_worker_and_retains_result_delivery() { - let temp = tempfile::tempdir().unwrap(); - let (api, execution) = test_api_with_recording_backend(temp.path()).await; - let notification_target = spawn_ticket_check_source(&api, "job-cleanup-target"); - let request = BackendJobRequest { - job_id: "runner-cleanup-success-1".to_string(), - purpose: "cleanup_contract_check".to_string(), - input_ref: "test://runner/cleanup/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: Some(notification_target.clone()), - notification_target: Some(notification_target.clone()), - limits: crate::backend_job::BackendJobLimits::default(), - }; - let dispatched = api.dispatch_backend_job(&request).unwrap(); - execution.take_inputs(); - execution.reject_inputs("notification unavailable"); - let worker = dispatched.attempt.worker.clone().unwrap(); - seed_worker_session_for_cleanup(temp.path(), &worker); - let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap(); - let _ = subscriber.take_snapshot(); - let submission = BackendJobResultSubmission { - job_id: request.job_id.clone(), - attempt_id: dispatched.attempt.attempt_id.clone(), - input_digest: request.input_digest().unwrap(), - result: serde_json::json!({"valid": true}), - }; - - let accepted = api.accept_backend_job_result(&worker, &submission).unwrap(); - assert_eq!(accepted.job.state, BackendJobState::Completed); - let cleaned = wait_for_backend_job_worker_cleanup( - &api, - &request.job_id, - &dispatched.attempt.attempt_id, - BackendJobWorkerCleanupState::Completed, - ) - .await; - assert_eq!(cleaned.result, Some(serde_json::json!({"valid": true}))); - assert!(matches!( - api.runtime.worker(&worker), - Err(RuntimeRegistryError::UnknownWorker { .. }) - )); - assert!( - !temp - .path() - .join(".test-embedded-runtime-store/workers") - .join(&worker.worker_id) - .exists(), - "Runtime Worker and Session storage must be removed" - ); - assert!( - api.store - .get_worker_registry(TEST_WORKSPACE_ID, &worker) - .unwrap() - .is_none(), - "canonical removal must delete the Backend catalog projection" - ); - tokio::time::timeout(std::time::Duration::from_secs(1), async { - loop { - let event = subscriber.recv().await.expect("projection event"); - if event.changes.iter().any(|change| { - matches!(change, crate::store::WorkerCatalogChange::Removed(removed) if removed == &worker) - }) { - return; - } - } - }) - .await - .expect("Backend Job cleanup must publish canonical Worker removal"); - assert!( - api.runtime.worker(¬ification_target).is_ok(), - "origin/notification Workers are not cleanup targets" - ); - assert!( - api.store - .get_worker_registry(TEST_WORKSPACE_ID, ¬ification_target) - .unwrap() - .is_some() - ); - let replay = api.accept_backend_job_result(&worker, &submission).unwrap(); - assert!( - replay.replayed, - "duplicate result remains idempotent after cleanup" - ); - let delivery_id = - crate::backend_job::delivery_id(&request.job_id, &dispatched.attempt.attempt_id); - let evidence = job_notification_evidence(&api, &delivery_id); - assert_eq!(evidence.0, "deliver"); - assert!(!evidence.2); - assert_eq!(evidence.3.as_deref(), Some("acceptance_unknown")); - assert!( - api.store - .job_notification_request(TEST_WORKSPACE_ID, &delivery_id) - .unwrap() - .is_some() - ); - } - - #[tokio::test] - async fn backend_job_cleanup_guard_failure_retries_old_attempt_without_touching_new_worker() { - let temp = tempfile::tempdir().unwrap(); - let (api, _execution) = test_api_with_recording_backend(temp.path()).await; - let request = BackendJobRequest { - job_id: "runner-cleanup-retry-1".to_string(), - purpose: "cleanup_retry_contract_check".to_string(), - input_ref: "test://runner/cleanup-retry/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits::default(), - }; - let first = api.dispatch_backend_job(&request).unwrap(); - let first_worker = first.attempt.worker.clone().unwrap(); - seed_worker_session_for_cleanup(temp.path(), &first_worker); - let _ = set_worker_retention( - api.clone(), - first_worker.runtime_id.clone(), - first_worker.worker_id.clone(), - true, - ) - .await - .unwrap(); - api.finish_backend_job_attempt( - &request.job_id, - &first.attempt.attempt_id, - BackendJobAttemptState::Failed, - "model_failure", - "model rejected the request", - ) - .unwrap(); - let blocked = wait_for_backend_job_worker_cleanup( - &api, - &request.job_id, - &first.attempt.attempt_id, - BackendJobWorkerCleanupState::Failed, - ) - .await; - assert_eq!(blocked.failure_category.as_deref(), Some("model_failure")); - assert_eq!( - blocked.worker_cleanup_failure_category.as_deref(), - Some("backend_job_cleanup_removal_rejected") - ); - - assert!( - api.retry_backend_job(&request.job_id).is_err(), - "retry must not overlap an incompletely cleaned attempt" - ); - let _ = set_worker_retention( - api.clone(), - first_worker.runtime_id.clone(), - first_worker.worker_id.clone(), - false, - ) - .await - .unwrap(); - api.recover_backend_jobs().unwrap(); - wait_for_backend_job_worker_cleanup( - &api, - &request.job_id, - &first.attempt.attempt_id, - BackendJobWorkerCleanupState::Completed, - ) - .await; - let second = api.retry_backend_job(&request.job_id).unwrap(); - let second_worker = second.attempt.worker.clone().unwrap(); - seed_worker_session_for_cleanup(temp.path(), &second_worker); - assert_ne!(first_worker, second_worker); - api.recover_backend_jobs().unwrap(); - assert!(api.runtime.worker(&second_worker).is_ok()); - - api.accept_backend_job_result( - &second_worker, - &BackendJobResultSubmission { - job_id: request.job_id.clone(), - attempt_id: second.attempt.attempt_id.clone(), - input_digest: request.input_digest().unwrap(), - result: serde_json::json!({"retried": true}), - }, - ) - .unwrap(); - wait_for_backend_job_worker_cleanup( - &api, - &request.job_id, - &second.attempt.attempt_id, - BackendJobWorkerCleanupState::Completed, - ) - .await; - assert_eq!( - api.store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &request.job_id, - &first.attempt.attempt_id, - ) - .unwrap() - .unwrap() - .failure_detail - .as_deref(), - Some("model rejected the request") - ); - } - - #[tokio::test] - async fn backend_job_cleanup_restart_before_run_finish_marker_converges() { - let temp = tempfile::tempdir().unwrap(); - let config = test_server_config(temp.path()); - let request = BackendJobRequest { - job_id: "runner-cleanup-pre-marker-restart-1".to_string(), - purpose: "cleanup_pre_marker_restart_contract_check".to_string(), - input_ref: "test://runner/cleanup-pre-marker-restart/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits::default(), - }; - let store = - Arc::new(SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap()); - seed_test_registered_workspace(store.as_ref(), &config) - .await - .unwrap(); - let execution = Arc::new(DeterministicExecutionBackend::default()); - let embedded = EmbeddedWorkerRuntime::new_fs_store_with_execution_backend( - TEST_WORKSPACE_ID, - config.embedded_runtime_store_root.clone(), - execution.clone(), - ) - .unwrap(); - let binding = test_create_binding(); - let runtime = embedded.subscription_runtime(); - let runtime_worker_ref = worker_runtime::identity::WorkerRef::new(binding.worker_id); - let detail = runtime - .create_worker_scoped( - &worker_runtime::RuntimeWorkspaceScope::new(TEST_WORKSPACE_ID, "embedded-backend"), - worker_runtime::catalog::CreateWorkerRequest { - plugin_execution: None, - worker_id: binding.worker_id, - create_fingerprint: binding.create_fingerprint, - profile: ProfileSelector::Builtin(request.profile.clone()), - display_name: Some("job:cleanup_pre_marker_restart_contract_check".to_string()), - config_bundle: None, - profile_source: worker_runtime::catalog::ProfileSourceArchiveSource::Embedded { - archive: crate::profile_settings::builtin_profile_source_archive( - &ProfileSelector::Builtin(request.profile.clone()), - ) - .unwrap(), - }, - initial_input: None, - workdir_attachment_requests: Vec::new(), - workdir_attachments: Vec::new(), - worker_observation_enabled: false, - worker_observation_grants: Vec::new(), - workspace_api: Some(test_worker_workspace_api(EMBEDDED_WORKER_RUNTIME_ID)), - memory_settings: Some(test_worker_memory_settings()), - subjektiv_attached: false, - backend_job: None, - }, - ) - .unwrap(); - let worker = - RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, detail.worker_id.to_string()); - runtime - .observe_worker_event( - &runtime_worker_ref, - protocol::Event::WorkerState { - snapshot: protocol::WorkerStateSnapshot { - last_command_id: 0, - last_finished_submission_request_id: None, - state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( - protocol::WorkerRunState::Running, - )), - }, - }, - ) - .unwrap(); - let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - let reservation = store - .reserve_backend_job(TEST_WORKSPACE_ID, &request, &now) - .unwrap(); - let (_, claimed) = store - .claim_backend_job_attempt_dispatch( - TEST_WORKSPACE_ID, - &request.job_id, - &reservation.attempt.attempt_id, - &now, - ) - .unwrap(); - assert!(claimed); - let runtime_run_id = crate::backend_job::runtime_run_id(&reservation.attempt.attempt_id); - store - .upsert_worker_registry(&WorkerRegistryRecord { - workspace_id: TEST_WORKSPACE_ID.to_string(), - worker: worker.clone(), - display_name: "job:cleanup_pre_marker_restart_contract_check".to_string(), - profile: Some(request.profile.clone()), - retention_state: "normal".to_string(), - transcript_ref: None, - session_ref: None, - summary_ref: None, - diagnostics_ref: None, - created_at: now.clone(), - updated_at: now.clone(), - }) - .unwrap(); - store - .bind_backend_job_attempt_worker( - TEST_WORKSPACE_ID, - &request.job_id, - &reservation.attempt.attempt_id, - &worker, - Some(&runtime_run_id), - &now, - ) - .unwrap(); - seed_worker_session_for_cleanup(temp.path(), &worker); - let before_restart = runtime.worker_detail(&runtime_worker_ref).unwrap(); - assert!(!before_restart.execution_reconstructed); - assert!(before_restart.worker_state.is_some_and(|snapshot| { - snapshot.last_command_id == 0 - && snapshot.last_finished_submission_request_id.is_none() - && snapshot.state - == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( - protocol::WorkerRunState::Running, - )) - })); - store - .accept_backend_job_result( - TEST_WORKSPACE_ID, - &worker, - &BackendJobResultSubmission { - job_id: request.job_id.clone(), - attempt_id: reservation.attempt.attempt_id.clone(), - input_digest: request.input_digest().unwrap(), - result: serde_json::json!({"persisted": true}), - }, - &now, - ) - .unwrap(); - assert_eq!( - store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &request.job_id, - &reservation.attempt.attempt_id, - ) - .unwrap() - .unwrap() - .worker_cleanup_state, - Some(BackendJobWorkerCleanupState::Pending) - ); - let attempt_id = reservation.attempt.attempt_id; - drop(runtime); - drop(embedded); - drop(store); - drop(execution); - - let restarted_store = - SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); - let restarted_execution = Arc::new(DeterministicExecutionBackend::default()); - restarted_execution.accept_restores(); - let restarted = WorkspaceApi::new_with_execution_backend( - config, - Arc::new(restarted_store), - restarted_execution, - ) - .await - .unwrap(); - let cleaned = wait_for_backend_job_worker_cleanup( - &restarted, - &request.job_id, - &attempt_id, - BackendJobWorkerCleanupState::Completed, - ) - .await; - assert_eq!(cleaned.result, Some(serde_json::json!({"persisted": true}))); - assert!(matches!( - restarted.runtime.worker(&worker), - Err(RuntimeRegistryError::UnknownWorker { .. }) - )); - } - - #[tokio::test] - async fn backend_job_fast_result_is_accepted_after_prebound_tracked_run() { - let temp = tempfile::tempdir().unwrap(); - let (api, execution) = test_api_with_recording_backend(temp.path()).await; - let request = BackendJobRequest { - job_id: "r".repeat(256), - purpose: "fast_result_contract_check".to_string(), - input_ref: "test://runner/fast-result/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits::default(), - }; - let acceptance_observed = Arc::new(std::sync::Mutex::new(None)); - let hook_api = api.clone(); - let hook_request = request.clone(); - let hook_observed = acceptance_observed.clone(); - let hook_root = temp.path().to_path_buf(); - execution.before_input_returns(move |worker| { - let source = - RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, worker.worker_id.to_string()); - seed_worker_session_for_cleanup(&hook_root, &source); - let accepted = hook_api - .accept_backend_job_result( - &source, - &BackendJobResultSubmission { - job_id: hook_request.job_id.clone(), - attempt_id: crate::backend_job::attempt_id(&hook_request.job_id, 1), - input_digest: hook_request.input_digest().unwrap(), - result: serde_json::json!({"valid": true}), - }, - ) - .is_ok(); - *hook_observed.lock().unwrap() = Some(accepted); - }); - - let dispatched = api.dispatch_backend_job(&request).unwrap(); - assert_eq!(*acceptance_observed.lock().unwrap(), Some(true)); - assert_eq!(dispatched.job.state, BackendJobState::Completed); - let expected_run_id = - crate::backend_job::runtime_run_id(&crate::backend_job::attempt_id(&request.job_id, 1)); - assert_eq!( - dispatched.attempt.runtime_run_id.as_deref(), - Some(expected_run_id.as_str()) - ); - let worker = dispatched.attempt.worker.clone().unwrap(); - wait_for_backend_job_worker_cleanup( - &api, - &request.job_id, - &dispatched.attempt.attempt_id, - BackendJobWorkerCleanupState::Completed, - ) - .await; - assert!(matches!( - api.runtime.worker(&worker), - Err(RuntimeRegistryError::UnknownWorker { .. }) - )); - } - - #[tokio::test] - async fn backend_job_result_http_response_precedes_cleanup_until_tracked_run_finishes() { - let temp = tempfile::tempdir().unwrap(); - let (api, execution) = test_api_with_recording_backend(temp.path()).await; - let request = BackendJobRequest { - job_id: "runner-result-response-fence-1".to_string(), - purpose: "result_response_fence_check".to_string(), - input_ref: "test://runner/result-response-fence/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits::default(), - }; - let dispatched = api.dispatch_backend_job(&request).unwrap(); - let worker = dispatched.attempt.worker.clone().unwrap(); - let runtime_run_id = dispatched.attempt.runtime_run_id.clone().unwrap(); - seed_worker_session_for_cleanup(temp.path(), &worker); - execution.publish_worker_state( - &worker, - protocol::WorkerStateSnapshot { - last_command_id: 0, - last_finished_submission_request_id: None, - state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( - protocol::WorkerRunState::Running, - )), - }, - ); - assert!( - api.runtime - .worker(&worker) - .unwrap() - .worker_state - .is_some_and(|snapshot| snapshot.state - == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( - protocol::WorkerRunState::Running - ))) - ); - - let Json(acceptance) = scoped_submit_backend_job_result( - State(api.clone()), - AxumPath(ScopedWorkspacePath { - workspace_id: TEST_WORKSPACE_ID.to_string(), - }), - ticket_check_headers(&worker), - Json(SubmitBackendJobResultRequest { - result: serde_json::json!({"valid": true}), - }), - ) - .await - .unwrap(); - assert_eq!(acceptance.job.state, BackendJobState::Completed); - let still_running = api.runtime.worker(&worker).unwrap(); - assert!(still_running.worker_state.is_some_and(|snapshot| { - snapshot.state - == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( - protocol::WorkerRunState::Running, - )) - })); - // Cleanup admission precedes its durable claim. The accepted job may - // still be queued, but neither queued nor executing ownership may stop - // this Run before the exact tracked finish marker below is published. - assert!(matches!( - api.store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &request.job_id, - &dispatched.attempt.attempt_id, - ) - .unwrap() - .unwrap() - .worker_cleanup_state, - Some(BackendJobWorkerCleanupState::Pending | BackendJobWorkerCleanupState::Executing) - )); - - execution.publish_worker_state( - &worker, - protocol::WorkerStateSnapshot { - last_command_id: 0, - last_finished_submission_request_id: Some(runtime_run_id), - state: protocol::WorkerState::Idle, - }, - ); - wait_for_backend_job_worker_cleanup( - &api, - &request.job_id, - &dispatched.attempt.attempt_id, - BackendJobWorkerCleanupState::Completed, - ) - .await; - assert!(matches!( - api.runtime.worker(&worker), - Err(RuntimeRegistryError::UnknownWorker { .. }) - )); - } - - #[tokio::test] - async fn backend_job_delivery_recovery_replays_same_tracked_notification_id() { - let temp = tempfile::tempdir().unwrap(); - let config = test_server_config(temp.path()); - let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); - let execution = Arc::new(DeterministicExecutionBackend::default()); - let api = WorkspaceApi::new_with_execution_backend( - config.clone(), - Arc::new(store), - execution.clone(), - ) - .await - .unwrap(); - seed_test_delivery_binding(&api); - let target = api - .spawn_workspace_worker( - EMBEDDED_WORKER_RUNTIME_ID, - WorkerSpawnRequest { - requested_worker_name: Some("job-notification-target".to_string()), - intent: WorkerSpawnIntent::WorkspaceCompanion, - singleton_key: None, - acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { - expected_segments: 0, - }, - profile: ProfileSelector::Builtin("builtin:companion".to_string()), - ticket_assignment: None, - initial_submit: Vec::new(), - workdir_attachment_requests: Vec::new(), - resolved_workdir_attachment_requests: Vec::new(), - resolved_workdir_attachments: Vec::new(), - resolved_config_bundle: Some(runtime_test_bundle()), - resolved_worker_observation_enabled: false, - resolved_worker_observation_grants: Vec::new(), - resolved_workspace_api: None, - resolved_memory_settings: None, - resolved_subjektiv_attached: false, - resolved_control_operation: None, - resolved_execution_grant: None, - }, - ) - .unwrap() - .worker - .unwrap() - .worker; - let request = BackendJobRequest { - job_id: "n".repeat(256), - purpose: "delivery_recovery_contract_check".to_string(), - input_ref: "test://runner/delivery-recovery/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: Some(target.clone()), - limits: crate::backend_job::BackendJobLimits { - timeout_seconds: 1, - ..Default::default() - }, - }; - let dispatched = api.dispatch_backend_job(&request).unwrap(); - let worker = dispatched.attempt.worker.clone().unwrap(); - let acceptance = api - .store - .accept_backend_job_result( - TEST_WORKSPACE_ID, - &worker, - &BackendJobResultSubmission { - job_id: request.job_id.clone(), - attempt_id: dispatched.attempt.attempt_id.clone(), - input_digest: request.input_digest().unwrap(), - result: serde_json::json!({"valid": true}), - }, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - .unwrap(); - let delivery_id = - crate::backend_job::delivery_id(&request.job_id, &acceptance.attempt.attempt_id); - let (_, claimed) = api - .store - .reserve_backend_job_delivery( - TEST_WORKSPACE_ID, - &delivery_id, - &request.job_id, - &target, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - .unwrap(); - assert!(claimed); - let recoverable = api - .store - .recover_backend_job_deliveries( - TEST_WORKSPACE_ID, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - .unwrap(); - assert_eq!(recoverable.len(), 1); - execution.take_input_request_ids(); - api.deliver_backend_job_result(&acceptance, &target); - let request_ids = execution.take_input_request_ids(); - assert!( - request_ids - .iter() - .any(|request_id| request_id.as_deref() == Some(delivery_id.as_str())), - "recovery must replay the durable delivery id as the tracked notification id: {request_ids:?}" - ); - assert_eq!(job_notification_evidence(&api, &delivery_id).2, true); - assert!( - api.store - .list_pending_job_notifications(TEST_WORKSPACE_ID, 100) - .unwrap() - .is_empty() - ); - notification_delivery::recover_job_notifications(&api); - assert!(execution.take_input_request_ids().is_empty()); - } - - #[tokio::test] - async fn backend_job_spawn_transport_error_terminalizes_reserved_attempt() { - let temp = tempfile::tempdir().unwrap(); - let api = test_api(temp.path()).await; - api.runtime - .unregister_if_idle(EMBEDDED_WORKER_RUNTIME_ID, 10) - .unwrap(); - let request = BackendJobRequest { - job_id: "runner-spawn-error-1".to_string(), - purpose: "spawn_error_contract_check".to_string(), - input_ref: "test://runner/spawn-error/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits::default(), - }; - - assert!(api.dispatch_backend_job(&request).is_err()); - let attempt = api - .store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &request.job_id, - &crate::backend_job::attempt_id(&request.job_id, 1), - ) - .unwrap() - .unwrap(); - assert_eq!(attempt.state, BackendJobAttemptState::Failed); - assert_eq!(attempt.failure_category.as_deref(), Some("dispatch_error")); - } - - #[tokio::test] - async fn backend_job_input_error_is_durable_unknown_and_not_automatically_retried() { - let temp = tempfile::tempdir().unwrap(); - let (api, execution) = test_api_with_recording_backend(temp.path()).await; - let cleanup_root = temp.path().to_path_buf(); - execution.before_input_returns(move |worker| { - let worker = - RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, worker.worker_id.to_string()); - seed_worker_session_for_cleanup(&cleanup_root, &worker); - }); - execution.reject_inputs("model transport unavailable"); - let request = BackendJobRequest { - job_id: "runner-input-failure-1".to_string(), - purpose: "failure_contract_check".to_string(), - input_ref: "test://runner/failure/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits::default(), - }; - - assert!(api.dispatch_backend_job(&request).is_err()); - let failed = api - .store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &request.job_id, - &crate::backend_job::attempt_id(&request.job_id, 1), - ) - .unwrap() - .unwrap(); - assert_eq!(failed.state, BackendJobAttemptState::Unknown); - assert_eq!( - failed.failure_category.as_deref(), - Some("input_outcome_unknown") - ); - let worker = failed.worker.clone().unwrap(); - wait_for_backend_job_worker_cleanup( - &api, - &request.job_id, - &failed.attempt_id, - BackendJobWorkerCleanupState::Completed, - ) - .await; - assert!(matches!( - api.runtime.worker(&worker), - Err(RuntimeRegistryError::UnknownWorker { .. }) - )); - assert!( - api.retry_backend_job(&request.job_id).is_err(), - "an ambiguous input outcome must not be retried automatically" - ); - assert!( - api.store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &request.job_id, - &crate::backend_job::attempt_id(&request.job_id, 2), - ) - .unwrap() - .is_none() - ); - } - - #[tokio::test] - async fn backend_job_recovery_drains_more_than_one_reserved_page() { - let temp = tempfile::tempdir().unwrap(); - let api = test_api(temp.path()).await; - api.runtime - .unregister_if_idle(EMBEDDED_WORKER_RUNTIME_ID, 10) - .unwrap(); - let mut jobs = Vec::new(); - for index in 0..(usize::from(ABSOLUTE_MAX_CONCURRENT_JOBS) + 6) { - let request = BackendJobRequest { - job_id: format!("restart-page-{index:03}"), - purpose: "restart_page_contract_check".to_string(), - input_ref: format!("fixture://restart-page/{index}"), - input: serde_json::json!({"index": index}), - instruction: "Return a structured fixture result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits { - max_concurrent_jobs: ABSOLUTE_MAX_CONCURRENT_JOBS, - ..Default::default() - }, - }; - api.store - .reserve_backend_job( - TEST_WORKSPACE_ID, - &request, - &format!("2026-09-01T00:{:02}:{:02}Z", index / 60, index % 60), - ) - .unwrap(); - jobs.push(request.job_id); - } - - api.recover_backend_jobs().unwrap(); - assert!( - api.store - .list_reserved_backend_job_attempts(TEST_WORKSPACE_ID, jobs.len()) - .unwrap() - .is_empty(), - "restart recovery must continue beyond its first bounded page when dispatches terminalize" - ); - assert!(jobs.iter().all(|job_id| { - api.store - .get_backend_job(TEST_WORKSPACE_ID, job_id) - .unwrap() - .is_some_and(|job| job.state == BackendJobState::Failed) - })); - } - - #[tokio::test] - async fn backend_job_recovery_dispatches_reserved_attempt_without_duplicate_worker() { - let temp = tempfile::tempdir().unwrap(); - let api = test_api(temp.path()).await; - let request = BackendJobRequest { - job_id: "runner-restart-1".to_string(), - purpose: "restart_contract_check".to_string(), - input_ref: "test://runner/restart/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits::default(), - }; - let reserved = api - .store - .reserve_backend_job( - TEST_WORKSPACE_ID, - &request, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - .unwrap(); - assert_eq!(reserved.attempt.state, BackendJobAttemptState::Reserved); - api.recover_backend_jobs().unwrap(); - - let recovered = api - .store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &request.job_id, - &reserved.attempt.attempt_id, - ) - .unwrap() - .unwrap(); - assert_eq!(recovered.state, BackendJobAttemptState::Dispatched); - let recovered_worker = recovered.worker.clone().expect("recovered Job Worker"); - let replay = api.dispatch_backend_job(&request).unwrap(); - assert!(replay.replayed); - assert_eq!(replay.attempt.worker, Some(recovered_worker)); - } - - #[tokio::test] - async fn backend_job_recovery_marks_unreconcilable_worker_execution_unknown() { - let temp = tempfile::tempdir().unwrap(); - let api = test_api(temp.path()).await; - let request = BackendJobRequest { - job_id: "runner-unknown-1".to_string(), - purpose: "unknown_contract_check".to_string(), - input_ref: "test://runner/unknown/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Return a structured check result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits::default(), - }; - let reserved = api - .store - .reserve_backend_job( - TEST_WORKSPACE_ID, - &request, - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - .unwrap(); - let missing_worker = RuntimeWorkerRef::new( - EMBEDDED_WORKER_RUNTIME_ID, - "0199a3ae-ecdf-7eb2-883e-a5b7b5f04a0d", - ); - api.store - .upsert_worker_registry(&WorkerRegistryRecord { - workspace_id: TEST_WORKSPACE_ID.to_string(), - worker: missing_worker.clone(), - display_name: "Missing Job Worker".to_string(), - profile: Some("builtin:backend-job".to_string()), - retention_state: "normal".to_string(), - transcript_ref: None, - session_ref: None, - summary_ref: None, - diagnostics_ref: None, - created_at: "2026-09-01T00:00:00Z".to_string(), - updated_at: "2026-09-01T00:00:00Z".to_string(), - }) - .unwrap(); - let (_, claimed) = api - .store - .claim_backend_job_attempt_dispatch( - TEST_WORKSPACE_ID, - &request.job_id, - &reserved.attempt.attempt_id, - "2026-09-01T00:00:00Z", - ) - .unwrap(); - assert!(claimed); - api.store - .bind_backend_job_attempt_worker( - TEST_WORKSPACE_ID, - &request.job_id, - &reserved.attempt.attempt_id, - &missing_worker, - Some("missing-runtime-run"), - &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), - ) - .unwrap(); - - api.recover_backend_jobs().unwrap(); - let attempt = api - .store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &request.job_id, - &reserved.attempt.attempt_id, - ) - .unwrap() - .unwrap(); - assert_eq!(attempt.state, BackendJobAttemptState::Unknown); - assert_eq!( - attempt.failure_category.as_deref(), - Some("recovery_worker_unknown") - ); - assert_eq!( - api.store - .get_backend_job(TEST_WORKSPACE_ID, &request.job_id) - .unwrap() - .unwrap() - .state, - BackendJobState::Unknown - ); - } - - #[tokio::test] - async fn backend_job_timeout_fences_late_structured_result() { - let temp = tempfile::tempdir().unwrap(); - let api = test_api(temp.path()).await; - let request = BackendJobRequest { - job_id: "runner-timeout-1".to_string(), - purpose: "timeout_contract_check".to_string(), - input_ref: "test://runner/timeout/1".to_string(), - input: serde_json::json!({"value": "immutable"}), - instruction: "Wait for a structured result.".to_string(), - profile: "builtin:backend-job".to_string(), - grants: Default::default(), - serialization_key: None, - source_worker: None, - notification_target: None, - limits: crate::backend_job::BackendJobLimits { - timeout_seconds: 1, - max_attempts: 1, - ..Default::default() - }, - }; - let dispatched = api.dispatch_backend_job(&request).unwrap(); - let worker = dispatched.attempt.worker.clone().unwrap(); - seed_worker_session_for_cleanup(temp.path(), &worker); - tokio::time::sleep(std::time::Duration::from_millis(1_200)).await; - let attempt = api - .store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &request.job_id, - &dispatched.attempt.attempt_id, - ) - .unwrap() - .unwrap(); - assert_eq!(attempt.state, BackendJobAttemptState::Failed); - assert_eq!(attempt.failure_category.as_deref(), Some("timeout")); - wait_for_backend_job_worker_cleanup( - &api, - &request.job_id, - &dispatched.attempt.attempt_id, - BackendJobWorkerCleanupState::Completed, - ) - .await; - assert!(matches!( - api.runtime.worker(&worker), - Err(RuntimeRegistryError::UnknownWorker { .. }) - )); - assert!( - api.accept_backend_job_result( - &worker, - &BackendJobResultSubmission { - job_id: request.job_id.clone(), - attempt_id: dispatched.attempt.attempt_id, - input_digest: request.input_digest().unwrap(), - result: serde_json::json!({"late": true}), - }, - ) - .is_err() - ); - } + mod backend_job_tests; #[tokio::test] async fn scoped_runtime_workdir_access_requires_workspace_binding() { diff --git a/crates/workspace-server/src/server/backend_job_recovery.rs b/crates/workspace-server/src/server/backend_job_recovery.rs new file mode 100644 index 000000000..7c1619296 --- /dev/null +++ b/crates/workspace-server/src/server/backend_job_recovery.rs @@ -0,0 +1,1217 @@ +//! Recovery belongs to the producer of the accepted attempt, not to callers replaying it. +//! Runtime receipts, domain completion and execution/Host termination are independent facts. +//! Only initial dispatch opens Host admission; a reconstructed empty map proves nothing. +use super::*; +use crate::backend_job::BackendJobRecord; +use crate::backend_job_host_effects::{JobHostEffects, JobHostOwner, JobHostRequestLease}; +use std::collections::HashSet; +use std::time::Duration; + +pub(super) const SETTLEMENT_BACKOFF_MS: [u64; 5] = [100, 200, 400, 800, 1600]; + +#[derive(Default)] +pub(super) struct RecoveryOwners { + running: Mutex>, + cleanup_running: Mutex>, + gates: Mutex>>>, + hosts: Mutex)>>, + results: Mutex>, + submitted: Mutex>, + stopping: std::sync::atomic::AtomicBool, + changed: tokio::sync::Notify, + #[cfg(test)] + pub(super) storage_probe_started: tokio::sync::Notify, +} + +impl RecoveryOwners { + async fn pause(&self, duration: Duration) { + let notified = self.changed.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.stopping.load(Ordering::Acquire) { + return; + } + tokio::select! { _ = tokio::time::sleep(duration) => {}, _ = notified => {} } + } + + pub(super) fn dispatch_gate(&self, attempt: &str) -> Arc> { + self.gates + .lock() + .unwrap_or_else(|p| p.into_inner()) + .entry(attempt.to_owned()) + .or_default() + .clone() + } + + pub(super) fn initially_admit(&self, owner: JobHostOwner) -> Result<()> { + let mut hosts = self + .hosts + .lock() + .map_err(|_| Error::Store("Job Host owners poisoned".into()))?; + match hosts.get(&owner.attempt_id) { + Some((existing, _)) if existing == &owner => Ok(()), + Some(_) => Err(Error::RepositoryConflict("Job Host owner changed".into())), + None => { + hosts.insert( + owner.attempt_id.clone(), + (owner.clone(), JobHostEffects::initially_admit(owner)), + ); + Ok(()) + } + } + } + + fn initial_submission_pending(&self, owner: &JobHostOwner) -> Result { + self.effects(owner)?; + Ok(!self + .submitted + .lock() + .map_err(|_| Error::Store("Job submission ownership poisoned".into()))? + .contains(&owner.attempt_id)) + } + + fn claim_initial_submission(&self, owner: &JobHostOwner) -> Result { + self.effects(owner)?; + // Set BEFORE the external call. Once set, even a lost response can only + // be reconciled by receipt; restoring an empty owner never reaches here. + Ok(self + .submitted + .lock() + .map_err(|_| Error::Store("Job submission ownership poisoned".into()))? + .insert(owner.attempt_id.clone())) + } + + pub(super) fn effects(&self, expected: &JobHostOwner) -> Result> { + self.hosts.lock().map_err(|_| Error::Store("Job Host owners poisoned".into()))? + .get(&expected.attempt_id).filter(|(owner, _)| owner == expected) + .map(|(_, effects)| effects.clone()) + .ok_or_else(|| Error::RepositoryConflict( + "Original live Job Host owner unavailable; retained operation/writer fences required".into(), + )) + } +} + +struct RecoveryLease { + owners: Arc, + attempt: String, + cleanup: bool, +} +impl Drop for RecoveryLease { + fn drop(&mut self) { + let set = if self.cleanup { + &self.owners.cleanup_running + } else { + &self.owners.running + }; + set.lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&self.attempt); + self.owners.changed.notify_waiters(); + } +} + +fn input_request(attempt: &BackendJobAttemptRecord, content: String) -> WorkerInputRequest { + WorkerInputRequest { + source: None, + kind: WorkerInputKind::User, + content, + submission_request_id: attempt.runtime_run_id.clone(), + segments: None, + } +} + +fn receipt_matches( + attempt: &BackendJobAttemptRecord, + result: &crate::hosts::WorkerInputResult, +) -> bool { + result.state == InternalWorkerOperationState::Accepted + && result.disposition == WorkerInputDisposition::Accepted + && attempt.worker.as_ref() == Some(&result.worker) + && attempt.runtime_run_id.is_some() + && result.runtime_run_id == attempt.runtime_run_id +} + +fn transient_storage(error: &Error) -> bool { + matches!(error, Error::Sqlite(rusqlite::Error::SqliteFailure(error, _)) + if matches!(error.code, rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked + | rusqlite::ErrorCode::SystemIoFailure | rusqlite::ErrorCode::DiskFull + | rusqlite::ErrorCode::CannotOpen)) + // Lost DB responses are not definitive execution failures. Reconcile the + // same owner through a serialized storage probe before refilling budget. + || matches!(error, Error::DbCapacityExceeded | Error::DbOperationOutcomeUnknown) +} + +impl WorkspaceApi { + pub(super) async fn stop_backend_job_recovery(&self) { + self.backend_job_recovery + .stopping + .store(true, Ordering::Release); + self.backend_job_recovery.changed.notify_waiters(); + loop { + let notified = self.backend_job_recovery.changed.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + let executions_empty = self + .backend_job_recovery + .running + .lock() + .unwrap_or_else(|p| p.into_inner()) + .is_empty(); + let cleanups_empty = self + .backend_job_recovery + .cleanup_running + .lock() + .unwrap_or_else(|p| p.into_inner()) + .is_empty(); + if executions_empty && cleanups_empty { + return; + } + // Already-admitted Host/DB closures are awaited, never aborted/dropped + // into a fabricated termination receipt. Stop only future producer steps. + notified.await; + } + } + + /// Retain a bounded result on its original live producer when storage fails. + /// Reconciliation re-enters all domain/source/CAS checks; memory is not acceptance. + pub fn accept_backend_job_result( + &self, + source: &RuntimeWorkerRef, + submission: &BackendJobResultSubmission, + ) -> Result { + let result = self.accept_backend_job_result_inner(source, submission); + if result.as_ref().is_err_and(|error| { + transient_storage(error) || matches!(error, Error::DbOperationOutcomeUnknown) + }) { + let bytes = serde_json::to_vec(submission) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + if bytes.len() + <= usize::try_from(crate::backend_job::ABSOLUTE_MAX_RESULT_BYTES) + .unwrap_or(usize::MAX) + + 4096 + { + let valid = self + .backend_job_recovery + .hosts + .lock() + .map_err(|_| Error::Store("Job Host owners poisoned".into()))? + .get(&submission.attempt_id) + .is_some_and(|(owner, _)| { + &owner.worker == source + && owner.job_id == submission.job_id + && owner.input_digest == submission.input_digest + }); + if valid { + let mut results = self + .backend_job_recovery + .results + .lock() + .map_err(|_| Error::Store("Job retained results poisoned".into()))?; + if let Some((saved_source, saved)) = results.get(&submission.attempt_id) { + if saved_source != source + || serde_json::to_value(saved).ok() + != serde_json::to_value(submission).ok() + { + return Err(Error::RepositoryConflict( + "Different result already retained for this execution".into(), + )); + } + } else { + results.insert( + submission.attempt_id.clone(), + (source.clone(), submission.clone()), + ); + } + } + } + } + if result.is_ok() { + self.backend_job_recovery + .results + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&submission.attempt_id); + } + result + } + + pub(super) fn retain_backend_job_recovery( + &self, + attempt: &BackendJobAttemptRecord, + category: &str, + detail: &str, + next: &str, + ) -> Result { + self.store.record_backend_job_reconciliation( + self.workspace_id(), + attempt, + category, + detail, + next, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + } + + pub(super) fn submit_initial_backend_job_input( + &self, + job: &BackendJobRecord, + attempt: &BackendJobAttemptRecord, + content: String, + ) -> Result<()> { + let worker = attempt + .worker + .as_ref() + .ok_or_else(|| Error::Store("Job input missing Worker".into()))?; + let request = input_request(attempt, content); + if !self + .backend_job_recovery + .claim_initial_submission(&JobHostOwner::try_from(attempt)?)? + { + let receipt = self + .runtime + .reconcile_worker_input(worker, request) + .map_err(RuntimeRegistryError::into_error)?; + return if receipt_matches(attempt, &receipt) { + Ok(()) + } else { + Err(Error::RepositoryConflict( + "Previously attempted Job input requires exact receipt reconciliation".into(), + )) + }; + } + let dispatched = self.runtime.send_input(worker, request.clone()); + if let Ok(result) = &dispatched { + if receipt_matches(attempt, result) { + return Ok(()); + } + if result.worker == *worker && result.disposition == WorkerInputDisposition::Rejected { + self.finish_backend_job_attempt( + &attempt.job_id, + &attempt.attempt_id, + BackendJobAttemptState::Failed, + "input_rejected", + &runtime_diagnostics_message(&result.diagnostics), + )?; + return Err(Error::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "backend_job_input_rejected".into(), + message: "Runtime definitively rejected Job input".into(), + }); + } + } + // Lost/mismatched acknowledgements require read-only receipt lookup, never Submit replay. + let receipt = self.runtime.reconcile_worker_input(worker, request); + if receipt + .as_ref() + .is_ok_and(|result| receipt_matches(attempt, result)) + { + return Ok(()); + } + let detail = match dispatched { + Err(error) => error.message(), + Ok(result) => runtime_diagnostics_message(&result.diagnostics), + }; + let current = self + .store + .get_backend_job_attempt( + self.workspace_id(), + &job.request.job_id, + &attempt.attempt_id, + )? + .ok_or_else(|| Error::Store("Job input owner disappeared".into()))?; + if !current.state.terminal() { + self.retain_backend_job_recovery(¤t, "submission_receipt_unconfirmed", &detail, + "Read exact Runtime submission and domain receipts; never resubmit non-idempotent input")?; + } + Err(Error::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: "backend_job_submission_receipt_unconfirmed".into(), + message: detail, + }) + } + + pub(super) fn own_backend_job_recovery(&self, attempt: &BackendJobAttemptRecord) { + let Ok(handle) = tokio::runtime::Handle::try_current() else { + return; + }; + let mut running = self + .backend_job_recovery + .running + .lock() + .unwrap_or_else(|p| p.into_inner()); + if self.backend_job_recovery.stopping.load(Ordering::Acquire) + || !running.insert(attempt.attempt_id.clone()) + { + return; + } + drop(running); + let lease = RecoveryLease { + owners: self.backend_job_recovery.clone(), + attempt: attempt.attempt_id.clone(), + cleanup: false, + }; + let api = self.clone(); + let expected = attempt.clone(); + handle.spawn(async move { + let _lease = lease; + if let Err(error) = api.run_backend_job_recovery(expected.clone()).await { + tracing::error!(job_id=%expected.job_id, attempt_id=%expected.attempt_id, %error, + "Backend Job recovery stopped with retained execution/input; no replacement dispatched"); + } + }); + } + + pub(super) fn own_backend_job_cleanup(&self, attempt: &BackendJobAttemptRecord) -> bool { + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return false; + }; + let mut running = self + .backend_job_recovery + .cleanup_running + .lock() + .unwrap_or_else(|p| p.into_inner()); + if self.backend_job_recovery.stopping.load(Ordering::Acquire) + || !running.insert(attempt.attempt_id.clone()) + { + return false; + } + drop(running); + let lease = RecoveryLease { + owners: self.backend_job_recovery.clone(), + attempt: attempt.attempt_id.clone(), + cleanup: true, + }; + let api = self.clone(); + let expected = attempt.clone(); + // Tokio tasks do not inherit OPERATION_LIFETIME. Admitting cleanup from + // inside the result's Job lease would make close_and_drain wait on itself. + runtime.spawn(async move { + let _lease = lease; + if let Err(error) = api.run_backend_job_cleanup(expected.clone()).await { + tracing::warn!(job_id=%expected.job_id, attempt_id=%expected.attempt_id, %error, + "Job cleanup stopped; execution/input/resource fences remain retained"); + } + }); + true + } + + pub(super) async fn run_backend_job_cleanup( + &self, + expected: BackendJobAttemptRecord, + ) -> Result<()> { + let claimed = Arc::new(std::sync::atomic::AtomicBool::new(false)); + // Pin the observed canonical operation across this owner's storage retries. + let removal_owner = Arc::new(Mutex::new(None::<(String, String)>)); + let mut failures = 0; + loop { + if self.backend_job_recovery.stopping.load(Ordering::Acquire) { + return Ok(()); + } + let api = self.clone(); + let owner = expected.clone(); + let mine = claimed.clone(); + let removal_owner = removal_owner.clone(); + let runtime = tokio::runtime::Handle::current(); + let result = self + .host_executors + .cleanup()? + .run(move || { + let current = api + .store + .get_backend_job_attempt( + api.workspace_id(), + &owner.job_id, + &owner.attempt_id, + )? + .ok_or_else(|| Error::Store("Cleanup owner disappeared".into()))?; + if current.worker != owner.worker + || current.runtime_run_id != owner.runtime_run_id + || current.input_digest != owner.input_digest + { + return Err(Error::RepositoryConflict( + "Cleanup execution owner changed".into(), + )); + } + if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) + { + return api.finish_backend_job_cleanup_retirement( + &owner.job_id, + &owner.attempt_id, + ); + } + let attempt = if current.worker_cleanup_state + == Some(BackendJobWorkerCleanupState::Executing) + { + if !mine.load(Ordering::Acquire) { + // Executing is not a lease we can steal after restart. + // First require the exact durable execution/Host fence, + // then recover (never plan) its canonical removal owner. + api.require_backend_job_attempt_effects_ended(¤t)?; + let worker = current.worker.as_ref().ok_or_else(|| + Error::RepositoryConflict("Executing cleanup Worker missing".into()))?; + let saved = api.config_store.recover_worker_removal_execution( + api.workspace_id(), worker, + ).map_err(|error| Error::Store(error.to_string()))? + .ok_or_else(|| Error::RepositoryConflict( + "Executing cleanup has no retained canonical removal operation; ownership cannot be reconstructed".into()))?; + if saved.plan.workspace_id != api.workspace_id() + || saved.plan.worker != *worker + || saved.plan.operation_id.is_empty() + || saved.plan.input_fingerprint.is_empty() + || saved.runtime_request.workspace_id != api.workspace_id() + || saved.runtime_request.source_runtime_id != worker.runtime_id + || saved.runtime_request.worker_id.to_string() != worker.worker_id + || saved.runtime_request.operation_id != saved.plan.operation_id + || saved.runtime_request.input_fingerprint != saved.plan.input_fingerprint + { + return Err(Error::RepositoryConflict( + "Executing cleanup canonical removal owner mismatch".into())); + } + let expected = (saved.plan.operation_id, saved.plan.input_fingerprint); + let mut pinned = removal_owner.lock().map_err(|_| Error::Store( + "Cleanup removal owner poisoned".into()))?; + if pinned.as_ref().is_some_and(|prior| prior != &expected) { + return Err(Error::RepositoryConflict( + "Executing cleanup canonical removal owner changed".into())); + } + *pinned = Some(expected); + } + current + } else { + let history = api + .store + .list_backend_job_attempts(api.workspace_id(), &owner.job_id)?; + let (attempt, acquired) = + api.store.claim_backend_job_worker_cleanup_owned( + api.workspace_id(), + &history, + ¤t, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + )?; + if !acquired { + return Err(Error::RepositoryConflict( + "Cleanup belongs to another admitted owner".into(), + )); + } + mine.store(true, Ordering::Release); + attempt + }; + let retained_removal = removal_owner.lock().map_err(|_| Error::Store( + "Cleanup removal owner poisoned".into()))?.clone(); + runtime.block_on(async { + if attempt.state == BackendJobAttemptState::Completed + || attempt.failure_category.as_deref() == Some("invalid_result") + { + api.wait_for_backend_job_worker_run_teardown(&attempt) + .await?; + } + api.cleanup_backend_job_attempt_worker(attempt, retained_removal).await + }) + }) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) + if failures < 5 + && (transient_storage(&error) + || matches!(error, Error::RuntimeOperationFailed { .. })) => + { + self.backend_job_recovery + .pause(Duration::from_millis(100 << failures)) + .await; + failures += 1; + } + Err(error) if transient_storage(&error) => { + tracing::warn!(attempt_id=%expected.attempt_id, %error, "Cleanup settlement budget exhausted; exact-owner storage probe only"); + let mut delay = Duration::from_secs(1); + loop { + if self.backend_job_recovery.stopping.load(Ordering::Acquire) { + return Ok(()); + } + self.backend_job_recovery.pause(delay).await; + let api = self.clone(); + let owner = expected.clone(); + let probe = self + .host_executors + .cleanup()? + .run(move || { + let current = api + .store + .get_backend_job_attempt( + api.workspace_id(), + &owner.job_id, + &owner.attempt_id, + )? + .ok_or_else(|| { + Error::Store("Cleanup probe owner disappeared".into()) + })?; + api.store + .probe_backend_job_recovery(api.workspace_id(), ¤t) + }) + .await; + match probe { + Ok(true) => { + failures = 0; + break; + } + Ok(false) => { + return Err(Error::RepositoryConflict( + "Cleanup probe owner changed".into(), + )); + } + Err(error) if transient_storage(&error) => { + delay = (delay * 2).min(Duration::from_secs(30)); + } + Err(error) => return Err(error), + } + } + } + Err(error) => return Err(error), + } + } + } + + async fn run_backend_job_recovery(&self, expected: BackendJobAttemptRecord) -> Result<()> { + // Only transient storage failure enters probe mode. Timers, unrelated writes and + // receipt failures never refill this settlement budget or authorize execution. + let mut storage_failures = 0; + let mut definitive_failure: Option = None; + loop { + if self.backend_job_recovery.stopping.load(Ordering::Acquire) { + return Ok(()); + } + let api = self.clone(); + let owner = expected.clone(); + let failure = definitive_failure.clone(); + let step = self + .host_executors + .host()? + .run(move || { + if let Some(detail) = failure { + api.fail_backend_job_recovery(&owner, &detail) + } else { + api.recover_backend_job_step(&owner) + } + }) + .await; + match step { + Ok(true) => return Ok(()), + Ok(false) => { + self.backend_job_recovery + .pause(Duration::from_millis(250)) + .await; + } + Err(error) if transient_storage(&error) => { + if let Some(delay) = SETTLEMENT_BACKOFF_MS.get(storage_failures) { + storage_failures += 1; + self.backend_job_recovery + .pause(Duration::from_millis(*delay)) + .await; + continue; + } + tracing::warn!(job_id=%expected.job_id, attempt_id=%expected.attempt_id, %error, + "Job settlement budget exhausted; owner now probes exact-attempt storage only"); + let mut delay = Duration::from_secs(1); + loop { + if self.backend_job_recovery.stopping.load(Ordering::Acquire) { + return Ok(()); + } + self.backend_job_recovery.pause(delay).await; + let api = self.clone(); + let owner = expected.clone(); + match self + .host_executors + .host()? + .run(move || api.probe_backend_job_storage(&owner)) + .await + { + Ok(()) => { + storage_failures = 0; + break; + } + Err(error) if transient_storage(&error) => { + delay = (delay * 2).min(Duration::from_secs(30)); + } + Err(error) => return Err(error), + } + } + } + Err(error) if definitive_failure.is_none() => { + definitive_failure = Some(error.to_string()); + } + Err(error) => return Err(error), + } + } + } + + fn fail_backend_job_recovery( + &self, + owner: &BackendJobAttemptRecord, + detail: &str, + ) -> Result { + let current = self + .store + .get_backend_job_attempt(self.workspace_id(), &owner.job_id, &owner.attempt_id)? + .ok_or_else(|| Error::Store("Failed recovery attempt disappeared".into()))?; + let job = self + .store + .get_backend_job(self.workspace_id(), &owner.job_id)? + .ok_or_else(|| Error::Store("Failed recovery Job disappeared".into()))?; + if job.current_attempt != owner.attempt + || job.input_digest != owner.input_digest + || self + .store + .backend_job_was_migrated(self.workspace_id(), &owner.job_id)? + { + return Ok(true); + } + if !current.state.terminal() { + self.finish_backend_job_attempt( + &owner.job_id, + &owner.attempt_id, + BackendJobAttemptState::Failed, + "recovery_cannot_continue", + detail, + )?; + } + self.schedule_backend_job_worker_cleanup(¤t); + Ok(true) + } + + fn probe_backend_job_storage(&self, owner: &BackendJobAttemptRecord) -> Result<()> { + #[cfg(test)] + self.backend_job_recovery.storage_probe_started.notify_one(); + let current = self + .store + .get_backend_job_attempt(self.workspace_id(), &owner.job_id, &owner.attempt_id)? + .ok_or_else(|| Error::Store("Job storage probe owner disappeared".into()))?; + // A read or another Job's commit cannot declare this accepted operation + // recoverable. The Store commits an exact-owner probe without rewriting + // the saved result or original cause, including terminal cleanup owners. + if self + .store + .probe_backend_job_recovery(self.workspace_id(), ¤t)? + { + Ok(()) + } else { + Err(Error::RepositoryConflict( + "Job storage probe owner changed".into(), + )) + } + } + + fn recover_backend_job_step(&self, expected: &BackendJobAttemptRecord) -> Result { + let Some(attempt) = self.store.get_backend_job_attempt( + self.workspace_id(), + &expected.job_id, + &expected.attempt_id, + )? + else { + return Err(Error::Store("Accepted Job attempt missing".into())); + }; + let job = self + .store + .get_backend_job(self.workspace_id(), &attempt.job_id)? + .ok_or_else(|| Error::Store("Accepted Job missing".into()))?; + if job.current_attempt != attempt.attempt || job.input_digest != expected.input_digest { + return Err(Error::RepositoryConflict( + "Recovery owner no longer current".into(), + )); + } + let retained = self + .backend_job_recovery + .results + .lock() + .map_err(|_| Error::Store("Job retained results poisoned".into()))? + .get(&attempt.attempt_id) + .cloned(); + if let Some((source, submission)) = retained { + match self.accept_backend_job_result(&source, &submission) { + Ok(_) => return Ok(false), + Err(error) if transient_storage(&error) => return Err(error), + Err(error @ Error::DbOperationOutcomeUnknown) => return Err(error), + Err(error) => { + self.backend_job_recovery + .results + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&attempt.attempt_id); + if !attempt.state.terminal() { + self.finish_backend_job_attempt( + &attempt.job_id, + &attempt.attempt_id, + BackendJobAttemptState::Failed, + "retained_result_rejected", + &error.to_string(), + )?; + } + return Ok(false); + } + } + } + if attempt.state.terminal() { + self.schedule_backend_job_worker_cleanup(&attempt); + self.drain_reserved_backend_jobs(); + return Ok(true); + } + if self + .store + .backend_job_was_migrated(self.workspace_id(), &attempt.job_id)? + { + return Ok(true); + } + if attempt.state == BackendJobAttemptState::Reserved { + self.dispatch_backend_job_reservation(BackendJobReservation { + job, + attempt, + replayed: true, + resource_reused: false, + })?; + return Ok(false); + } + let gate = self.backend_job_recovery.dispatch_gate(&attempt.attempt_id); + let _guard = gate + .lock() + .map_err(|_| Error::Store("Job recovery dispatch gate poisoned".into()))?; + // Reread after acquiring the producer gate: initial dispatch might have bound/finished it. + let mut attempt = self + .store + .get_backend_job_attempt(self.workspace_id(), &attempt.job_id, &attempt.attempt_id)? + .ok_or_else(|| Error::Store("Job receipt owner missing".into()))?; + if attempt.state.terminal() { + return Ok(false); + } + if job.request.grants.subjektiv_journal.is_some() { + if let Err(error) = subjektiv_journal::reconcile_attempt(self, &job, &attempt) { + if transient_storage(&error) { + return Err(error); + } + // Incomplete generation/missing termination evidence is not a + // rejected dispatch or permission to reselect Journal input. + self.retain_backend_job_recovery(&attempt, "journal_receipt_reconciliation", + &error.to_string(), "Read retained generation/checkpoints and exact execution fences until deadline")?; + } + attempt = self + .store + .get_backend_job_attempt(self.workspace_id(), &attempt.job_id, &attempt.attempt_id)? + .ok_or_else(|| Error::Store("Journal receipt owner missing".into()))?; + if attempt.state.terminal() { + return Ok(false); + } + } + if job.request.grants.subjektiv_consolidation.is_some() + && let Some(result) = subjektiv_job::lookup_result(self, &job, &attempt)? + { + let current = self.store.reconcile_backend_job_domain_result( + self.workspace_id(), + &attempt, + &BackendJobResultSubmission { + job_id: attempt.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_digest: attempt.input_digest.clone(), + result, + }, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + )?; + let saved = self + .store + .get_backend_job(self.workspace_id(), &attempt.job_id)? + .ok_or_else(|| Error::Store("Reconciled consolidation Job missing".into()))?; + notification_delivery::deliver_job_result( + self, + &BackendJobResultAcceptance { + job: saved, + attempt: current.clone(), + replayed: false, + }, + ); + self.schedule_backend_job_worker_cleanup(¤t); + return Ok(false); + } + let deadline = chrono::DateTime::parse_from_rfc3339(&attempt.deadline_at) + .map_err(|error| Error::Store(error.to_string()))?; + if Utc::now() >= deadline { + self.finish_backend_job_attempt(&attempt.job_id, &attempt.attempt_id, + BackendJobAttemptState::Failed, "timeout", + "Attempt deadline exceeded while reconciling exact Runtime/domain receipts; termination and Host cleanup remain independent")?; + return Ok(false); + } + match attempt.state { + BackendJobAttemptState::Dispatching => { + self.reconcile_backend_job_allocation(&job, &attempt)? + } + BackendJobAttemptState::Dispatched => { + let owner = JobHostOwner::try_from(&attempt)?; + if self + .backend_job_recovery + .initial_submission_pending(&owner) + .unwrap_or(false) + { + // A failed binding save can precede the first send. Only the + // original live producer's never-sent claim permits that send; + // restored/reconciled allocations cannot create this authority. + if let Err(error) = self.submit_initial_backend_job_input( + &job, + &attempt, + job.load_executable_request()? + .worker_input(&attempt.attempt_id)?, + ) { + if transient_storage(&error) { + return Err(error); + } + } + return Ok(false); + } + let worker = attempt + .worker + .as_ref() + .ok_or_else(|| Error::Store("Dispatched Job Worker missing".into()))?; + let receipt = self.runtime.reconcile_worker_input( + worker, + input_request( + &attempt, + job.load_executable_request()? + .worker_input(&attempt.attempt_id)?, + ), + ); + match receipt { + Ok(result) if receipt_matches(&attempt, &result) => { + subjektiv_journal::attest_delivery( + self, + &BackendJobReservation { + job, + attempt, + replayed: true, + resource_reused: false, + }, + )?; + } + Ok(result) => { + self.retain_backend_job_recovery(&attempt, "submission_receipt_unconfirmed", + &runtime_diagnostics_message(&result.diagnostics), + "Read same submission/domain receipts until deadline; never blindly repeat input")?; + } + Err(error) => { + self.retain_backend_job_recovery(&attempt, "submission_receipt_unavailable", &error.message(), + "Read same submission/domain receipts until deadline; keep execution and input owned")?; + } + } + } + _ => {} + } + Ok(false) + } + + /// Only initial dispatch resolves the live Profile and constructs create intent. + /// Accepted-allocation receipt recovery reads the immutable saved reservation instead. + pub(super) fn backend_job_spawn_request( + &self, + job: &BackendJobRecord, + attempt: &BackendJobAttemptRecord, + ) -> Result { + let request = job.load_executable_request()?; + let (profile, config_bundle) = self.resolve_backend_job_profile(&request)?; + Ok(WorkerSpawnRequest { + requested_worker_name: Some(format!("job:{}", request.purpose)), + singleton_key: None, + intent: WorkerSpawnIntent::BackendJob { + job_id: request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + purpose: request.purpose.clone(), + input_digest: request.input_digest()?, + subjektiv_consolidation: request.grants.subjektiv_consolidation.is_some(), + subjektiv_journal: request.grants.subjektiv_journal.is_some(), + }, + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 0, + }, + profile, + ticket_assignment: None, + initial_submit: Vec::new(), + workdir_attachment_requests: Vec::new(), + resolved_workdir_attachment_requests: Vec::new(), + resolved_workdir_attachments: Vec::new(), + resolved_config_bundle: Some(config_bundle), + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_workspace_api: None, + resolved_memory_settings: None, + // Journal Jobs use their independent frozen Job Host grant, not the + // main-Subject/Memory attachment or its resident/extraction policy. + resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some(), + resolved_control_operation: Some(WorkerControlOperation { + operation_id: allocation_key(&attempt.attempt_id), + input_fingerprint: job.intent_fingerprint.clone(), + }), + resolved_execution_grant: None, + }) + } + + fn reconcile_backend_job_allocation( + &self, + job: &BackendJobRecord, + attempt: &BackendJobAttemptRecord, + ) -> Result<()> { + let allocation = + self.config_store + .lookup_backend_job_allocation(self.workspace_id(), job, attempt)?; + let Some((allocation, runtime_id)) = allocation else { + self.retain_backend_job_recovery( + attempt, + "allocation_receipt_unconfirmed", + "Original create reservation is not yet available", + "Read the original reservation until deadline; never allocate a replacement", + )?; + return Ok(()); + }; + let worker = RuntimeWorkerRef::new(runtime_id, allocation.worker_id.to_string()); + // Removed catalog entries and foreign bindings cannot be resurrected from + // Runtime metadata. Before attempt binding, a Job projection may be absent. + if let Some(projection) = self + .store + .worker_registry_projection(self.workspace_id(), &worker)? + { + if projection.registry.retention_state == "removed" + || projection.job.as_ref().is_some_and(|bound| { + bound.job_id != attempt.job_id || bound.attempt_id != attempt.attempt_id + }) + { + return Err(Error::RepositoryConflict( + "Original Job allocation was removed or rebound".into(), + )); + } + } + let request = job.load_executable_request()?; + let binding = worker_runtime::catalog::BackendJobExecutionBinding { + job_id: attempt.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_digest: Some(attempt.input_digest.clone()), + subjektiv_consolidation: request.grants.subjektiv_consolidation.is_some(), + subjektiv_journal: request.grants.subjektiv_journal.is_some(), + }; + match self.runtime.lookup_job_worker_creation( + &worker, + &allocation.create_fingerprint, + &binding, + ) { + Ok(Some(receipt)) if receipt.worker == worker => { + // This read never activates/restores the Worker, submits input, or + // creates a Host owner from a reconstructed empty effects map. + self.worker_projection.publish_ordered(|| { + self.store + .bind_backend_job_attempt_worker( + self.workspace_id(), + &attempt.job_id, + &attempt.attempt_id, + &worker, + Some(&crate::backend_job::runtime_run_id(&attempt.attempt_id)), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .map(|(commit, bound)| (bound, commit)) + })?; + } + outcome => { + let detail = match outcome { + Err(error) => error.message(), + _ => "No exact Runtime Job creation receipt available".into(), + }; + self.retain_backend_job_recovery(attempt, "allocation_receipt_unconfirmed", &detail, + "Read original allocation receipt until attempt deadline; do not allocate a replacement")?; + } + } + Ok(()) + } + + pub(super) fn finish_backend_job_cleanup_retirement( + &self, + job_id: &str, + attempt_id: &str, + ) -> Result<()> { + let current = self + .store + .get_backend_job_attempt(self.workspace_id(), job_id, attempt_id)? + .ok_or_else(|| Error::Store("Cleaned Job attempt disappeared".into()))?; + self.require_backend_job_attempt_effects_ended(¤t)?; + if current.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) { + return Err(Error::RepositoryConflict( + "Canonical Job cleanup not completed".into(), + )); + } + let job = self + .store + .get_backend_job(self.workspace_id(), job_id)? + .ok_or_else(|| Error::Store("Cleaned Job disappeared".into()))?; + if job.request.grants.subjektiv_journal.is_some() + && job.state == BackendJobState::Completed + && job.current_attempt == current.attempt + && current.state == BackendJobAttemptState::Completed + && !matches!( + job.input_disposition, + crate::backend_job::JobInputDisposition::Erased + | crate::backend_job::JobInputDisposition::Invalidated + ) + { + // Input retirement is later than execution termination and cleanup. + // Check every historical owner, not just the latest terminal result. + for prior in self + .store + .list_backend_job_attempts(self.workspace_id(), job_id)? + { + self.require_backend_job_attempt_effects_ended(&prior)?; + if prior.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) { + return Err(Error::RepositoryConflict( + "Historical Job cleanup not completed".into(), + )); + } + } + if job.input_disposition == crate::backend_job::JobInputDisposition::Available { + self.store.reclaim_backend_job_input( + self.workspace_id(), + job_id, + &job.input_digest, + crate::backend_job::JobInputRetirementReason::Completed, + )?; + } + subjektiv_journal::drain_after_completion(self, &job.request); + } + // Only after durable fences/retirement may live bookkeeping be forgotten. + self.backend_job_recovery + .hosts + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(attempt_id); + self.backend_job_recovery + .submitted + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(attempt_id); + self.backend_job_recovery + .gates + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(attempt_id); + Ok(()) + } + + /// Nonblocking consumer boundary; terminal status/cleanup markers are not proof. + pub(super) fn require_backend_job_attempt_effects_ended( + &self, + attempt: &BackendJobAttemptRecord, + ) -> Result<()> { + if self + .store + .has_backend_job_execution_fence(self.workspace_id(), attempt)? + { + return Ok(()); + } + Err(Error::RepositoryConflict( + "Exact Controller/Feature/background and Host writer/drain evidence missing".into(), + )) + } + + pub(super) async fn fence_backend_job_effects( + &self, + attempt: &BackendJobAttemptRecord, + ) -> Result<()> { + if self + .store + .has_backend_job_execution_fence(self.workspace_id(), attempt)? + { + return Ok(()); + } + let worker = attempt.worker.as_ref().ok_or_else(|| { + Error::RepositoryConflict("Unbound allocation cannot prove execution ended".into()) + })?; + let termination = self + .runtime + .backend_job_termination(worker) + .map_err(RuntimeRegistryError::into_error)? + .ok_or_else(|| { + Error::RepositoryConflict("Actual Runtime scope termination receipt missing".into()) + })?; + if !termination.feature_failures.is_empty() + || termination.scope + != worker_runtime::execution::WorkerExecutionTerminationScope::Controller + || termination.worker_ref.worker_id.to_string() != worker.worker_id + { + return Err(Error::RepositoryConflict( + "Runtime receipt does not prove exact Controller/Feature/background termination" + .into(), + )); + } + let owner = JobHostOwner::try_from(attempt)?; + let effects = self.backend_job_recovery.effects(&owner)?; + let drain = effects.close_and_drain(&owner).await?; + self.store.record_backend_job_execution_fence( + self.workspace_id(), + attempt, + &serde_json::to_string(&termination) + .map_err(|error| Error::Store(error.to_string()))?, + &serde_json::to_string(drain.owner()) + .map_err(|error| Error::Store(error.to_string()))?, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + )?; + Ok(()) + } +} + +/// All authenticated Job callbacks retain the exact live attempt lease through Host/DB +/// work, including accepted synchronous descendants and discarded response cleanup. +/// Identity hints are never admission authority; the outer request proof supplies source. +pub(super) async fn admit_host_request( + State(api): State, + request: Request, + next: Next, +) -> Response { + let Some(source) = request + .extensions() + .get::() + .cloned() + else { + return next.run(request).await; + }; + let Some(worker_id) = source.worker_id else { + return next.run(request).await; + }; + let worker = RuntimeWorkerRef::new(source.runtime_id, worker_id); + let lookup = api.clone(); + let admission = api + .store + .db_executor() + .run(move || -> Result> { + let Some(job) = lookup + .store + .get_active_backend_job_for_worker(lookup.workspace_id(), &worker)? + else { + // Completed/failed Jobs must not become ordinary Workers and bypass + // their closed admission fence. The projection retains Job provenance. + if lookup + .store + .worker_registry_projection(lookup.workspace_id(), &worker)? + .is_some_and(|p| p.job.is_some()) + { + return Err(Error::WorkspacePermissionDenied( + "Backend Job execution is no longer admitted".into(), + )); + } + return Ok(None); + }; + let id = crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt); + let attempt = lookup + .store + .get_backend_job_attempt(lookup.workspace_id(), &job.request.job_id, &id)? + .ok_or_else(|| Error::Store("Job Host attempt missing".into()))?; + let owner = JobHostOwner::try_from(&attempt)?; + Ok(Some( + lookup.backend_job_recovery.effects(&owner)?.admit(&owner)?, + )) + }) + .await; + match admission { + Ok(Some(lease)) => { + crate::db_executor::with_operation_lifetime(Arc::new(lease), next.run(request)).await + } + Ok(None) => next.run(request).await, + Err(error) => server_error_response(error), + } +} diff --git a/crates/workspace-server/src/server/subjektiv_job.rs b/crates/workspace-server/src/server/subjektiv_job.rs index e816010c9..d778962ab 100644 --- a/crates/workspace-server/src/server/subjektiv_job.rs +++ b/crates/workspace-server/src/server/subjektiv_job.rs @@ -241,6 +241,47 @@ pub(super) fn dispatch( }) } +/// Read the existing candidate dispositions and terminal surface outcome for this +/// exact consolidation attempt. Recovery does not rerun candidate decisions or +/// surface generation, and migration-retired Jobs never reach the domain lookup. +pub(super) fn lookup_result( + api: &WorkspaceApi, + job: &crate::backend_job::BackendJobRecord, + attempt: &BackendJobAttemptRecord, +) -> Result> { + if api + .store + .backend_job_was_migrated(api.workspace_id(), &job.request.job_id)? + { + return Ok(None); + } + let Some(grant) = &job.request.grants.subjektiv_consolidation else { + return Ok(None); + }; + if job.workspace_id != api.workspace_id() + || attempt.workspace_id != job.workspace_id + || attempt.job_id != job.request.job_id + || attempt.attempt != job.current_attempt + || attempt.attempt_id != crate::backend_job::attempt_id(&attempt.job_id, attempt.attempt) + || attempt.input_digest != job.input_digest + { + return Err(Error::WorkspacePermissionDenied( + "Consolidation recovery owner mismatch".into(), + )); + } + job.load_executable_request()?; + subject_authorization::authorize_job_input(api, &job.request).map_err(|error| error.error)?; + let store = open_subjektiv_store(api).map_err(|error| error.error)?; + subjektiv::job::lookup_result( + &store, + &grant.subject_id, + &grant.candidate_ids, + &job.request.job_id, + &attempt.attempt_id, + ) + .map_err(Error::from) +} + pub(super) fn validate_result( api: &WorkspaceApi, request: &BackendJobRequest, diff --git a/crates/workspace-server/src/server/subjektiv_journal.rs b/crates/workspace-server/src/server/subjektiv_journal.rs index c5da3b83b..1d310fb51 100644 --- a/crates/workspace-server/src/server/subjektiv_journal.rs +++ b/crates/workspace-server/src/server/subjektiv_journal.rs @@ -22,7 +22,7 @@ fn store(api: &WorkspaceApi) -> Result { } pub(super) fn host( - api: &WorkspaceApi, + _api: &WorkspaceApi, store: &crate::subjektiv::SubjektivStore, grant: &SubjektivJournalGrant, ) -> Result { @@ -33,50 +33,9 @@ pub(super) fn host( .ok_or_else(|| { Error::WorkspacePermissionDenied("Journal Session attribution mismatch".into()) })?; - let mut host = - JournalHostContext::validated_session(store, attribution.clone()).map_err(domain)?; - if grant.binding_id.is_empty() { - // Legacy wire/digest remains unchanged. Recovery is anchored exclusively to the - // immutable original Session's Worker and only while that body is still current. - let worker = RuntimeWorkerRef::new(&attribution.runtime_id, &attribution.worker_id); - let lease = api - .store - .require_current_worker_singleton_owner(api.workspace_id(), &worker)? - .ok_or_else(|| { - Error::WorkspacePermissionDenied( - "legacy Journal original body is no longer current".into(), - ) - })?; - if lease.key - != crate::subjektiv::subject_worker_singleton_key(&attribution.subject_id) - .map_err(|e| Error::InvalidInput(e.to_string()))? - { - return Err(Error::WorkspacePermissionDenied( - "legacy Journal original Subject mismatch".into(), - )); - } - if host.writer_binding_id().is_none() { - bind_host_session(api, attribution.clone())?; - host = JournalHostContext::validated_session(store, attribution.clone()) - .map_err(domain)?; - } - let binding = JournalWriterBinding { - subject_id: attribution.subject_id, - runtime_id: attribution.runtime_id, - worker_id: attribution.worker_id, - binding_id: host - .writer_binding_id() - .ok_or_else(|| { - Error::WorkspacePermissionDenied("legacy original Session writer absent".into()) - })? - .into(), - }; - if !store.journal_writer_active(&binding).map_err(domain)? { - return Err(Error::WorkspacePermissionDenied( - "legacy original writer was released".into(), - )); - } - } else if host.writer_binding_id() != Some(grant.binding_id.as_str()) { + let host = JournalHostContext::validated_session(store, attribution).map_err(domain)?; + // Legacy bindings are retained history, not authority to renew a writer on recovery. + if grant.binding_id.is_empty() || host.writer_binding_id() != Some(grant.binding_id.as_str()) { return Err(Error::WorkspacePermissionDenied( "Journal Job writer generation does not match its frozen grant".into(), )); @@ -90,6 +49,7 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) - let Some(grant) = &reservation.job.request.grants.subjektiv_journal else { return Ok(()); }; + require_current_job(api, &reservation.job)?; let store = store(api)?; let attribution = store .session_attribution(&grant.session_id) @@ -143,37 +103,27 @@ fn legacy_descriptor(request: &BackendJobRequest) -> bool { == serde_json::to_value(&grant.selected_ranges).ok().as_ref() } -/// Retire only the explicitly recognized descriptor wire. Malformed body snapshots are -/// failures, not migration requests and never a reason to reselect from current Sessions. +/// Reject descriptor-only current admission without fetching its body or retiring evidence. +/// Historical migration is owned by the schema transaction, never by ordinary dispatch. pub(super) fn migrate_reserved_legacy( api: &WorkspaceApi, reservation: &BackendJobReservation, ) -> Result { + require_current_job(api, &reservation.job)?; if !legacy_descriptor(&reservation.job.request) { return Ok(false); } - reservation.job.load_executable_request()?; - close(api, &reservation.job.request, &reservation.attempt)?; + // Descriptor input is never materialized from a live Session. Historical migration + // owns old rows; a malformed current admission fails without retiring its evidence. api.store.finish_backend_job_attempt( api.workspace_id(), &reservation.job.request.job_id, &reservation.attempt.attempt_id, BackendJobAttemptState::Failed, "journal_descriptor_invalidated", - "Descriptor-only Journal input retired; saved checkpoints retained", + "Descriptor-only Journal input cannot execute; saved input and checkpoints retained", &Utc::now().to_rfc3339(), )?; - api.store.reclaim_backend_job_input( - api.workspace_id(), - &reservation.job.request.job_id, - &reservation.job.input_digest, - crate::backend_job::JobInputRetirementReason::LegacyInvalidation, - )?; - let queued = api.clone(); - let request = reservation.job.request.clone(); - api.host_executors.host()?.dispatch(move || { - drain_after_completion(&queued, &request); - })?; Ok(true) } @@ -210,6 +160,7 @@ fn live( .store .get_backend_job(api.workspace_id(), &binding.job_id)? .ok_or_else(|| Error::WorkspacePermissionDenied("Journal Job missing".into()))?; + require_current_job(api, &job)?; job.load_executable_request()?; let attempt = api .store @@ -687,6 +638,15 @@ fn dispatch( api: &WorkspaceApi, attribution: crate::subjektiv::SubjectSessionAttribution, journal_id: &str, +) -> Result<()> { + dispatch_saved_or_new(api, attribution, journal_id, true) +} + +fn dispatch_saved_or_new( + api: &WorkspaceApi, + attribution: crate::subjektiv::SubjectSessionAttribution, + journal_id: &str, + capture_new: bool, ) -> Result<()> { let gate = lifecycle_lock(api, &attribution.subject_id)?; let _gate = gate @@ -716,17 +676,17 @@ fn dispatch( .store .get_backend_job(api.workspace_id(), &g.job.job_id)? .ok_or_else(|| Error::Store("active Journal generation has no reserved Job".into()))?; - let attempt = api + if job.state == BackendJobState::Pending { + return Ok(()); + } + if api .store - .get_backend_job_attempt(api.workspace_id(), &g.job.job_id, &g.job.attempt_id)? - .ok_or_else(|| { - Error::Store("active Journal generation has no reserved attempt".into()) - })?; - if job.state == BackendJobState::Pending || attempt.state == BackendJobAttemptState::Unknown + .backend_job_was_migrated(api.workspace_id(), &job.request.job_id)? { return Ok(()); } - // Definitive terminal outcomes retain saved receipts; they do not mark any more input read. + require_job_effects_ended(api, &job)?; + // Terminal outcome, execution shutdown and coverage are independent evidence. store .close_journal_generation(&host, &g.id, &g.job) .map_err(domain)?; @@ -746,14 +706,24 @@ fn dispatch( .subjektiv_journal .as_ref() .is_none_or(|g| g.journal_id != journal_id) - || job.input_disposition != crate::backend_job::JobInputDisposition::Available { continue; } - if job.state == BackendJobState::Pending || job.state == BackendJobState::Unknown { + if api + .store + .backend_job_was_migrated(api.workspace_id(), &job.request.job_id)? + { + // A migrated prefix must not be reselected into an implicitly new Job. return Ok(()); } - if job.state == BackendJobState::Failed && !legacy_descriptor(&job.request) { + if job.state == BackendJobState::Pending { + return Ok(()); + } + if job.state == BackendJobState::Failed { + if !can_resume_partial(job) { + return Ok(()); + } + require_job_effects_ended(api, job)?; job.load_executable_request()?; saved_input( &job.request, @@ -776,6 +746,9 @@ fn dispatch( } after = jobs.last().map(|j| j.request.job_id.clone()); } + if !capture_new { + return Ok(()); + } let input = JournalInput::build(&store, &host, journal_id, |order, offset, budget| { capture_input(api, &attribution, order, offset, budget) .map_err(|e| JournalError::Conflict(e.to_string())) @@ -1251,22 +1224,9 @@ pub(super) fn release_subject( .journal_session_writer_matches(&binding, &attr.session_id) .map_err(domain)? { - // Legacy saved-before-admission Sessions are not optional. Prove canonical - // source identity first, then bind only this original Host-attributed Session. - let _canonical = capture_input(api, &attr, 0, 0, 48 * 1024)?; - let host = - JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?; - if host - .writer_binding_id() - .is_some_and(|id| id != binding.binding_id) - { - return Err(Error::RepositoryConflict( - "legacy Session belongs to another writer".into(), - )); - } - store - .bind_journal_session(&binding, attr.clone()) - .map_err(domain)?; + return Err(Error::RepositoryConflict( + "Session writer binding is missing or different; recovery cannot rebind legacy history".into(), + )); } sessions.push(attr); } @@ -1287,32 +1247,7 @@ pub(super) fn release_subject( 100, )?; for job in &jobs { - if matches!( - job.state, - BackendJobState::Pending | BackendJobState::Unknown - ) { - return Err(Error::RepositoryConflict( - "Journal Job remains pending, running or Unknown".into(), - )); - } - let attempt = api - .store - .get_backend_job_attempt( - api.workspace_id(), - &job.request.job_id, - &crate::backend_job::attempt_id( - &job.request.job_id, - job.current_attempt, - ), - )? - .ok_or_else(|| { - Error::Store("Journal attempt evidence is missing".into()) - })?; - if !attempt.state.terminal() { - return Err(Error::RepositoryConflict( - "Journal attempt result remains unconfirmed".into(), - )); - } + require_job_effects_ended(api, job)?; } if jobs.len() < 100 { break; @@ -1515,7 +1450,10 @@ pub(super) fn drain_after_completion(api: &WorkspaceApi, request: &BackendJobReq .session_attribution(&grant.session_id) .map_err(|e| Error::Store(e.to_string()))? .ok_or_else(|| Error::Store("Journal Session attribution missing".into()))?; - dispatch(api, attr.clone(), &grant.journal_id)?; + // The completed immutable prefix has already passed every execution/cleanup + // fence and input retirement. Admit a new bounded prefix; never replace or + // reselect input for an unfinished Job during recovery. + dispatch_saved_or_new(api, attr.clone(), &grant.journal_id, true)?; release_completed_source(api, &attr) })(); if let Err(error) = result { @@ -1617,23 +1555,14 @@ pub(super) fn recover(api: &WorkspaceApi) -> Result<()> { } continue; } - let source = RuntimeWorkerRef::new(&attr.runtime_id, &attr.worker_id); - if require_enabled(api, &source).is_err() { - // Disablement or removal does not erase previously admitted work, but - // also does not authorize collecting a new disabled Worker's history. - let host = JournalHostContext::validated_session(&store, attr.clone()) - .map_err(domain)?; - if let Ok(Some(id)) = store.journal_for_session(&host) { - if let Err(error) = dispatch(api, attr.clone(), &id) - .and_then(|_| release_completed_source(api, attr)) - { - tracing::warn!(%error,session_id=%attr.session_id,"retained Journal prefix recovery blocked"); - } + // Recovery only consumes saved Job input and domain receipts. A missed + // admission notification is not permission to reread/reselect live history. + if let Ok(Some(id)) = store.journal_for_session(&session_host) { + if let Err(error) = dispatch_saved_or_new(api, attr.clone(), &id, false) + .and_then(|_| release_completed_source(api, attr)) + { + tracing::warn!(%error,session_id=%attr.session_id,"retained Journal prefix recovery blocked"); } - continue; - } - if let Err(error) = admit(api, attr.clone()) { - tracing::warn!(%error,session_id=%attr.session_id,"Journal admission recovery blocked; original input not marked processed"); } } if !attrs.has_more { @@ -1686,7 +1615,7 @@ pub(super) fn recording_request( job_id: format!("journal-recording:{}",Uuid::new_v4()), purpose: "subjektiv_journal".into(), input_ref: format!("subjektiv://{}/journal/{journal_id}",grant.subject_id), input: serde_json::to_value(input).map_err(|e| Error::Store(e.to_string()))?, - instruction: "Record ONLY the immutable body-bearing capture supplied in this input. These exact canonical source bytes, identities and ranges are the whole input for this Job; do not fetch original Sessions or use ReadJournalText Source. First ReadJournalState; initial/current Journal position/open text remain separately paged by ReadJournalText. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, current position and receipt-backed CheckpointJournal. Input delivery is not understanding or reflection. Never infer event completion from turn/task/job/compaction boundaries. Preserve exact operation ID/full input after Unknown. SubmitBackendJobResult only after the complete selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(), + instruction: "Record ONLY the immutable body-bearing capture supplied in this input. These exact canonical source bytes, identities and ranges are the whole input for this Job; do not fetch original Sessions or use ReadJournalText Source. First ReadJournalState; initial/current Journal position/open text remain separately paged by ReadJournalText. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, current position and receipt-backed CheckpointJournal. Input delivery is not understanding or reflection. Never infer event completion from turn/task/job/compaction boundaries. Preserve the exact operation ID/full input after response loss and reconcile its saved receipt. SubmitBackendJobResult only after the complete selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(), profile: "builtin:subjektiv-journal-recording".into(), grants: BackendJobGrants { subjektiv_journal: Some(grant), ..Default::default() }, serialization_key: Some(journal_serialization_key(journal_id)), source_worker: Some(source_worker), notification_target: None, @@ -1889,8 +1818,8 @@ fn finish_erased_release( .release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation) } -/// Ordinary removal owns writer revocation and occupancy release. Before execution -/// only preserve/erase source requirements are checked; an active writer is not a blocker. +/// Ordinary removal may not treat a terminal Job or an archived source as proof of +/// recording or shutdown. Explicit erasure uses its separate tombstone/Purge contract. pub(super) fn prepare_worker_removal( service: &WorkerRemovalService, runtime: &RuntimeRegistry, @@ -1934,9 +1863,7 @@ pub(super) fn prepare_worker_removal( finish_erased_release(&api, &attr)?; return Ok(Some(proof)); } - // Do not require a live Controller, complete Journal, or prior release here. - // Runtime removal is the physical fence; finish_worker_removal revokes the - // exact old body only after its successful, matching retention receipt. + require_body_recording_and_effects_ended(&api, &store, &attr)?; Ok(None) } @@ -2064,6 +1991,10 @@ pub(super) fn validate_removal_retention( // in-memory catalog still contains the Worker. Pin release is best effort // in this historical branch; exact Runtime retention replay still enforces // pins if the source exists, and requires a receipt if it is already gone. + let api = service.journal_dispatch_api.upgrade().ok_or_else(|| { + Error::Store("Workspace API unavailable for Journal effect reconciliation".into()) + })?; + require_session_job_effects_ended(&api, &attr.session_id)?; if let Err(error) = retain_source(&runtime, &service.workspace_id, &attr, source, true) { tracing::debug!(%error, "Historical Purge source unavailable; reconcile exact Runtime receipt"); } @@ -2079,6 +2010,12 @@ pub(super) fn validate_removal_retention( )); } let release = !retention_required; + if release { + let api = service.journal_dispatch_api.upgrade().ok_or_else(|| { + Error::Store("Workspace API unavailable for Journal effect reconciliation".into()) + })?; + require_session_job_effects_ended(&api, &attr.session_id)?; + } if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, release) { // A lost reply may follow the physical transfer. Reconcile only the exact // reserved archive and Session, whether or not an admitted remainder exists. @@ -2173,6 +2110,7 @@ pub(super) fn finish_worker_removal( } return Ok(commit()); } + require_body_recording_and_effects_ended(&api, &store, &attr)?; store.release_removed_journal_body(&host).map_err(domain)?; // Result delivery takes the same Subject gate. Revocation is durable; release // the gate while reconciling jobs, then recheck erasure before publishing. @@ -2281,6 +2219,7 @@ fn release_completed_source( let store = store(api)?; let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?; let target = RuntimeWorkerRef::new(&attr.runtime_id, &attr.worker_id); + require_session_job_effects_ended(api, &attr.session_id)?; if !store.journal_retention_required(&host).map_err(domain)? && api .store @@ -2337,186 +2276,351 @@ pub(super) fn session_erased(api: &WorkspaceApi, request: &BackendJobRequest) -> .map_err(domain) } -/// Unknown remains owned by the original Job until domain completion is proven or -/// canonical stop/Purge cleanup AND the generation fence have both committed. -pub(super) fn reconcile_unknown( +/// A historical migration receipt permanently excludes this Job from normal execution. +/// The saved cause alone is insufficient: callers cannot erase the migration boundary by +/// changing a failure string or by submitting a retained, pre-migration snapshot. +fn require_current_job(api: &WorkspaceApi, job: &BackendJobRecord) -> Result<()> { + if job.workspace_id != api.workspace_id() + || api + .store + .backend_job_was_migrated(api.workspace_id(), &job.request.job_id)? + { + return Err(Error::InvalidInput( + "Migrated Journal Jobs cannot resume or acquire another attempt".into(), + )); + } + Ok(()) +} + +fn can_resume_partial(job: &BackendJobRecord) -> bool { + job.state == BackendJobState::Failed + && job.failure_category.as_deref() == Some("journal_attempt_fenced") + && job.input_disposition == crate::backend_job::JobInputDisposition::Available + && job.current_attempt < job.request.limits.max_attempts + && !legacy_descriptor(&job.request) +} + +/// Outcome is not a shutdown receipt. Inspect every earlier attempt as well: a late +/// Host operation/cleanup from one of them must not affect the next owner. The root +/// checks exact Runtime/Controller, Feature/background, Host-drain and cleanup evidence +/// without waiting under this Subject gate or reconstructing empty live ownership. +pub(super) fn require_job_effects_ended(api: &WorkspaceApi, job: &BackendJobRecord) -> Result<()> { + if job.state == BackendJobState::Pending { + return Err(Error::RepositoryConflict( + "Journal Job is still pending".into(), + )); + } + for number in 1..=job.current_attempt { + let id = crate::backend_job::attempt_id(&job.request.job_id, number); + let attempt = api + .store + .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? + .ok_or_else(|| Error::Store("Journal attempt evidence is missing".into()))?; + if !attempt.state.terminal() { + return Err(Error::RepositoryConflict( + "Journal attempt has no terminal outcome".into(), + )); + } + api.require_backend_job_attempt_effects_ended(&attempt)?; + } + Ok(()) +} + +fn require_session_job_effects_ended(api: &WorkspaceApi, session: &str) -> Result<()> { + let mut after = None; + loop { + let jobs = api.store.list_backend_journal_jobs( + api.workspace_id(), + session, + after.as_deref(), + 100, + )?; + for job in &jobs { + require_job_effects_ended(api, job)?; + } + if jobs.len() < 100 { + return Ok(()); + } + after = jobs.last().map(|job| job.request.job_id.clone()); + } +} + +fn require_body_recording_and_effects_ended( + api: &WorkspaceApi, + store: &crate::subjektiv::SubjektivStore, + attr: &crate::subjektiv::SubjectSessionAttribution, +) -> Result<()> { + for session in store + .subject_session_attributions(&attr.subject_id) + .map_err(|error| Error::Store(error.to_string()))? + .into_iter() + .filter(|session| { + session.runtime_id == attr.runtime_id && session.worker_id == attr.worker_id + }) + { + require_session_job_effects_ended(api, &session.session_id)?; + let host = JournalHostContext::validated_session(store, session).map_err(domain)?; + if store.journal_retention_required(&host).map_err(domain)? { + return Err(Error::RepositoryConflict( + "Journal has unreflected input; ordinary removal retains the original body and evidence".into(), + )); + } + } + Ok(()) +} + +/// A definitive inability to use the frozen input/authority is an outcome, not a +/// claim that execution ended. Cleanup still owns its independent exact-owner proof. +fn fail_recovery( + api: &WorkspaceApi, + job: &BackendJobRecord, + attempt: &BackendJobAttemptRecord, + category: &str, + detail: &str, +) -> Result<()> { + let failed = api.store.finish_backend_job_attempt( + api.workspace_id(), + &job.request.job_id, + &attempt.attempt_id, + BackendJobAttemptState::Failed, + category, + detail, + &Utc::now().to_rfc3339(), + )?; + api.schedule_backend_job_worker_cleanup(&failed); + Ok(()) +} + +/// Reconcile the current accepted attempt using only immutable input and the exact +/// retained generation. A complete checkpoint settles the same Job independently of +/// teardown. Partial continuation requires independent shutdown AND writer fencing. +pub(super) fn reconcile_attempt( api: &WorkspaceApi, job: &BackendJobRecord, attempt: &BackendJobAttemptRecord, ) -> Result<()> { - if attempt.state != BackendJobAttemptState::Unknown { - return Ok(()); - } let grant = job .request .grants .subjektiv_journal .as_ref() .ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?; - let store = store(api)?; - let attr = store - .session_attribution(&grant.session_id) - .map_err(|e| Error::Store(e.to_string()))? - .filter(|a| a.subject_id == grant.subject_id) - .ok_or_else(|| Error::WorkspacePermissionDenied("Journal attribution mismatch".into()))?; - let released = store - .journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id) - .map_err(domain)?; - let host = if released { - // Recovery reads immutable evidence, never renews a legacy writer grant. - let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?; - if !grant.binding_id.is_empty() - && host.writer_binding_id() != Some(grant.binding_id.as_str()) - { - return Err(Error::WorkspacePermissionDenied( - "Journal recovery binding mismatch".into(), - )); - } - host - } else { - host(api, &store, grant)? - }; - let erased = store.journal_session_erased(&host).map_err(domain)?; - if !erased { - job.load_executable_request()?; - let binding = subjektiv::api::JobAttemptBinding { - job_id: job.request.job_id.clone(), - attempt_id: attempt.attempt_id.clone(), - }; - let generation = if released { - store - .retained_journal_generation(&host, &grant.journal_id, &binding) - .map_err(domain)? - } else if legacy_descriptor(&job.request) { - Some( - store - .prepare_journal_generation( - &host, - &grant.journal_id, - binding, - grant.selected_ranges.clone(), - ) - .map_err(domain)?, - ) - } else { - // Preserve active-body recovery: its saved intent can still establish - // the original generation fence. A released body must only read it. - Some( - store - .prepare_journal_input(&host, &saved_input(&job.request, grant)?, binding) - .map_err(domain)?, - ) - }; - if let Some(g) = generation { - if g.selected_ranges != grant.selected_ranges { - return Err(Error::WorkspacePermissionDenied( - "Journal recovery selection mismatch".into(), - )); - } - if !legacy_descriptor(&job.request) { - saved_input(&job.request, grant)? - .validate_generation(&g) - .map_err(domain)?; - } - let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone()) - .map_err(domain)?; - match journal::service::finish(&store, &ctx) { - Ok(completion) => { - let submission = BackendJobResultSubmission { - job_id: job.request.job_id.clone(), - attempt_id: attempt.attempt_id.clone(), - input_digest: attempt.input_digest.clone(), - result: serde_json::to_value(completion) - .map_err(|e| Error::Store(e.to_string()))?, - }; - let current = api.store.reconcile_backend_journal_attempt( - api.workspace_id(), - attempt, - Some(&submission), - &Utc::now().to_rfc3339(), - )?; - let saved = api - .store - .get_backend_job(api.workspace_id(), &job.request.job_id)? - .ok_or_else(|| Error::Store("reconciled Job missing".into()))?; - notification_delivery::deliver_job_result( - api, - &BackendJobResultAcceptance { - job: saved, - attempt: current.clone(), - replayed: false, - }, - ); - if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) - { - if !released { - drain_after_completion(api, &job.request); - } - if job.input_disposition - == crate::backend_job::JobInputDisposition::Available - { - api.store.reclaim_backend_job_input( - api.workspace_id(), - &job.request.job_id, - &job.input_digest, - crate::backend_job::JobInputRetirementReason::Completed, - )?; - } - } else { - api.schedule_backend_job_worker_cleanup(¤t); - } - return Ok(()); - } - Err(JournalError::Conflict(_) | JournalError::PermissionDenied(_)) => {} - Err(error) => return Err(domain(error)), - } - } - } - if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) { + let gate = lifecycle_lock(api, &grant.subject_id)?; + let guard = gate + .lock() + .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?; + // Reread rather than letting a delayed callback settle an old owner or resurrect + // erased/retired input. The Store repeats this full CAS when saving completion. + let current_job = api + .store + .get_backend_job(api.workspace_id(), &job.request.job_id)? + .ok_or_else(|| Error::Store("Journal recovery Job missing".into()))?; + if api + .store + .backend_job_was_migrated(api.workspace_id(), ¤t_job.request.job_id)? + { return Ok(()); } - let confirmed_purge = if let Some(worker) = &attempt.worker { - api.config_store - .recover_worker_removal_execution(api.workspace_id(), worker) - .map_err(|e| Error::Store(e.to_string()))? - .is_some_and(|p| { - p.plan.state == crate::retention::WorkerRemovalPlanState::Succeeded - && p.runtime_request.session_disposition - == worker_runtime::retention::SessionDisposition::Purge - }) - } else { - false - }; - if !confirmed_purge { - if released { - // Source deletion already fenced this generation. An unbound Unknown - // still lacks execution-cleanup proof: preserve Unknown, but it no - // longer owns the Subject or blocks removal of the original body. - return Ok(()); - } + let current_attempt = api + .store + .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &attempt.attempt_id)? + .ok_or_else(|| Error::Store("Journal recovery attempt missing".into()))?; + if ¤t_job != job || ¤t_attempt != attempt { return Err(Error::RepositoryConflict( - "Unknown Journal attempt has no committed stop/Purge fence; not retried".into(), + "Journal recovery owner changed; reread receipts".into(), )); } - if !erased && !released { - close(api, &job.request, attempt)?; + if job.state != BackendJobState::Pending || attempt.state.terminal() { + return Ok(()); } - api.store.reconcile_backend_journal_attempt( - api.workspace_id(), - attempt, - None, + let expected_run = crate::backend_job::runtime_run_id(&attempt.attempt_id); + if job.workspace_id != api.workspace_id() + || attempt.workspace_id != job.workspace_id + || attempt.job_id != job.request.job_id + || attempt.attempt != job.current_attempt + || attempt.attempt_id + != crate::backend_job::attempt_id(&job.request.job_id, attempt.attempt) + || attempt.input_digest != job.input_digest + || attempt + .runtime_run_id + .as_ref() + .is_some_and(|run| run != &expected_run) + || (attempt.state == BackendJobAttemptState::Dispatched + && (attempt.worker.is_none() || attempt.runtime_run_id.is_none())) + { + return Err(Error::WorkspacePermissionDenied( + "Journal recovery identity mismatch".into(), + )); + } + if job.input_disposition != crate::backend_job::JobInputDisposition::Available { + return fail_recovery( + api, + job, + attempt, + "journal_input_unavailable", + "Retired or erased Journal input cannot be executed or reconstructed from a Session", + ); + } + if job.load_executable_request().is_err() { + return fail_recovery( + api, + job, + attempt, + "journal_input_invalid", + "Immutable Journal input or intent fingerprint is invalid; input and checkpoints retained", + ); + } + let input = match saved_input(&job.request, grant) { + Ok(input) => input, + Err(_) => { + return fail_recovery( + api, + job, + attempt, + "journal_input_invalid", + "Body-bearing Journal snapshot does not match its frozen grant; no live input fallback", + ); + } + }; + let store = store(api)?; + let host = match host(api, &store, grant) { + Ok(host) => host, + Err(Error::WorkspacePermissionDenied(_)) => { + return fail_recovery( + api, + job, + attempt, + "journal_writer_unavailable", + "Original Journal writer binding is missing or different; cannot renew or delegate it", + ); + } + Err(error) => return Err(error), + }; + let attr = store + .session_attribution(&grant.session_id) + .map_err(|error| Error::Store(error.to_string()))? + .ok_or_else(|| Error::WorkspacePermissionDenied("Journal attribution missing".into()))?; + if store.journal_session_erased(&host).map_err(domain)? { + return fail_recovery( + api, + job, + attempt, + "journal_session_erased", + "Source Session was explicitly erased", + ); + } + if store + .journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id) + .map_err(domain)? + { + return fail_recovery( + api, + job, + attempt, + "journal_body_released", + "Source Worker was removed; unfinished Journal history retained", + ); + } + if subjektiv_deletion::pending(api, &grant.subject_id)? { + return fail_recovery( + api, + job, + attempt, + "journal_subject_deleting", + "Subject deletion fences Journal execution", + ); + } + let source = RuntimeWorkerRef::new(&attr.runtime_id, &attr.worker_id); + if job.request.source_worker.as_ref() != Some(&source) + || api + .store + .get_worker_registry(api.workspace_id(), &source)? + .is_none() + { + return fail_recovery( + api, + job, + attempt, + "journal_source_worker_missing", + "Original Journal source Worker is missing or mismatched; cannot resurrect or delegate saved input", + ); + } + let binding = subjektiv::api::JobAttemptBinding { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + }; + if let Some(generation) = store + .retained_journal_generation(&host, &grant.journal_id, &binding) + .map_err(domain)? + { + input.validate_generation(&generation).map_err(domain)?; + let context = JournalJobContext::validated_recovery(&store, &host, &generation.id, binding) + .map_err(domain)?; + match journal::service::finish(&store, &context) { + Ok(completion) => { + let submission = BackendJobResultSubmission { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_digest: attempt.input_digest.clone(), + result: serde_json::to_value(completion) + .map_err(|e| Error::Store(e.to_string()))?, + }; + let current = api.store.reconcile_backend_journal_attempt( + api.workspace_id(), + attempt, + Some(&submission), + &Utc::now().to_rfc3339(), + )?; + let saved = api + .store + .get_backend_job(api.workspace_id(), &job.request.job_id)? + .ok_or_else(|| Error::Store("Reconciled Journal Job missing".into()))?; + // Delivery reacquires the Subject gate and rechecks deletion. Only the + // durable reconciliation belongs to this critical section. + drop(guard); + notification_delivery::deliver_job_result( + api, + &BackendJobResultAcceptance { + job: saved, + attempt: current.clone(), + replayed: false, + }, + ); + // Completion must not reclaim input or release pins before all necessary + // attempts/Host effects/cleanup end. The root cleanup rechecks these fences. + api.schedule_backend_job_worker_cleanup(¤t); + return Ok(()); + } + Err(JournalError::Conflict(_)) => {} + Err(error) => return Err(domain(error)), + } + } + if attempt.state == BackendJobAttemptState::Reserved { + // The generic reserved-attempt owner can still dispatch this exact saved input. + return Ok(()); + } + // This is a nonblocking receipt check, not a timeout or a Worker/map-absence test. + // Missing evidence leaves the accepted attempt owned by root receipt reconciliation. + api.require_backend_job_attempt_effects_ended(attempt)?; + for number in 1..attempt.attempt { + let id = crate::backend_job::attempt_id(&job.request.job_id, number); + let prior = api + .store + .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? + .ok_or_else(|| Error::Store("Prior Journal attempt evidence missing".into()))?; + api.require_backend_job_attempt_effects_ended(&prior)?; + } + close(api, &job.request, attempt)?; + api.store.finish_backend_job_attempt( + api.workspace_id(), &job.request.job_id, &attempt.attempt_id, + BackendJobAttemptState::Failed, "journal_attempt_fenced", + "Execution and Host effects ended; generation fenced; retained input/checkpoints permit bounded same-Job continuation", &Utc::now().to_rfc3339(), )?; - if !erased && !released { - // Resume the same saved body input only after canonical execution/domain fences. - // dispatch selects its retained intent before it can capture any newer source. - let attr = store - .session_attribution(&grant.session_id) - .map_err(|e| Error::Store(e.to_string()))? - .ok_or_else(|| Error::Store("Journal attribution missing".into()))?; - dispatch(api, attr, &grant.journal_id)?; - } - Ok(()) + // Do not call admission here: it can select newer live Session bytes. Existing + // bounded retry uses only this Job's input and the operation/checkpoint receipts. + drop(guard); + dispatch_saved_or_new(api, attr, &grant.journal_id, false) } fn reconcile_session_jobs( @@ -2524,11 +2628,10 @@ fn reconcile_session_jobs( attr: &crate::subjektiv::SubjectSessionAttribution, erased: bool, ) -> Result<()> { - let released = !erased - && store(api)? - .journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id) - .map_err(domain)?; - let mut after: Option = None; + let released = store(api)? + .journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id) + .map_err(domain)?; + let mut after = None; loop { let jobs = api.store.list_backend_journal_jobs( api.workspace_id(), @@ -2538,69 +2641,41 @@ fn reconcile_session_jobs( )?; for job in &jobs { let id = crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt); - let mut attempt = api + let attempt = api .store .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? .ok_or_else(|| Error::Store("Journal current attempt missing".into()))?; - if released { - // Writer revocation has already fenced all Journal mutations. End - // pending/Unknown attempts without claiming their input was reflected; - // saved input, operations and original source remain retained. - if !attempt.state.terminal() { - attempt = api.store.finish_backend_job_attempt( - api.workspace_id(), - &job.request.job_id, - &id, - BackendJobAttemptState::Failed, + let migrated = api + .store + .backend_job_was_migrated(api.workspace_id(), &job.request.job_id)?; + if (erased || released) && !attempt.state.terminal() { + let (category, detail) = if erased { + ( + "journal_session_erased", + "Source Session was explicitly erased", + ) + } else { + ( "journal_body_released", "Source Worker was removed; unfinished Journal history retained", - &Utc::now().to_rfc3339(), - )?; - } - api.schedule_backend_job_worker_cleanup(&attempt); - if attempt.state == BackendJobAttemptState::Unknown { - // A saved checkpoint may settle success; otherwise retain Unknown - // until canonical Job-worker cleanup has supplied its Purge fence. - reconcile_unknown(api, job, &attempt)?; - } - continue; - } - if !erased - && legacy_descriptor(&job.request) - && job.input_disposition == crate::backend_job::JobInputDisposition::Available - { - job.load_executable_request()?; - close(api, &job.request, &attempt)?; - if !attempt.state.terminal() { - attempt = api.store.finish_backend_job_attempt( - api.workspace_id(), - &job.request.job_id, - &id, - BackendJobAttemptState::Failed, - "journal_descriptor_invalidated", - "Descriptor-only Journal input retired; saved checkpoints retained", - &Utc::now().to_rfc3339(), - )?; - } - api.store.reclaim_backend_job_input( - api.workspace_id(), - &job.request.job_id, - &job.input_digest, - crate::backend_job::JobInputRetirementReason::LegacyInvalidation, - )?; - } - if erased && !attempt.state.terminal() { - // The domain tombstone precedes this cancellation and also fences any - // crash window before it. No erased Job can prepare or use tools again. - attempt = api.store.finish_backend_job_attempt( + ) + }; + api.store.finish_backend_job_attempt( api.workspace_id(), &job.request.job_id, &id, BackendJobAttemptState::Failed, - "journal_session_erased", - "Source Session was explicitly erased", + category, + detail, &Utc::now().to_rfc3339(), )?; + } else if !erased && !released && !migrated && job.state == BackendJobState::Pending { + if let Err(error) = reconcile_attempt(api, job, &attempt) { + api.retain_backend_job_recovery( + &attempt, "journal_receipt_reconciliation", &error.to_string(), + "Reconcile the same input/generation and exact Runtime/Host receipts; never reselect Session input", + )?; + } } if erased { api.store.reclaim_backend_job_input( @@ -2609,36 +2684,25 @@ fn reconcile_session_jobs( &job.input_digest, crate::backend_job::JobInputRetirementReason::ExplicitErasure, )?; - // Every historical execution can own a delivered copy, not just the current - // attempt or an unacknowledged result. Cleanup is canonical Purge and retryable. - for number in 1..=job.current_attempt { - let historical_id = crate::backend_job::attempt_id(&job.request.job_id, number); - let historical = api - .store - .get_backend_job_attempt( - api.workspace_id(), - &job.request.job_id, - &historical_id, - )? - .ok_or_else(|| Error::Store("historical Journal attempt missing".into()))?; - if historical.state.terminal() { - api.schedule_backend_job_worker_cleanup(&historical); - } + } + // Every historical attempt may own an admitted effect or delivered copy. + // Terminalization and explicit erasure do not assert that these effects ended. + for number in 1..=job.current_attempt { + let id = crate::backend_job::attempt_id(&job.request.job_id, number); + let historical = api + .store + .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? + .ok_or_else(|| Error::Store("Historical Journal attempt missing".into()))?; + if historical.state.terminal() { + api.schedule_backend_job_worker_cleanup(&historical); } } - if attempt.state == BackendJobAttemptState::Unknown { - reconcile_unknown(api, job, &attempt)?; - } - if attempt.state.terminal() { - api.schedule_backend_job_worker_cleanup(&attempt); - } } if jobs.len() < 100 { - break; + return Ok(()); } - after = jobs.last().map(|j| j.request.job_id.clone()); + after = jobs.last().map(|job| job.request.job_id.clone()); } - Ok(()) } fn require_erased_job_copies(api: &WorkspaceApi, session: &str) -> Result<()> { @@ -2668,6 +2732,9 @@ fn require_erased_job_copies(api: &WorkspaceApi, session: &str) -> Result<()> { return Err(Error::RepositoryConflict("Journal delivered-copy Purge remains pending; retry the same erasure after cleanup".into())); } } + // Report the pending canonical cleanup first, but even its Completed + // marker cannot acknowledge erasure without independent execution fences. + require_job_effects_ended(api, job)?; } if jobs.len() < 100 { return Ok(()); @@ -2712,12 +2779,13 @@ impl WorkspaceApi { } } -/// Called from generic Job restart recovery too: accepted source notifications and -/// source Worker availability are not prerequisites for retained result reconciliation. -pub(super) fn recover_unknowns(api: &WorkspaceApi) -> Result<()> { - let mut after: Option = None; +/// Generic Job recovery also reaches Journal receipts when the original source Worker +/// or its admission notification is unavailable. Errors stay attached to the exact attempt +/// and root's bounded recovery owner, rather than creating a replacement Job/state. +pub(super) fn recover_attempts(api: &WorkspaceApi) -> Result<()> { + let mut after = None; loop { - let page = api.store.list_backend_journal_unknown_attempts( + let page = api.store.list_backend_journal_pending_attempts( api.workspace_id(), after.as_deref(), 100, @@ -2726,15 +2794,198 @@ pub(super) fn recover_unknowns(api: &WorkspaceApi) -> Result<()> { let job = api .store .get_backend_job(api.workspace_id(), &attempt.job_id)? - .ok_or_else(|| Error::Store("retained Unknown Journal Job missing".into()))?; - if let Err(error) = reconcile_unknown(api, &job, attempt) { - tracing::warn!(%error,job_id=%attempt.job_id,attempt_id=%attempt.attempt_id,"Journal Unknown is unresolved; no replacement dispatched"); + .ok_or_else(|| Error::Store("Retained Journal recovery Job missing".into()))?; + if api + .store + .backend_job_was_migrated(api.workspace_id(), &job.request.job_id)? + { + continue; + } + if let Err(error) = reconcile_attempt(api, &job, attempt) { + api.retain_backend_job_recovery( + attempt, "journal_receipt_reconciliation", &error.to_string(), + "Reconcile the same input/generation and exact Runtime/Host receipts; never reselect Session input", + )?; } } if page.len() < 100 { - break; + return Ok(()); } after = page.last().map(|attempt| attempt.job_id.clone()); } - Ok(()) +} + +#[cfg(test)] +mod recovery_input_tests { + //! Local wire/policy tests only. Runtime/Host shutdown and persistence-fault recovery + //! belong to the connected Server suite; these tests do not substitute for that evidence. + use super::*; + use session_store::capture::{ + CaptureBody, CaptureDto, CaptureEntryDto, CaptureSource, CaptureUsePolicy, + }; + use session_store::public_material::{ + PublicMaterialDescriptor, PublicMaterialIdentity, PublicMaterialOrigin, + }; + use sha2::{Digest, Sha256}; + + fn snapshot_request() -> BackendJobRequest { + let segment = uuid::Uuid::from_u128(1); + let text = "saved original input"; + let capture = CaptureDto { + version: 1, + session_id: "original-session".into(), + segment_id: segment.to_string(), + view_entry_count: 1, + entries: vec![CaptureEntryDto { + entry_id: "entry-1".into(), + order: 0, + source: CaptureSource { + segment_id: segment.to_string(), + entry_index: 0, + provenance: protocol::SessionEntryProvenance::HumanInput, + history_origin: Some( + session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + ), + derived_from: vec![], + }, + policy: CaptureUsePolicy { + recording_material: true, + memory_evidence: false, + observation_readable: false, + }, + body: CaptureBody::CanonicalPublic { + descriptor: PublicMaterialDescriptor { + identity: PublicMaterialIdentity { + segment_id: segment, + entry_ref: "Eentry-1".into(), + }, + canonical_order: 0, + origin: PublicMaterialOrigin::History { + origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + }, + sha256: Sha256::digest(text.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(), + total_bytes: text.len() as u64, + }, + byte_offset: 0, + text: text.into(), + boundary: None, + }, + }], + }; + let ranges = subjektiv::journal::recording::capture_ranges(&capture).unwrap(); + let input = JournalInput { + version: 1, + subject_id: "original-subject".into(), + journal_id: "original-journal".into(), + initial_state_token: "saved-state".into(), + initial_open_id: None, + previous_fixed_end: None, + selected_ranges: ranges.clone(), + capture, + }; + BackendJobRequest { + job_id: "original-job".into(), + purpose: "subjektiv_journal".into(), + input_ref: "journal://original-journal".into(), + input: serde_json::to_value(input).unwrap(), + instruction: "Record only the immutable input".into(), + profile: "builtin:subjektiv-journal-recording".into(), + grants: BackendJobGrants { + subjektiv_journal: Some(SubjektivJournalGrant { + binding_id: "original-writer".into(), + subject_id: "original-subject".into(), + session_id: "original-session".into(), + journal_id: "original-journal".into(), + selected_ranges: ranges, + }), + ..Default::default() + }, + serialization_key: Some(journal_serialization_key("original-journal")), + source_worker: None, + notification_target: None, + limits: Default::default(), + } + } + + #[test] + fn saved_body_rejects_reselection_descriptor_and_corruption() { + let request = snapshot_request(); + let grant = request.grants.subjektiv_journal.as_ref().unwrap(); + assert!(saved_input(&request, grant).is_ok()); + for field in ["subject", "session", "journal", "selection"] { + let mut changed = grant.clone(); + match field { + "subject" => changed.subject_id = "successor-subject".into(), + "session" => changed.session_id = "successor-session".into(), + "journal" => changed.journal_id = "successor-journal".into(), + "selection" => changed.selected_ranges.clear(), + _ => unreachable!(), + } + assert!(saved_input(&request, &changed).is_err(), "{field}"); + } + let mut descriptor = request.clone(); + descriptor.input = serde_json::json!({ + "journal_id": grant.journal_id, + "selected_ranges": grant.selected_ranges, + }); + assert!(legacy_descriptor(&descriptor)); + assert!(saved_input(&descriptor, grant).is_err()); + let mut corrupt = request.clone(); + let mut input: JournalInput = serde_json::from_value(corrupt.input.clone()).unwrap(); + let CaptureBody::CanonicalPublic { text, .. } = &mut input.capture.entries[0].body else { + unreachable!() + }; + *text = "substituted new input".into(); + corrupt.input = serde_json::to_value(input).unwrap(); + assert!(saved_input(&corrupt, grant).is_err()); + assert!(!legacy_descriptor(&corrupt)); + } + + #[test] + fn partial_retry_requires_fenced_cause_available_body_and_remaining_budget() { + let mut request = snapshot_request(); + request.limits.max_attempts = 2; + let job = BackendJobRecord { + workspace_id: "workspace".into(), + input_digest: request.input_digest().unwrap(), + intent_fingerprint: request.fingerprint().unwrap(), + request, + input_disposition: crate::backend_job::JobInputDisposition::Available, + state: BackendJobState::Failed, + current_attempt: 1, + result: None, + result_digest: None, + failure_category: Some("journal_attempt_fenced".into()), + failure_detail: Some("Exact execution and writer fences confirmed".into()), + created_at: "2026-01-01T00:00:00Z".into(), + updated_at: "2026-01-01T00:00:00Z".into(), + completed_at: Some("2026-01-01T00:00:00Z".into()), + }; + assert!(can_resume_partial(&job)); + for cause in [ + "version_upgrade_execution_aborted", + "journal_body_released", + "journal_session_erased", + "execution_failed", + ] { + let mut failed = job.clone(); + failed.failure_category = Some(cause.into()); + assert!(!can_resume_partial(&failed), "{cause}"); + } + for disposition in [ + crate::backend_job::JobInputDisposition::Reclaimed, + crate::backend_job::JobInputDisposition::Erased, + crate::backend_job::JobInputDisposition::Invalidated, + ] { + let mut retired = job.clone(); + retired.input_disposition = disposition; + assert!(!can_resume_partial(&retired)); + } + let mut exhausted = job; + exhausted.current_attempt = exhausted.request.limits.max_attempts; + assert!(!can_resume_partial(&exhausted)); + } } diff --git a/crates/workspace-server/src/server/tests/backend_job_tests.rs b/crates/workspace-server/src/server/tests/backend_job_tests.rs new file mode 100644 index 000000000..ef623d7d2 --- /dev/null +++ b/crates/workspace-server/src/server/tests/backend_job_tests.rs @@ -0,0 +1,1664 @@ +//! Server Job/Runtime integration; the execution backend is fixture-owned. +//! Persistent Job state and production dispatch/reconciliation/Host admission remain real. +use super::*; + +#[tokio::test] +async fn backend_job_storage_lock_exhausts_budget_then_same_owner_probes_and_saves_result() { + let temp = tempfile::tempdir().unwrap(); + let config = test_server_config(temp.path()); + let store = Arc::new(SqliteWorkspaceStore::open_api_fixture(&config.database_path).unwrap()); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + let execution = Arc::new(DeterministicExecutionBackend::default()); + let api = WorkspaceApi::new_with_database_preparation( + config, + store.clone(), + execution.clone(), + BackendResourceBroker::default(), + HostExecutors::default(), + None, + WorkspaceDatabasePreparation::PreparedFixture, + ) + .await + .unwrap(); + let request = recovery_request("storage-recovered"); + let reservation = api.dispatch_backend_job(&request).unwrap(); + let worker = reservation.attempt.worker.clone().unwrap(); + store + .with_conn(|conn| { + conn.busy_timeout(std::time::Duration::ZERO)?; + Ok(()) + }) + .unwrap(); + let mut blocker = rusqlite::Connection::open(&api.config.database_path).unwrap(); + let lock = blocker + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .unwrap(); + let submission = BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: reservation.attempt.attempt_id.clone(), + input_digest: reservation.attempt.input_digest.clone(), + result: serde_json::json!({"saved":true}), + }; + assert!(api.accept_backend_job_result(&worker, &submission).is_err()); + // Observe the actual owner transition after its finite settlement window; + // busy callbacks on another connection or unrelated writers are not evidence. + tokio::time::timeout( + std::time::Duration::from_secs(30), + api.backend_job_recovery.storage_probe_started.notified(), + ) + .await + .expect("accepted owner never entered storage probe after finite budget"); + assert_eq!( + api.store + .get_backend_job(api.workspace_id(), &request.job_id) + .unwrap() + .unwrap() + .state, + BackendJobState::Pending + ); + lock.rollback().unwrap(); + let completed = wait_for_attempt( + &api, + &request.job_id, + &reservation.attempt.attempt_id, + |attempt| attempt.state == BackendJobAttemptState::Completed, + ) + .await; + assert_eq!(completed.result, Some(submission.result)); + assert_eq!(completed.attempt, 1); + let probes: i64 = api.config_store.with_conn(|conn| { + conn.query_row("SELECT successful_writes FROM backend_job_recovery_probes WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3", + rusqlite::params![api.workspace_id(), request.job_id, reservation.attempt.attempt_id], |row| row.get(0)).map_err(Error::from) + }).unwrap(); + assert!( + probes > 0, + "only an actual exact-owner probe commit may refill the settlement budget" + ); + assert_eq!( + execution.take_inputs().len(), + 1, + "result persistence recovery did not rerun the model" + ); + api.stop_backend_job_recovery().await; +} + +fn recovery_request(id: &str) -> BackendJobRequest { + BackendJobRequest { + job_id: id.into(), + purpose: "receipt_recovery_contract".into(), + input_ref: format!("test://saved/{id}"), + input: serde_json::json!({"body":"frozen"}), + instruction: "Return the bounded structured result".into(), + profile: "builtin:backend-job".into(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits { + timeout_seconds: 30, + ..Default::default() + }, + } +} + +async fn wait_for_attempt( + api: &WorkspaceApi, + job: &str, + attempt: &str, + condition: impl Fn(&BackendJobAttemptRecord) -> bool, +) -> BackendJobAttemptRecord { + tokio::time::timeout(std::time::Duration::from_secs(15), async { + loop { + let current = api + .store + .get_backend_job_attempt(api.workspace_id(), job, attempt) + .unwrap() + .unwrap(); + if condition(¤t) { + return current; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .expect("same accepted attempt did not reach its expected durable state") +} + +#[tokio::test] +async fn backend_job_lost_allocation_response_binds_original_receipt_without_host_admission() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + let request = recovery_request("lost-allocation-response"); + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let reserved = api + .store + .reserve_backend_job(api.workspace_id(), &request, &now) + .unwrap(); + let (claimed, acquired) = api + .store + .claim_backend_job_attempt_dispatch( + api.workspace_id(), + &request.job_id, + &reserved.attempt.attempt_id, + &now, + ) + .unwrap(); + assert!(acquired); + // Fault boundary: the real Host/Runtime creation committed, but its caller + // never saved the binding or acquired the initial producer's Host admission. + let spawn = api + .backend_job_spawn_request(&claimed.job, &claimed.attempt) + .unwrap(); + let created = api + .spawn_workspace_worker(EMBEDDED_WORKER_RUNTIME_ID, spawn) + .unwrap(); + let original = created.worker.unwrap().worker; + assert!(execution.take_inputs().is_empty()); + // Fault injection: live configuration/registry Profile resolution is no + // longer available. Accepted allocation evidence must not depend on it. + api.config_store + .with_conn_mut(|conn| { + conn.execute( + "DELETE FROM workspace_config_trees WHERE workspace_id=?1", + [api.workspace_id()], + )?; + Ok(()) + }) + .unwrap(); + assert!(api.resolve_backend_job_profile(&request).is_err()); + api.own_backend_job_recovery(&claimed.attempt); + let recovered = wait_for_attempt( + &api, + &request.job_id, + &claimed.attempt.attempt_id, + |attempt| attempt.state == BackendJobAttemptState::Dispatched, + ) + .await; + assert_eq!(recovered.worker, Some(original.clone())); + assert_eq!(recovered.attempt, 1); + let owner = crate::backend_job_host_effects::JobHostOwner::try_from(&recovered).unwrap(); + assert!( + api.backend_job_recovery.effects(&owner).is_err(), + "a creation receipt must not synthesize a historical Host drain owner" + ); + let replay = api.dispatch_backend_job(&request).unwrap(); + assert!(replay.replayed); + assert_eq!(replay.attempt.worker, Some(original)); + assert!( + execution.take_inputs().is_empty(), + "restored receipt evidence cannot authorize a new non-idempotent Submit" + ); + api.stop_backend_job_recovery().await; +} + +#[tokio::test] +async fn backend_job_executing_cleanup_restart_requires_fence_and_original_removal() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + let request = recovery_request("executing-cleanup-restart"); + let dispatched = api.dispatch_backend_job(&request).unwrap(); + let worker = dispatched.attempt.worker.clone().unwrap(); + execution.take_inputs(); + seed_worker_session_for_cleanup(temp.path(), &worker); + // Stop only the recovery tasks, not execution. Inject a crash boundary after + // cleanup claim without a live cleanup task. This is not a process-exit test. + api.stop_backend_job_recovery().await; + api.store + .finish_backend_job_attempt( + api.workspace_id(), + &request.job_id, + &dispatched.attempt.attempt_id, + BackendJobAttemptState::Failed, + "fixture_failure", + "retained original failure", + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .unwrap(); + let current = api + .store + .get_backend_job_attempt( + api.workspace_id(), + &request.job_id, + &dispatched.attempt.attempt_id, + ) + .unwrap() + .unwrap(); + let history = api + .store + .list_backend_job_attempts(api.workspace_id(), &request.job_id) + .unwrap(); + let (claimed, acquired) = api + .store + .claim_backend_job_worker_cleanup_owned( + api.workspace_id(), + &history, + ¤t, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .unwrap(); + assert!(acquired); + let mut restarted = api.clone(); + restarted.backend_job_recovery = Arc::default(); + let error = restarted + .run_backend_job_cleanup(claimed.clone()) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("Host writer/drain evidence missing") + ); + assert!( + api.config_store + .recover_worker_removal_execution(api.workspace_id(), &worker) + .unwrap() + .is_none() + ); + + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + stop_intent: None, + reason: Some("fixture exact execution termination".into()), + ticket_assignment: None, + }, + ) + .unwrap(); + api.fence_backend_job_effects(&claimed).await.unwrap(); + let error = restarted + .run_backend_job_cleanup(claimed.clone()) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("no retained canonical removal operation") + ); + assert!( + api.config_store + .recover_worker_removal_execution(api.workspace_id(), &worker) + .unwrap() + .is_none(), + "a restarted Executing marker cannot authorize a new removal" + ); + assert_eq!( + api.store + .get_backend_job_attempt(api.workspace_id(), &request.job_id, &claimed.attempt_id) + .unwrap() + .unwrap() + .worker_cleanup_state, + Some(BackendJobWorkerCleanupState::Executing) + ); + + // Checked restart entry cannot fall through to planning without the pinned + // original operation, even though execution/Host fences now exist. + assert!( + WorkerRemovalService::new(&api) + .execute_target_removal_on_host_checked( + api.runtime.as_ref(), + &worker, + "must not create a replacement", + None, + Some(("missing-operation", "missing-fingerprint")) + ) + .await + .is_err() + ); + assert!( + api.config_store + .recover_worker_removal_execution(api.workspace_id(), &worker) + .unwrap() + .is_none() + ); + + // The original owner performs canonical removal, then loses its cleanup save. + let response = WorkerRemovalService::new(&api) + .execute_target_removal_on_host( + api.runtime.as_ref(), + &worker, + &format!("Terminal Backend Job attempt {}", claimed.attempt_id), + None, + ) + .await + .unwrap(); + assert_eq!(response.status, StatusCode::OK.as_u16()); + let original = api + .config_store + .recover_worker_removal_execution(api.workspace_id(), &worker) + .unwrap() + .unwrap(); + assert_eq!( + original.plan.state, + crate::retention::WorkerRemovalPlanState::Succeeded + ); + restarted.recover_backend_job_worker_cleanups().unwrap(); + let cleaned = wait_for_backend_job_worker_cleanup( + &restarted, + &request.job_id, + &claimed.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert_eq!(cleaned.failure_category.as_deref(), Some("fixture_failure")); + let retained = api + .config_store + .recover_worker_removal_execution(api.workspace_id(), &worker) + .unwrap() + .unwrap(); + assert_eq!(retained.plan.operation_id, original.plan.operation_id); + assert_eq!( + retained.plan.input_fingerprint, + original.plan.input_fingerprint + ); + assert!(execution.take_inputs().is_empty()); + assert!(api.runtime.worker(&worker).is_err()); + restarted.stop_backend_job_recovery().await; +} + +#[tokio::test] +async fn backend_job_lost_input_ack_uses_exact_receipt_without_second_submit() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + execution.lose_input_ack.store(true, Ordering::Release); + let request = recovery_request("lost-ack"); + let reservation = api.dispatch_backend_job(&request).unwrap(); + assert_eq!( + reservation.attempt.state, + BackendJobAttemptState::Dispatched + ); + let worker = reservation.attempt.worker.clone().unwrap(); + assert_eq!(execution.take_inputs().len(), 1); + api.recover_backend_jobs().unwrap(); + let accepted = api + .accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: reservation.attempt.attempt_id.clone(), + input_digest: reservation.attempt.input_digest.clone(), + result: serde_json::json!({"saved":true}), + }, + ) + .unwrap(); + assert_eq!(accepted.attempt.attempt, 1); + assert_eq!(accepted.job.state, BackendJobState::Completed); + assert!(execution.take_inputs().is_empty()); + api.stop_backend_job_recovery().await; +} + +#[tokio::test] +async fn backend_job_definitive_input_rejection_is_not_receipt_ambiguity() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + execution + .reject_input_definitively + .store(true, Ordering::Release); + let request = recovery_request("rejected-input"); + assert!(api.dispatch_backend_job(&request).is_err()); + let attempt = api + .store + .get_backend_job_attempt( + api.workspace_id(), + &request.job_id, + &crate::backend_job::attempt_id(&request.job_id, 1), + ) + .unwrap() + .unwrap(); + assert_eq!(attempt.state, BackendJobAttemptState::Failed); + assert_eq!(attempt.failure_category.as_deref(), Some("input_rejected")); + assert!(execution.receipts.lock().unwrap().is_empty()); + assert_eq!(execution.take_inputs().len(), 1); + api.stop_backend_job_recovery().await; +} + +#[tokio::test] +async fn backend_job_cancelled_callback_keeps_actual_host_db_effect_owned_until_drain() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + let request = recovery_request("host-drain"); + let reservation = api.dispatch_backend_job(&request).unwrap(); + let owner = + crate::backend_job_host_effects::JobHostOwner::try_from(&reservation.attempt).unwrap(); + let effects = api.backend_job_recovery.effects(&owner).unwrap(); + let worker = reservation.attempt.worker.clone().unwrap(); + let (entered, started) = tokio::sync::oneshot::channel(); + let (release, permit) = std::sync::mpsc::channel(); + let gate = Arc::new(Mutex::new(Some((entered, permit)))); + let executor = api.store.db_executor(); + let router = Router::new() + .route( + "/callback", + get(move || { + let executor = executor.clone(); + let gate = gate.lock().unwrap().take().unwrap(); + async move { + let result: Result<()> = executor + .run(move || { + let _ = gate.0.send(()); + gate.1 + .recv_timeout(std::time::Duration::from_secs(10)) + .unwrap(); + Ok(()) + }) + .await; + result + .map(|_| StatusCode::NO_CONTENT) + .map_err(ApiError::from) + } + }), + ) + .layer(middleware::from_fn_with_state( + api.clone(), + backend_job_recovery::admit_host_request, + )); + let mut callback = Request::builder() + .uri("/callback") + .body(Body::empty()) + .unwrap(); + // Authentication is the substituted outer boundary in this test. The real + // Job Host middleware consumes only the verified extension, not hint headers. + callback + .extensions_mut() + .insert(crate::worker_source::VerifiedRuntimeRequestSource { + runtime_id: worker.runtime_id.clone(), + worker_id: Some(worker.worker_id.clone()), + authentication_binding_id: "fixture-verified-binding".into(), + token_id_hash: None, + }); + let waiter = tokio::spawn(async move { router.oneshot(callback).await }); + tokio::time::timeout(std::time::Duration::from_secs(5), started) + .await + .unwrap() + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + api.accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: reservation.attempt.attempt_id.clone(), + input_digest: reservation.attempt.input_digest.clone(), + result: serde_json::json!({"saved":true}), + }, + ) + .unwrap(); + // The fake execution boundary is independently stopped; the accepted Host/DB + // closure is real and still blocked despite its cancelled async waiter. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if !execution.terminations.lock().unwrap().is_empty() && effects.admit(&owner).is_err() + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let pending = api + .store + .get_backend_job_attempt( + api.workspace_id(), + &request.job_id, + &reservation.attempt.attempt_id, + ) + .unwrap() + .unwrap(); + assert_ne!( + pending.worker_cleanup_state, + Some(BackendJobWorkerCleanupState::Completed) + ); + assert!( + api.require_backend_job_attempt_effects_ended(&pending) + .is_err() + ); + assert!(api.runtime.worker(&worker).is_ok()); + release.send(()).unwrap(); + api.stop_backend_job_recovery().await; + let drained = api + .store + .get_backend_job_attempt( + api.workspace_id(), + &request.job_id, + &reservation.attempt.attempt_id, + ) + .unwrap() + .unwrap(); + assert!( + api.require_backend_job_attempt_effects_ended(&drained) + .is_ok() + ); +} + +#[tokio::test] +async fn backend_job_runner_uses_registered_embedded_worker_and_structured_result_authority() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + let request = BackendJobRequest { + job_id: "runner-check-1".to_string(), + purpose: "runner_contract_check".to_string(), + input_ref: "test://runner/input/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits::default(), + }; + + let stale_reservation = api + .store + .reserve_backend_job( + TEST_WORKSPACE_ID, + &request, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .unwrap(); + let dispatched = api + .dispatch_backend_job_reservation(stale_reservation.clone()) + .unwrap(); + let concurrent_replay = api + .dispatch_backend_job_reservation(stale_reservation) + .unwrap(); + assert!(concurrent_replay.replayed); + assert_eq!( + execution.take_inputs().len(), + 1, + "a stale concurrent dispatch replay must not submit the attempt twice" + ); + assert_eq!(dispatched.attempt.state, BackendJobAttemptState::Dispatched); + assert!( + dispatched.attempt.runtime_run_id.is_some(), + "accepted Job input must be fenced to durable Runtime run evidence" + ); + let worker = dispatched.attempt.worker.clone().expect("bound Job Worker"); + assert_eq!(worker.runtime_id, EMBEDDED_WORKER_RUNTIME_ID); + assert!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &worker) + .unwrap() + .is_some(), + "Job dispatch must use the ordinary Workspace Worker registry" + ); + assert_eq!( + api.store + .worker_registry_projection(TEST_WORKSPACE_ID, &worker) + .unwrap() + .unwrap() + .job, + Some(crate::backend_job::BackendJobWorkerBinding { + job_id: request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id.clone(), + purpose: request.purpose.clone(), + }) + ); + let input_error = send_runtime_worker_input( + State(api.clone()), + AxumPath((worker.runtime_id.clone(), worker.worker_id.clone())), + Json(server_api::RuntimeWorkerInputRequest { + submission_request_id: None, + kind: Some("user".to_string()), + content: "attempt to rerun immutable Job input".to_string(), + segments: None, + }), + ) + .await + .unwrap_err(); + assert_eq!( + input_error.into_response().status(), + StatusCode::BAD_REQUEST + ); + assert!( + execution.take_inputs().is_empty(), + "Job-owned Workers must reject interactive reruns" + ); + let replay = api.dispatch_backend_job(&request).unwrap(); + assert!(replay.replayed); + assert_eq!(replay.attempt.worker, Some(worker.clone())); + + let accepted = api + .accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id, + input_digest: request.input_digest().unwrap(), + result: serde_json::json!({"valid": true}), + }, + ) + .unwrap(); + assert_eq!(accepted.job.state, BackendJobState::Completed); +} + +#[tokio::test] +async fn terminal_backend_job_cleanup_removes_only_bound_worker_and_retains_result_delivery() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + let notification_target = spawn_ticket_check_source(&api, "job-cleanup-target"); + let request = BackendJobRequest { + job_id: "runner-cleanup-success-1".to_string(), + purpose: "cleanup_contract_check".to_string(), + input_ref: "test://runner/cleanup/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: Some(notification_target.clone()), + notification_target: Some(notification_target.clone()), + limits: crate::backend_job::BackendJobLimits::default(), + }; + let dispatched = api.dispatch_backend_job(&request).unwrap(); + execution.take_inputs(); + execution.reject_inputs("notification unavailable"); + let worker = dispatched.attempt.worker.clone().unwrap(); + seed_worker_session_for_cleanup(temp.path(), &worker); + let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap(); + let _ = subscriber.take_snapshot(); + let submission = BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id.clone(), + input_digest: request.input_digest().unwrap(), + result: serde_json::json!({"valid": true}), + }; + + let accepted = api.accept_backend_job_result(&worker, &submission).unwrap(); + assert_eq!(accepted.job.state, BackendJobState::Completed); + let cleaned = wait_for_backend_job_worker_cleanup( + &api, + &request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert_eq!(cleaned.result, Some(serde_json::json!({"valid": true}))); + assert!(matches!( + api.runtime.worker(&worker), + Err(RuntimeRegistryError::UnknownWorker { .. }) + )); + assert!( + !temp + .path() + .join(".test-embedded-runtime-store/workers") + .join(&worker.worker_id) + .exists(), + "Runtime Worker and Session storage must be removed" + ); + assert!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &worker) + .unwrap() + .is_none(), + "canonical removal must delete the Backend catalog projection" + ); + tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + let event = subscriber.recv().await.expect("projection event"); + if event.changes.iter().any(|change| { + matches!(change, crate::store::WorkerCatalogChange::Removed(removed) if removed == &worker) + }) { + return; + } + } + }) + .await + .expect("Backend Job cleanup must publish canonical Worker removal"); + assert!( + api.runtime.worker(¬ification_target).is_ok(), + "origin/notification Workers are not cleanup targets" + ); + assert!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, ¬ification_target) + .unwrap() + .is_some() + ); + let replay = api.accept_backend_job_result(&worker, &submission).unwrap(); + assert!( + replay.replayed, + "duplicate result remains idempotent after cleanup" + ); + let delivery_id = + crate::backend_job::delivery_id(&request.job_id, &dispatched.attempt.attempt_id); + let evidence = job_notification_evidence(&api, &delivery_id); + assert_eq!(evidence.0, "deliver"); + assert!(!evidence.2); + assert_eq!(evidence.3.as_deref(), Some("acceptance_unknown")); + assert!( + api.store + .job_notification_request(TEST_WORKSPACE_ID, &delivery_id) + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn backend_job_cleanup_guard_failure_retries_old_attempt_without_touching_new_worker() { + let temp = tempfile::tempdir().unwrap(); + let (api, _execution) = test_api_with_recording_backend(temp.path()).await; + let request = BackendJobRequest { + job_id: "runner-cleanup-retry-1".to_string(), + purpose: "cleanup_retry_contract_check".to_string(), + input_ref: "test://runner/cleanup-retry/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits::default(), + }; + let first = api.dispatch_backend_job(&request).unwrap(); + let first_worker = first.attempt.worker.clone().unwrap(); + seed_worker_session_for_cleanup(temp.path(), &first_worker); + let _ = set_worker_retention( + api.clone(), + first_worker.runtime_id.clone(), + first_worker.worker_id.clone(), + true, + ) + .await + .unwrap(); + api.finish_backend_job_attempt( + &request.job_id, + &first.attempt.attempt_id, + BackendJobAttemptState::Failed, + "model_failure", + "model rejected the request", + ) + .unwrap(); + let blocked = wait_for_backend_job_worker_cleanup( + &api, + &request.job_id, + &first.attempt.attempt_id, + BackendJobWorkerCleanupState::Failed, + ) + .await; + assert_eq!(blocked.failure_category.as_deref(), Some("model_failure")); + assert_eq!( + blocked.worker_cleanup_failure_category.as_deref(), + Some("backend_job_cleanup_removal_rejected") + ); + + assert!( + api.retry_backend_job(&request.job_id).is_err(), + "retry must not overlap an incompletely cleaned attempt" + ); + let _ = set_worker_retention( + api.clone(), + first_worker.runtime_id.clone(), + first_worker.worker_id.clone(), + false, + ) + .await + .unwrap(); + api.recover_backend_jobs().unwrap(); + wait_for_backend_job_worker_cleanup( + &api, + &request.job_id, + &first.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + let second = api.retry_backend_job(&request.job_id).unwrap(); + let second_worker = second.attempt.worker.clone().unwrap(); + seed_worker_session_for_cleanup(temp.path(), &second_worker); + assert_ne!(first_worker, second_worker); + api.recover_backend_jobs().unwrap(); + assert!(api.runtime.worker(&second_worker).is_ok()); + + api.accept_backend_job_result( + &second_worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: second.attempt.attempt_id.clone(), + input_digest: request.input_digest().unwrap(), + result: serde_json::json!({"retried": true}), + }, + ) + .unwrap(); + wait_for_backend_job_worker_cleanup( + &api, + &request.job_id, + &second.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert_eq!( + api.store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &first.attempt.attempt_id, + ) + .unwrap() + .unwrap() + .failure_detail + .as_deref(), + Some("model rejected the request") + ); +} + +#[tokio::test] +async fn backend_job_restart_without_live_host_owner_keeps_cleanup_unconfirmed() { + let temp = tempfile::tempdir().unwrap(); + let config = test_server_config(temp.path()); + let request = BackendJobRequest { + job_id: "runner-cleanup-pre-marker-restart-1".to_string(), + purpose: "cleanup_pre_marker_restart_contract_check".to_string(), + input_ref: "test://runner/cleanup-pre-marker-restart/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits::default(), + }; + let store = + Arc::new(SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap()); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + let execution = Arc::new(DeterministicExecutionBackend::default()); + let embedded = EmbeddedWorkerRuntime::new_fs_store_with_execution_backend( + TEST_WORKSPACE_ID, + config.embedded_runtime_store_root.clone(), + execution.clone(), + ) + .unwrap(); + let binding = test_create_binding(); + let runtime = embedded.subscription_runtime(); + let runtime_worker_ref = worker_runtime::identity::WorkerRef::new(binding.worker_id); + let detail = runtime + .create_worker_scoped( + &worker_runtime::RuntimeWorkspaceScope::new(TEST_WORKSPACE_ID, "embedded-backend"), + worker_runtime::catalog::CreateWorkerRequest { + plugin_execution: None, + worker_id: binding.worker_id, + create_fingerprint: binding.create_fingerprint, + profile: ProfileSelector::Builtin(request.profile.clone()), + display_name: Some("job:cleanup_pre_marker_restart_contract_check".to_string()), + config_bundle: None, + profile_source: worker_runtime::catalog::ProfileSourceArchiveSource::Embedded { + archive: crate::profile_settings::builtin_profile_source_archive( + &ProfileSelector::Builtin(request.profile.clone()), + ) + .unwrap(), + }, + initial_input: None, + workdir_attachment_requests: Vec::new(), + workdir_attachments: Vec::new(), + worker_observation_enabled: false, + worker_observation_grants: Vec::new(), + workspace_api: Some(test_worker_workspace_api(EMBEDDED_WORKER_RUNTIME_ID)), + memory_settings: Some(test_worker_memory_settings()), + subjektiv_attached: false, + backend_job: None, + }, + ) + .unwrap(); + let worker = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, detail.worker_id.to_string()); + runtime + .observe_worker_event( + &runtime_worker_ref, + protocol::Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + last_command_id: 0, + last_finished_submission_request_id: None, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + }, + }, + ) + .unwrap(); + let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let reservation = store + .reserve_backend_job(TEST_WORKSPACE_ID, &request, &now) + .unwrap(); + let (_, claimed) = store + .claim_backend_job_attempt_dispatch( + TEST_WORKSPACE_ID, + &request.job_id, + &reservation.attempt.attempt_id, + &now, + ) + .unwrap(); + assert!(claimed); + let runtime_run_id = crate::backend_job::runtime_run_id(&reservation.attempt.attempt_id); + store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: TEST_WORKSPACE_ID.to_string(), + worker: worker.clone(), + display_name: "job:cleanup_pre_marker_restart_contract_check".to_string(), + profile: Some(request.profile.clone()), + retention_state: "normal".to_string(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: now.clone(), + updated_at: now.clone(), + }) + .unwrap(); + store + .bind_backend_job_attempt_worker( + TEST_WORKSPACE_ID, + &request.job_id, + &reservation.attempt.attempt_id, + &worker, + Some(&runtime_run_id), + &now, + ) + .unwrap(); + seed_worker_session_for_cleanup(temp.path(), &worker); + let before_restart = runtime.worker_detail(&runtime_worker_ref).unwrap(); + assert!(!before_restart.execution_reconstructed); + assert!(before_restart.worker_state.is_some_and(|snapshot| { + snapshot.last_command_id == 0 + && snapshot.last_finished_submission_request_id.is_none() + && snapshot.state + == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )) + })); + store + .accept_backend_job_result( + TEST_WORKSPACE_ID, + &worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: reservation.attempt.attempt_id.clone(), + input_digest: request.input_digest().unwrap(), + result: serde_json::json!({"persisted": true}), + }, + &now, + ) + .unwrap(); + assert_eq!( + store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &reservation.attempt.attempt_id, + ) + .unwrap() + .unwrap() + .worker_cleanup_state, + Some(BackendJobWorkerCleanupState::Pending) + ); + let attempt_id = reservation.attempt.attempt_id; + drop(runtime); + drop(embedded); + drop(store); + drop(execution); + + let restarted_store = + SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let restarted_execution = Arc::new(DeterministicExecutionBackend::default()); + restarted_execution.accept_restores(); + let restarted = WorkspaceApi::new_with_execution_backend( + config, + Arc::new(restarted_store), + restarted_execution, + ) + .await + .unwrap(); + let cleaned = wait_for_backend_job_worker_cleanup( + &restarted, + &request.job_id, + &attempt_id, + BackendJobWorkerCleanupState::Failed, + ) + .await; + assert_eq!(cleaned.result, Some(serde_json::json!({"persisted": true}))); + assert!( + restarted + .require_backend_job_attempt_effects_ended(&cleaned) + .is_err() + ); + assert!( + restarted.runtime.worker(&worker).is_ok(), + "missing Host drain is not removal authority" + ); + restarted.stop_backend_job_recovery().await; +} + +#[tokio::test] +async fn backend_job_fast_result_is_accepted_after_prebound_tracked_run() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + let request = BackendJobRequest { + job_id: "r".repeat(256), + purpose: "fast_result_contract_check".to_string(), + input_ref: "test://runner/fast-result/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits::default(), + }; + let acceptance_observed = Arc::new(std::sync::Mutex::new(None)); + let hook_api = api.clone(); + let hook_request = request.clone(); + let hook_observed = acceptance_observed.clone(); + let hook_root = temp.path().to_path_buf(); + execution.before_input_returns(move |worker| { + let source = + RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, worker.worker_id.to_string()); + seed_worker_session_for_cleanup(&hook_root, &source); + let accepted = hook_api + .accept_backend_job_result( + &source, + &BackendJobResultSubmission { + job_id: hook_request.job_id.clone(), + attempt_id: crate::backend_job::attempt_id(&hook_request.job_id, 1), + input_digest: hook_request.input_digest().unwrap(), + result: serde_json::json!({"valid": true}), + }, + ) + .is_ok(); + *hook_observed.lock().unwrap() = Some(accepted); + }); + + let dispatched = api.dispatch_backend_job(&request).unwrap(); + assert_eq!(*acceptance_observed.lock().unwrap(), Some(true)); + assert_eq!(dispatched.job.state, BackendJobState::Completed); + let expected_run_id = + crate::backend_job::runtime_run_id(&crate::backend_job::attempt_id(&request.job_id, 1)); + assert_eq!( + dispatched.attempt.runtime_run_id.as_deref(), + Some(expected_run_id.as_str()) + ); + let worker = dispatched.attempt.worker.clone().unwrap(); + wait_for_backend_job_worker_cleanup( + &api, + &request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert!(matches!( + api.runtime.worker(&worker), + Err(RuntimeRegistryError::UnknownWorker { .. }) + )); +} + +#[tokio::test] +async fn backend_job_result_http_response_precedes_cleanup_until_tracked_run_finishes() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + let request = BackendJobRequest { + job_id: "runner-result-response-fence-1".to_string(), + purpose: "result_response_fence_check".to_string(), + input_ref: "test://runner/result-response-fence/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits::default(), + }; + let dispatched = api.dispatch_backend_job(&request).unwrap(); + let worker = dispatched.attempt.worker.clone().unwrap(); + let runtime_run_id = dispatched.attempt.runtime_run_id.clone().unwrap(); + seed_worker_session_for_cleanup(temp.path(), &worker); + execution.publish_worker_state( + &worker, + protocol::WorkerStateSnapshot { + last_command_id: 0, + last_finished_submission_request_id: None, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + }, + ); + assert!( + api.runtime + .worker(&worker) + .unwrap() + .worker_state + .is_some_and(|snapshot| snapshot.state + == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running + ))) + ); + + let Json(acceptance) = scoped_submit_backend_job_result( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + ticket_check_headers(&worker), + Json(SubmitBackendJobResultRequest { + result: serde_json::json!({"valid": true}), + }), + ) + .await + .unwrap(); + assert_eq!(acceptance.job.state, BackendJobState::Completed); + let still_running = api.runtime.worker(&worker).unwrap(); + assert!(still_running.worker_state.is_some_and(|snapshot| { + snapshot.state + == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )) + })); + // Cleanup admission precedes its durable claim. The accepted job may + // still be queued, but neither queued nor executing ownership may stop + // this Run before the exact tracked finish marker below is published. + assert!(matches!( + api.store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &dispatched.attempt.attempt_id, + ) + .unwrap() + .unwrap() + .worker_cleanup_state, + Some(BackendJobWorkerCleanupState::Pending | BackendJobWorkerCleanupState::Executing) + )); + + execution.publish_worker_state( + &worker, + protocol::WorkerStateSnapshot { + last_command_id: 0, + last_finished_submission_request_id: Some(runtime_run_id), + state: protocol::WorkerState::Idle, + }, + ); + wait_for_backend_job_worker_cleanup( + &api, + &request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert!(matches!( + api.runtime.worker(&worker), + Err(RuntimeRegistryError::UnknownWorker { .. }) + )); +} + +#[tokio::test] +async fn backend_job_delivery_recovery_replays_same_tracked_notification_id() { + let temp = tempfile::tempdir().unwrap(); + let config = test_server_config(temp.path()); + let store = SqliteWorkspaceStore::open_for_test(config.database_path.clone()).unwrap(); + let execution = Arc::new(DeterministicExecutionBackend::default()); + let api = WorkspaceApi::new_with_execution_backend( + config.clone(), + Arc::new(store), + execution.clone(), + ) + .await + .unwrap(); + seed_test_delivery_binding(&api); + let target = api + .spawn_workspace_worker( + EMBEDDED_WORKER_RUNTIME_ID, + WorkerSpawnRequest { + requested_worker_name: Some("job-notification-target".to_string()), + intent: WorkerSpawnIntent::WorkspaceCompanion, + singleton_key: None, + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 0, + }, + profile: ProfileSelector::Builtin("builtin:companion".to_string()), + ticket_assignment: None, + initial_submit: Vec::new(), + workdir_attachment_requests: Vec::new(), + resolved_workdir_attachment_requests: Vec::new(), + resolved_workdir_attachments: Vec::new(), + resolved_config_bundle: Some(runtime_test_bundle()), + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_control_operation: None, + resolved_execution_grant: None, + }, + ) + .unwrap() + .worker + .unwrap() + .worker; + let request = BackendJobRequest { + job_id: "n".repeat(256), + purpose: "delivery_recovery_contract_check".to_string(), + input_ref: "test://runner/delivery-recovery/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: Some(target.clone()), + limits: crate::backend_job::BackendJobLimits { + timeout_seconds: 1, + ..Default::default() + }, + }; + let dispatched = api.dispatch_backend_job(&request).unwrap(); + let worker = dispatched.attempt.worker.clone().unwrap(); + let acceptance = api + .store + .accept_backend_job_result( + TEST_WORKSPACE_ID, + &worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id.clone(), + input_digest: request.input_digest().unwrap(), + result: serde_json::json!({"valid": true}), + }, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .unwrap(); + let delivery_id = + crate::backend_job::delivery_id(&request.job_id, &acceptance.attempt.attempt_id); + let (_, claimed) = api + .store + .reserve_backend_job_delivery( + TEST_WORKSPACE_ID, + &delivery_id, + &request.job_id, + &target, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .unwrap(); + assert!(claimed); + let recoverable = api + .store + .recover_backend_job_deliveries( + TEST_WORKSPACE_ID, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .unwrap(); + assert_eq!(recoverable.len(), 1); + execution.take_input_request_ids(); + api.deliver_backend_job_result(&acceptance, &target); + let request_ids = execution.take_input_request_ids(); + assert!( + request_ids + .iter() + .any(|request_id| request_id.as_deref() == Some(delivery_id.as_str())), + "recovery must replay the durable delivery id as the tracked notification id: {request_ids:?}" + ); + assert_eq!(job_notification_evidence(&api, &delivery_id).2, true); + assert!( + api.store + .list_pending_job_notifications(TEST_WORKSPACE_ID, 100) + .unwrap() + .is_empty() + ); + notification_delivery::recover_job_notifications(&api); + assert!(execution.take_input_request_ids().is_empty()); +} + +#[tokio::test] +async fn backend_job_missing_runtime_is_a_definitive_dispatch_rejection() { + let temp = tempfile::tempdir().unwrap(); + let api = test_api(temp.path()).await; + api.runtime + .unregister_if_idle(EMBEDDED_WORKER_RUNTIME_ID, 10) + .unwrap(); + let request = BackendJobRequest { + job_id: "runner-spawn-error-1".to_string(), + purpose: "spawn_error_contract_check".to_string(), + input_ref: "test://runner/spawn-error/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits::default(), + }; + + assert!(api.dispatch_backend_job(&request).is_err()); + let attempt = api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &crate::backend_job::attempt_id(&request.job_id, 1), + ) + .unwrap() + .unwrap(); + assert_eq!(attempt.state, BackendJobAttemptState::Failed); + assert_eq!( + attempt.failure_category.as_deref(), + Some("dispatch_rejected") + ); +} + +#[tokio::test] +async fn backend_job_input_response_loss_is_owned_until_deadline_without_resubmission() { + let temp = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(temp.path()).await; + let cleanup_root = temp.path().to_path_buf(); + execution.before_input_returns(move |worker| { + let worker = + RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, worker.worker_id.to_string()); + seed_worker_session_for_cleanup(&cleanup_root, &worker); + }); + execution.reject_inputs("model transport unavailable"); + let request = BackendJobRequest { + job_id: "runner-input-failure-1".to_string(), + purpose: "failure_contract_check".to_string(), + input_ref: "test://runner/failure/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits { + timeout_seconds: 1, + ..Default::default() + }, + }; + + assert!(api.dispatch_backend_job(&request).is_err()); + let failed = api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &crate::backend_job::attempt_id(&request.job_id, 1), + ) + .unwrap() + .unwrap(); + assert_eq!(failed.state, BackendJobAttemptState::Dispatched); + assert_eq!( + failed.failure_category.as_deref(), + Some("submission_receipt_unconfirmed") + ); + let worker = failed.worker.clone().unwrap(); + wait_for_backend_job_worker_cleanup( + &api, + &request.job_id, + &failed.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert!(matches!( + api.runtime.worker(&worker), + Err(RuntimeRegistryError::UnknownWorker { .. }) + )); + assert_eq!( + execution.take_inputs().len(), + 1, + "ambiguous input was never blindly resubmitted" + ); + assert!( + api.store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &crate::backend_job::attempt_id(&request.job_id, 2), + ) + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn backend_job_recovery_drains_more_than_one_reserved_page() { + let temp = tempfile::tempdir().unwrap(); + let api = test_api(temp.path()).await; + api.runtime + .unregister_if_idle(EMBEDDED_WORKER_RUNTIME_ID, 10) + .unwrap(); + let mut jobs = Vec::new(); + for index in 0..(usize::from(ABSOLUTE_MAX_CONCURRENT_JOBS) + 6) { + let request = BackendJobRequest { + job_id: format!("restart-page-{index:03}"), + purpose: "restart_page_contract_check".to_string(), + input_ref: format!("fixture://restart-page/{index}"), + input: serde_json::json!({"index": index}), + instruction: "Return a structured fixture result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits { + max_concurrent_jobs: ABSOLUTE_MAX_CONCURRENT_JOBS, + ..Default::default() + }, + }; + api.store + .reserve_backend_job( + TEST_WORKSPACE_ID, + &request, + &format!("2026-09-01T00:{:02}:{:02}Z", index / 60, index % 60), + ) + .unwrap(); + jobs.push(request.job_id); + } + + api.recover_backend_jobs().unwrap(); + assert!( + api.store + .list_reserved_backend_job_attempts(TEST_WORKSPACE_ID, jobs.len()) + .unwrap() + .is_empty(), + "restart recovery must continue beyond its first bounded page when dispatches terminalize" + ); + assert!(jobs.iter().all(|job_id| { + api.store + .get_backend_job(TEST_WORKSPACE_ID, job_id) + .unwrap() + .is_some_and(|job| job.state == BackendJobState::Failed) + })); +} + +#[tokio::test] +async fn backend_job_recovery_dispatches_reserved_attempt_without_duplicate_worker() { + let temp = tempfile::tempdir().unwrap(); + let api = test_api(temp.path()).await; + let request = BackendJobRequest { + job_id: "runner-restart-1".to_string(), + purpose: "restart_contract_check".to_string(), + input_ref: "test://runner/restart/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits::default(), + }; + let reserved = api + .store + .reserve_backend_job( + TEST_WORKSPACE_ID, + &request, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .unwrap(); + assert_eq!(reserved.attempt.state, BackendJobAttemptState::Reserved); + api.recover_backend_jobs().unwrap(); + + let recovered = api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &reserved.attempt.attempt_id, + ) + .unwrap() + .unwrap(); + assert_eq!(recovered.state, BackendJobAttemptState::Dispatched); + let recovered_worker = recovered.worker.clone().expect("recovered Job Worker"); + let replay = api.dispatch_backend_job(&request).unwrap(); + assert!(replay.replayed); + assert_eq!(replay.attempt.worker, Some(recovered_worker)); +} + +#[tokio::test] +async fn backend_job_missing_worker_reaches_failure_without_fabricated_cleanup() { + let temp = tempfile::tempdir().unwrap(); + let api = test_api(temp.path()).await; + let request = BackendJobRequest { + job_id: "runner-unknown-1".to_string(), + purpose: "unknown_contract_check".to_string(), + input_ref: "test://runner/unknown/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Return a structured check result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits::default(), + }; + let reserved = api + .store + .reserve_backend_job( + TEST_WORKSPACE_ID, + &request, + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .unwrap(); + let missing_worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + "0199a3ae-ecdf-7eb2-883e-a5b7b5f04a0d", + ); + api.store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: TEST_WORKSPACE_ID.to_string(), + worker: missing_worker.clone(), + display_name: "Missing Job Worker".to_string(), + profile: Some("builtin:backend-job".to_string()), + retention_state: "normal".to_string(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "2026-09-01T00:00:00Z".to_string(), + updated_at: "2026-09-01T00:00:00Z".to_string(), + }) + .unwrap(); + let (_, claimed) = api + .store + .claim_backend_job_attempt_dispatch( + TEST_WORKSPACE_ID, + &request.job_id, + &reserved.attempt.attempt_id, + "2026-09-01T00:00:00Z", + ) + .unwrap(); + assert!(claimed); + api.store + .bind_backend_job_attempt_worker( + TEST_WORKSPACE_ID, + &request.job_id, + &reserved.attempt.attempt_id, + &missing_worker, + Some("missing-runtime-run"), + &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), + ) + .unwrap(); + + api.recover_backend_jobs().unwrap(); + let attempt = wait_for_attempt( + &api, + &request.job_id, + &reserved.attempt.attempt_id, + |attempt| attempt.state == BackendJobAttemptState::Failed, + ) + .await; + assert_eq!(attempt.failure_category.as_deref(), Some("timeout")); + assert_ne!( + attempt.worker_cleanup_state, + Some(BackendJobWorkerCleanupState::Completed) + ); + assert!( + api.require_backend_job_attempt_effects_ended(&attempt) + .is_err() + ); + assert!(api.retry_backend_job(&request.job_id).is_err()); + api.stop_backend_job_recovery().await; +} + +#[tokio::test] +async fn backend_job_timeout_fences_late_structured_result() { + let temp = tempfile::tempdir().unwrap(); + let api = test_api(temp.path()).await; + let request = BackendJobRequest { + job_id: "runner-timeout-1".to_string(), + purpose: "timeout_contract_check".to_string(), + input_ref: "test://runner/timeout/1".to_string(), + input: serde_json::json!({"value": "immutable"}), + instruction: "Wait for a structured result.".to_string(), + profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits { + timeout_seconds: 1, + max_attempts: 1, + ..Default::default() + }, + }; + let dispatched = api.dispatch_backend_job(&request).unwrap(); + let worker = dispatched.attempt.worker.clone().unwrap(); + seed_worker_session_for_cleanup(temp.path(), &worker); + tokio::time::sleep(std::time::Duration::from_millis(1_200)).await; + let attempt = api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &dispatched.attempt.attempt_id, + ) + .unwrap() + .unwrap(); + assert_eq!(attempt.state, BackendJobAttemptState::Failed); + assert_eq!(attempt.failure_category.as_deref(), Some("timeout")); + wait_for_backend_job_worker_cleanup( + &api, + &request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert!(matches!( + api.runtime.worker(&worker), + Err(RuntimeRegistryError::UnknownWorker { .. }) + )); + assert!( + api.accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id, + input_digest: request.input_digest().unwrap(), + result: serde_json::json!({"late": true}), + }, + ) + .is_err() + ); +} diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs index 56327c331..f1c91ccf1 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs @@ -600,6 +600,7 @@ struct RuntimeFixture { dir: tempfile::TempDir, api: WorkspaceApi, execution: Arc, + job_store: Arc, store: crate::subjektiv::SubjektivStore, host: JournalHostContext, source: RuntimeWorkerRef, @@ -734,7 +735,24 @@ impl RuntimeFixture { frozen_origin: bool, ) -> Self { let dir = tempfile::tempdir().unwrap(); - let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let config = test_server_config(dir.path()); + let job_store = + Arc::new(SqliteWorkspaceStore::open_api_fixture(&config.database_path).unwrap()); + seed_test_registered_workspace(job_store.as_ref(), &config) + .await + .unwrap(); + let execution = Arc::new(DeterministicExecutionBackend::default()); + let api = WorkspaceApi::new_with_database_preparation( + config, + job_store.clone(), + execution.clone(), + BackendResourceBroker::default(), + HostExecutors::default(), + None, + WorkspaceDatabasePreparation::PreparedFixture, + ) + .await + .unwrap(); let store = open_subjektiv_store(&api).unwrap(); let subject = store .create_subject(crate::subjektiv::SubjectRole::new("source-reader-test").unwrap()) @@ -842,6 +860,7 @@ impl RuntimeFixture { dir, api, execution, + job_store, store, host, source, @@ -1111,49 +1130,247 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang } } -#[tokio::test] -async fn unknown_saved_completion_reconciles_same_binding_without_redispatch_after_lost_result() { - let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_input(); - let worker = dispatched.attempt.worker.as_ref().unwrap(); - f.checkpoint(worker); - let saved = subjektiv_journal::finish(&f.api, worker).unwrap(); // response/result delivery is lost - f.execution.take_inputs(); - let unknown = f - .api +/// A real SQLite writer lock blocks Job completion while the separate Journal store +/// can still save its checkpoint. Removing it does not drive recovery or resubmit input. +struct JournalCompletionSaveFault(rusqlite::Connection); + +impl JournalCompletionSaveFault { + fn install(f: &RuntimeFixture) -> Self { + f.job_store + .with_conn(|connection| { + connection.busy_timeout(std::time::Duration::ZERO)?; + Ok(()) + }) + .unwrap(); + let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap(); + connection.execute_batch("BEGIN IMMEDIATE").unwrap(); + Self(connection) + } +} + +impl Drop for JournalCompletionSaveFault { + fn drop(&mut self) { + self.0.execute_batch("ROLLBACK").unwrap(); + } +} + +fn current_journal_job(f: &RuntimeFixture) -> crate::backend_job::BackendJobRecord { + f.api .store - .finish_backend_job_attempt( - TEST_WORKSPACE_ID, - &f.request.job_id, - &dispatched.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "result_lost", - "no result acknowledgement", - &Utc::now().to_rfc3339(), - ) - .unwrap(); - subjektiv_journal::reconcile_unknown(&f.api, &dispatched.job, &unknown).unwrap(); - let cleaned = wait_for_backend_job_worker_cleanup( - &f.api, - &f.request.job_id, - &dispatched.attempt.attempt_id, - BackendJobWorkerCleanupState::Completed, - ) + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() + .unwrap() +} + +fn current_journal_attempt(f: &RuntimeFixture, attempt_id: &str) -> BackendJobAttemptRecord { + f.api + .store + .get_backend_job_attempt(TEST_WORKSPACE_ID, &f.request.job_id, attempt_id) + .unwrap() + .unwrap() +} + +async fn observe_journal_completion(f: &RuntimeFixture) -> crate::backend_job::BackendJobRecord { + let observed = tokio::time::timeout(std::time::Duration::from_secs(40), async { + loop { + let job = current_journal_job(f); + if job.state != BackendJobState::Pending { + return job; + } + // Observe only; do not drive result submission or a recovery tick. + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) .await; - assert_eq!(cleaned.state, BackendJobAttemptState::Completed); - assert_eq!(cleaned.result, Some(serde_json::to_value(saved).unwrap())); - assert_eq!(cleaned.worker, dispatched.attempt.worker); - assert!(f.execution.take_inputs().is_empty()); - subjektiv_journal::reconcile_unknown(&f.api, &dispatched.job, &cleaned).unwrap(); - f.assert_purged(worker); + assert!( + observed.is_ok(), + "autonomous recovery did not converge: job={:?}, attempt={:?}", + current_journal_job(f), + current_journal_attempt(f, &f.reserved.attempt.attempt_id) + ); + observed.unwrap() } #[tokio::test] -async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_cleanup_fails() { +async fn completion_save_fault_recovers_same_job_from_saved_checkpoint_without_user_retry() { let f = RuntimeFixture::new().await; let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); - let g = f + // Block cleanup independently so completion cannot hide teardown or retire + // the immutable body before its recovery is observed. + f.api + .store + .update_worker_retention( + TEST_WORKSPACE_ID, + worker, + "pinned", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + let fault = JournalCompletionSaveFault::install(&f); + f.checkpoint(worker); + let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); + let binding = subjektiv::api::JobAttemptBinding { + job_id: f.request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id.clone(), + }; + let generation = f + .store + .retained_journal_generation(&f.host, &f.id, &binding) + .unwrap() + .unwrap(); + let checkpoint = f + .store + .query_journal_operation(&f.host, &f.id, "checkpoint") + .unwrap() + .unwrap(); + let accepted = current_journal_attempt(&f, &dispatched.attempt.attempt_id); + assert!(accepted.runtime_run_id.is_some()); + f.execution.take_inputs(); + + let submission = BackendJobResultSubmission { + job_id: f.request.job_id.clone(), + attempt_id: accepted.attempt_id.clone(), + input_digest: accepted.input_digest.clone(), + result: serde_json::to_value(&completion).unwrap(), + }; + let error = f + .api + .accept_backend_job_result(worker, &submission) + .unwrap_err(); + assert!( + matches!(error, Error::Sqlite(rusqlite::Error::SqliteFailure(code, _)) + if code.code == rusqlite::ErrorCode::DatabaseBusy), + "{error}" + ); + let pending = current_journal_job(&f); + let pending_attempt = current_journal_attempt(&f, &accepted.attempt_id); + assert_eq!(pending.state, BackendJobState::Pending); + assert_eq!(pending_attempt.state, BackendJobAttemptState::Dispatched); + assert!(pending.result.is_none()); + assert!( + pending_attempt.result.is_none(), + "attempt and Job commit must roll back together" + ); + assert_eq!( + pending.load_executable_request().unwrap().input, + f.request.input + ); + assert_eq!(pending.input_digest, accepted.input_digest); + assert_eq!(pending_attempt.worker, accepted.worker); + assert_eq!(pending_attempt.runtime_run_id, accepted.runtime_run_id); + assert_eq!( + f.store + .retained_journal_generation(&f.host, &f.id, &generation.job) + .unwrap(), + Some(generation.clone()) + ); + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, "checkpoint") + .unwrap(), + Some(checkpoint.clone()) + ); + assert!( + f.api + .require_backend_job_attempt_effects_ended(&pending_attempt) + .is_err() + ); + assert!(f.execution.take_inputs().is_empty()); + + tokio::time::timeout( + std::time::Duration::from_secs(30), + f.api.backend_job_recovery.storage_probe_started.notified(), + ) + .await + .expect("Journal owner did not exhaust finite settlement before exact-attempt probe"); + drop(fault); + // Only storage recovers. No second accept/retry/admission/recover call: + // the production owner must resume this exact Job autonomously. + let completed = observe_journal_completion(&f).await; + assert_eq!(completed.state, BackendJobState::Completed); + assert_eq!(completed.request.job_id, f.request.job_id); + assert_eq!(completed.current_attempt, accepted.attempt); + assert_eq!(completed.input_digest, accepted.input_digest); + assert_eq!( + completed.load_executable_request().unwrap().input, + f.request.input + ); + assert_eq!(completed.result, Some(submission.result)); + let ended = current_journal_attempt(&f, &accepted.attempt_id); + assert_eq!(ended.state, BackendJobAttemptState::Completed); + assert_eq!(ended.worker, accepted.worker); + assert_eq!(ended.runtime_run_id, accepted.runtime_run_id); + assert_eq!(ended.input_digest, accepted.input_digest); + assert_eq!(ended.result, completed.result); + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, "checkpoint") + .unwrap(), + Some(checkpoint) + ); + assert!( + f.execution.take_inputs().is_empty(), + "checkpoint recovery must not redispatch" + ); + assert!( + f.api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &f.request.job_id, + &crate::backend_job::attempt_id(&f.request.job_id, accepted.attempt + 1) + ) + .unwrap() + .is_none() + ); + let blocked = wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &accepted.attempt_id, + BackendJobWorkerCleanupState::Failed, + ) + .await; + assert_eq!(blocked.state, BackendJobAttemptState::Completed); + // Stop has independently joined the fixture-owned execution/background and + // drained Host writers. Pinned retention rejects Purge, not that termination. + f.api + .require_backend_job_attempt_effects_ended(&blocked) + .unwrap(); + assert_eq!( + blocked.worker_cleanup_state, + Some(BackendJobWorkerCleanupState::Failed) + ); + assert!(blocked.worker_cleanup_completed_at.is_none()); + assert_eq!( + current_journal_job(&f).input_disposition, + crate::backend_job::JobInputDisposition::Available + ); + assert!( + f.api + .finish_backend_job_cleanup_retirement(&f.request.job_id, &accepted.attempt_id) + .is_err() + ); + assert!( + session_root(f.dir.path(), worker).exists(), + "completion does not prove Purge" + ); + assert_eq!( + f.store + .retained_journal_generation(&f.host, &f.id, &generation.job) + .unwrap() + .unwrap() + .job, + generation.job + ); +} + +#[tokio::test] +async fn partial_record_without_execution_receipts_preserves_input_and_rejects_continuation() { + let f = RuntimeFixture::new().await; + let dispatched = f.dispatch_input(); + let worker = dispatched.attempt.worker.as_ref().unwrap(); + let generation = f .store .active_journal_generation(&f.host, &f.id) .unwrap() @@ -1164,7 +1381,7 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ JournalJobRequest::Execute { operation: JournalOperation { request_id: "partial-record".into(), - expected_state: g.initial_journal.state_token, + expected_state: generation.initial_journal.state_token.clone(), action: JournalAction::Create { title: "ongoing".into(), body: "Not yet checkpointed".into(), @@ -1180,52 +1397,20 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ .unwrap() else { panic!("saved partial record") }; - f.api - .store - .update_worker_retention( - TEST_WORKSPACE_ID, - worker, - "pinned", - &Utc::now().to_rfc3339(), - ) - .unwrap(); - let unknown = f - .api - .finish_backend_job_attempt( - &f.request.job_id, - &dispatched.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "response_lost", - "dispatch outcome unknown", - ) - .unwrap(); - let failed = wait_for_backend_job_worker_cleanup( - &f.api, - &f.request.job_id, - &dispatched.attempt.attempt_id, - BackendJobWorkerCleanupState::Failed, - ) - .await; + assert!(subjektiv_journal::finish(&f.api, worker).is_err()); + f.execution.take_inputs(); + let job = current_journal_job(&f); + let attempt = current_journal_attempt(&f, &dispatched.attempt.attempt_id); assert!( - f.api - .store - .reconcile_backend_journal_attempt( - TEST_WORKSPACE_ID, - &failed, - None, - &Utc::now().to_rfc3339() - ) - .is_err() + subjektiv_journal::reconcile_attempt(&f.api, &job, &attempt).is_err(), + "missing owner receipts cannot be replaced by timeout or live-map observation" ); - let retained = f - .api - .store - .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) - .unwrap() - .unwrap(); - assert_eq!(retained.input_digest, f.request.input_digest().unwrap()); + assert_eq!(current_journal_job(&f).state, BackendJobState::Pending); assert_eq!( - retained.load_executable_request().unwrap().input, + current_journal_job(&f) + .load_executable_request() + .unwrap() + .input, f.request.input ); assert!( @@ -1236,150 +1421,11 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ &f.request.job_id, &Utc::now().to_rfc3339() ) - .is_err(), - "unknown cannot be reissued as a new attempt before the fence" + .is_err() ); - assert!(f.store.journal_retention_required(&f.host).unwrap()); - assert!(session_root(f.dir.path(), worker).exists()); assert_eq!( - f.api - .store - .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) - .unwrap() - .unwrap() - .state, - BackendJobState::Unknown - ); - assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); - assert!( - f.api - .store - .reconcile_backend_journal_attempt( - TEST_WORKSPACE_ID, - &unknown, - None, - &Utc::now().to_rfc3339() - ) - .is_err(), - "a stale cleanup snapshot must not overwrite newer evidence" - ); - subjektiv_journal::reconcile_unknown(&f.api, &dispatched.job, &unknown).unwrap(); - f.execution.take_inputs(); - f.api - .store - .update_worker_retention( - TEST_WORKSPACE_ID, - worker, - "normal", - &Utc::now().to_rfc3339(), - ) - .unwrap(); - // Discard only the original delivery, before recovery can automatically retry. - f.execution.take_inputs(); - f.api.recover_backend_job_worker_cleanups().unwrap(); - wait_for_backend_job_worker_cleanup( - &f.api, - &f.request.job_id, - &dispatched.attempt.attempt_id, - BackendJobWorkerCleanupState::Completed, - ) - .await; - // Cleanup commits before domain reconciliation. Job Failed is only transient: - // reconcile_unknown fences this attempt, then automatically reserves/queues its - // retry. Observe the immutable attempt outcome and the delivered new generation, - // not the Job's transient state; a manual retry would race the automatic one. - let reconciliation = tokio::time::timeout(std::time::Duration::from_secs(10), async { - loop { - let original = f.api.store.get_backend_job_attempt( - TEST_WORKSPACE_ID, - &f.request.job_id, - &dispatched.attempt.attempt_id, - ).unwrap().unwrap(); - if original.state == BackendJobAttemptState::Failed { - assert_eq!(original.failure_category.as_deref(), Some("journal_attempt_fenced")); - assert_eq!(original.worker_cleanup_state, Some(BackendJobWorkerCleanupState::Completed)); - assert!(original.worker_cleanup_completed_at.is_some()); - if let Some(generation) = f.store.active_journal_generation(&f.host, &f.id).unwrap() - && generation.job.attempt_id != original.attempt_id - && let Some(retry) = f.api.store.get_backend_job_attempt( - TEST_WORKSPACE_ID, - &f.request.job_id, - &generation.job.attempt_id, - ).unwrap() - && retry.state == BackendJobAttemptState::Dispatched - && let Some(resumed) = &retry.worker - // Worker binding precedes input delivery/attestation. ReadState - // succeeds only once the new generation's delivery is attested. - && subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok() - { - break (original, retry, generation); - } - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - }) - .await; - assert!( - reconciliation.is_ok(), - "post-cleanup reconciliation timed out: job={:?}, original_attempt={:?}, active_generation={:?}", - f.api - .store - .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) - .unwrap(), - f.api - .store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &f.request.job_id, - &dispatched.attempt.attempt_id - ) - .unwrap(), f.store.active_journal_generation(&f.host, &f.id).unwrap(), - ); - let (fenced, retry, generation) = reconciliation.unwrap(); - assert_eq!(fenced.worker, dispatched.attempt.worker); - assert_eq!(retry.attempt, dispatched.attempt.attempt + 1); - assert_eq!(generation.job.job_id, f.request.job_id); - assert_eq!(generation.job.attempt_id, retry.attempt_id); - assert_eq!(retry.input_digest, f.request.input_digest().unwrap()); - f.assert_purged(worker); - assert!(f.store.journal_retention_required(&f.host).unwrap()); - assert_eq!( - f.store - .query_journal_operation(&f.host, &f.id, "partial-record") - .unwrap() - .unwrap(), - recorded - ); - assert_eq!( - worker_runtime::retention::runtime_session_retention( - &f.dir.path().join(".test-embedded-runtime-store") - ) - .pins(f.session) - .unwrap() - .len(), - 1, - "source pin survives failed/partial Job cleanup" - ); - let job = f - .api - .store - .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) - .unwrap() - .unwrap(); - assert_eq!(job.state, BackendJobState::Pending); - assert_eq!(job.current_attempt, retry.attempt); - assert_eq!( - job.load_executable_request().unwrap().input, - f.request.input - ); - assert_eq!(job.input_digest, f.request.input_digest().unwrap()); - let resumed = retry.worker.as_ref().unwrap(); - let delivered = f.execution.take_inputs(); - assert_eq!(delivered.len(), 1); - assert_eq!( - delivered[0].1, - f.request.worker_input(&retry.attempt_id).unwrap() + Some(generation) ); assert_eq!( f.store @@ -1387,93 +1433,316 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ .unwrap(), Some(recorded) ); - assert_eq!( + assert!( f.store - .list_journal_entries(&f.host, &f.id, None, 16) + .query_journal_operation(&f.host, &f.id, "checkpoint") .unwrap() - .items - .len(), - 1 + .is_none() ); - assert!(subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok()); - assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); -} - -#[tokio::test] -async fn journal_reserved_restart_dispatches_exact_intent_once_and_unbound_unknown_is_not_replaced() -{ - let f = RuntimeFixture::new().await; - f.execution.take_inputs(); - f.api.recover_backend_jobs().unwrap(); - let first = f - .api - .store - .get_backend_job_attempt( - TEST_WORKSPACE_ID, - &f.request.job_id, - &f.reserved.attempt.attempt_id, - ) - .unwrap() - .unwrap(); - assert_eq!(first.state, BackendJobAttemptState::Dispatched); - assert_eq!(f.execution.take_inputs().len(), 1); - f.api.recover_backend_jobs().unwrap(); + assert!(f.store.journal_retention_required(&f.host).unwrap()); assert!(f.execution.take_inputs().is_empty()); - assert_eq!( - f.api - .store - .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) - .unwrap() - .unwrap() - .request, - f.request - ); - let unknown_fixture = RuntimeFixture::new().await; - let unknown = unknown_fixture + // Failure is neither recording coverage nor owner shutdown. Do not schedule + // cleanup here or manufacture an effects-ended receipt for this live owner. + let failed = f .api .store .finish_backend_job_attempt( TEST_WORKSPACE_ID, - &unknown_fixture.request.job_id, - &unknown_fixture.reserved.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "allocation_unknown", - "allocation outcome has no bound Worker", + &f.request.job_id, + &attempt.attempt_id, + BackendJobAttemptState::Failed, + "execution_failed", + "deterministic failure; effects still require owner evidence", &Utc::now().to_rfc3339(), ) .unwrap(); - assert_eq!( - unknown.worker_cleanup_state, - Some(BackendJobWorkerCleanupState::Completed) + assert_eq!(failed.state, BackendJobAttemptState::Failed); + assert!( + f.api + .require_backend_job_attempt_effects_ended(&failed) + .is_err() ); assert!( - subjektiv_journal::reconcile_unknown( - &unknown_fixture.api, - &unknown_fixture.reserved.job, - &unknown + subjektiv_journal::require_job_effects_ended(&f.api, ¤t_journal_job(&f)).is_err() + ); + assert!(f.store.journal_retention_required(&f.host).unwrap()); + assert!(subjektiv_journal::finish(&f.api, worker).is_err()); +} + +#[tokio::test] +async fn journal_reserved_restart_dispatches_exact_saved_intent_once() { + let f = RuntimeFixture::new().await; + f.execution.take_inputs(); + f.api.recover_backend_jobs().unwrap(); + let first = current_journal_attempt(&f, &f.reserved.attempt.attempt_id); + assert_eq!(first.state, BackendJobAttemptState::Dispatched); + assert!(first.runtime_run_id.is_some()); + assert_eq!(f.execution.take_inputs().len(), 1); + f.api.recover_backend_jobs().unwrap(); + assert!(f.execution.take_inputs().is_empty()); + assert_eq!(current_journal_job(&f).request, f.request); + let again = current_journal_attempt(&f, &first.attempt_id); + assert_eq!(again.worker, first.worker); + assert_eq!(again.runtime_run_id, first.runtime_run_id); +} + +// Seed the post-migration boundary, not a new live legacy state. Actual ordered +// migration/rollback coverage belongs to backend_job_upgrade_tests; these fixtures +// test consumers of its immutable marker and preserve independent cleanup evidence. +fn fixture_journal_migration_stopped(api: &WorkspaceApi, job_id: &str) { + let connection = rusqlite::Connection::open(&api.config.database_path).unwrap(); + let tx = connection.unchecked_transaction().unwrap(); + tx.execute( + "INSERT INTO backend_job_upgrade_history + (workspace_id,job_id,state,failure_category,failure_detail,updated_at,completed_at) + SELECT workspace_id,job_id,state,failure_category,failure_detail,updated_at,completed_at + FROM backend_jobs WHERE workspace_id=?1 AND job_id=?2", + rusqlite::params![TEST_WORKSPACE_ID, job_id], + ) + .unwrap(); + tx.execute("INSERT INTO backend_job_attempt_upgrade_history + (workspace_id,job_id,attempt_id,state,failure_category,failure_detail,updated_at,completed_at) + SELECT workspace_id,job_id,attempt_id,state,failure_category,failure_detail,updated_at,completed_at + FROM backend_job_attempts WHERE workspace_id=?1 AND job_id=?2", rusqlite::params![TEST_WORKSPACE_ID,job_id]).unwrap(); + for table in ["backend_job_attempts", "backend_jobs"] { + tx.execute(&format!("UPDATE {table} SET state='failed',failure_category='version_upgrade_execution_aborted', + failure_detail='version upgrade stopped this execution; no termination or recording proof', + completed_at=?3,updated_at=?3 WHERE workspace_id=?1 AND job_id=?2 AND state NOT IN ('completed','failed')"), + rusqlite::params![TEST_WORKSPACE_ID,job_id,Utc::now().to_rfc3339()]).unwrap(); + } + tx.commit().unwrap(); +} + +#[tokio::test] +async fn migration_stopped_checkpoint_is_excluded_from_recovery_result_acceptance_and_retry() { + let f = RuntimeFixture::new().await; + let dispatched = f.dispatch_input(); + let worker = dispatched.attempt.worker.as_ref().unwrap(); + f.checkpoint(worker); + let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); + let checkpoint = f + .store + .query_journal_operation(&f.host, &f.id, "checkpoint") + .unwrap(); + let generation = f.store.active_journal_generation(&f.host, &f.id).unwrap(); + fixture_journal_migration_stopped(&f.api, &f.request.job_id); + let stopped = current_journal_job(&f); + let stopped_attempt = current_journal_attempt(&f, &dispatched.attempt.attempt_id); + assert_eq!(stopped.state, BackendJobState::Failed); + assert_eq!( + stopped.failure_category.as_deref(), + Some("version_upgrade_execution_aborted") + ); + f.execution.take_inputs(); + // Even a callback retaining the pre-cutover Pending snapshot must consult the + // current migration marker before looking for an execution or saved completion. + subjektiv_journal::reconcile_attempt(&f.api, &dispatched.job, &dispatched.attempt).unwrap(); + subjektiv_journal::recover_attempts(&f.api).unwrap(); + f.api.recover_backend_jobs().unwrap(); + assert!( + f.api + .accept_backend_job_result( + worker, + &BackendJobResultSubmission { + job_id: f.request.job_id.clone(), + attempt_id: stopped_attempt.attempt_id.clone(), + input_digest: stopped_attempt.input_digest.clone(), + result: serde_json::to_value(completion).unwrap(), + } + ) + .is_err() + ); + assert!( + f.api + .store + .reserve_backend_job_retry( + TEST_WORKSPACE_ID, + &f.request.job_id, + &Utc::now().to_rfc3339() + ) + .is_err() + ); + let after = current_journal_job(&f); + assert_eq!(after.state, BackendJobState::Failed); + assert_eq!(after.current_attempt, stopped.current_attempt); + assert_eq!(after.failure_category, stopped.failure_category); + assert_eq!( + after.load_executable_request().unwrap().input, + f.request.input + ); + assert_eq!(after.input_digest, stopped.input_digest); + assert!( + after.result.is_none(), + "a migration Failed cannot be promoted from checkpoint" + ); + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, "checkpoint") + .unwrap(), + checkpoint + ); + assert_eq!( + f.store.active_journal_generation(&f.host, &f.id).unwrap(), + generation + ); + assert!(f.execution.take_inputs().is_empty()); + assert!(subjektiv_journal::require_job_effects_ended(&f.api, &after).is_err()); +} + +#[tokio::test] +async fn saved_checkpoint_never_promotes_erased_released_retired_or_missing_source_to_success() { + for unavailable in ["erased", "body_released", "retired", "deleted_worker"] { + let f = RuntimeFixture::new().await; + let dispatched = f.dispatch_input(); + let worker = dispatched.attempt.worker.as_ref().unwrap(); + f.checkpoint(worker); + let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); + let checkpoint = f + .store + .query_journal_operation(&f.host, &f.id, "checkpoint") + .unwrap(); + // These are pre-existing domain/Store facts at the recovery boundary, not + // a claim that physical removal or all effects ended in this fixture. + match unavailable { + "erased" => { + f.store + .erase_journal_session(&f.host, "erase-before-recovery") + .unwrap(); + } + "body_released" => { + f.store.release_removed_journal_body(&f.host).unwrap(); + } + "retired" => { + f.api + .store + .reclaim_backend_job_input( + TEST_WORKSPACE_ID, + &f.request.job_id, + &f.request.input_digest().unwrap(), + crate::backend_job::JobInputRetirementReason::LegacyInvalidation, + ) + .unwrap(); + } + "deleted_worker" => { + f.api + .store + .delete_worker_registry(TEST_WORKSPACE_ID, &f.source) + .unwrap(); + } + _ => unreachable!(), + } + let job = current_journal_job(&f); + let attempt = current_journal_attempt(&f, &dispatched.attempt.attempt_id); + f.execution.take_inputs(); + subjektiv_journal::reconcile_attempt(&f.api, &job, &attempt).unwrap(); + let failed = current_journal_job(&f); + assert_eq!(failed.state, BackendJobState::Failed, "{unavailable}"); + assert_eq!( + failed.failure_category.as_deref(), + Some(match unavailable { + "erased" => "journal_session_erased", + "body_released" => "journal_body_released", + "retired" => "journal_input_unavailable", + "deleted_worker" => "journal_source_worker_missing", + _ => unreachable!(), + }) + ); + assert!(failed.result.is_none()); + assert_eq!(failed.current_attempt, attempt.attempt); + assert_eq!(failed.input_digest, attempt.input_digest); + assert!( + f.api + .accept_backend_job_result( + worker, + &BackendJobResultSubmission { + job_id: f.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_digest: attempt.input_digest.clone(), + result: serde_json::to_value(completion).unwrap(), + } + ) + .is_err() + ); + assert!( + f.api + .dispatch_backend_job_reservation(dispatched.clone()) + .is_err() + ); + assert!( + f.execution.take_inputs().is_empty(), + "{unavailable} must not redispatch or delegate saved input" + ); + if unavailable != "erased" { + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, "checkpoint") + .unwrap(), + checkpoint + ); + } else { + assert!(f.store.journal_session_erased(&f.host).unwrap()); + } + if unavailable == "retired" { + assert!(failed.load_executable_request().is_err()); + assert_eq!( + failed.input_disposition, + crate::backend_job::JobInputDisposition::Invalidated + ); + } + if unavailable == "deleted_worker" { + assert!( + f.api + .store + .get_worker_registry(TEST_WORKSPACE_ID, &f.source) + .unwrap() + .is_none() + ); + } + } +} + +#[tokio::test] +async fn checkpoint_recovery_rejects_stale_snapshot_and_mismatched_durable_run() { + let f = RuntimeFixture::new().await; + let dispatched = f.dispatch_input(); + let worker = dispatched.attempt.worker.as_ref().unwrap(); + f.checkpoint(worker); + let job = current_journal_job(&f); + let attempt = current_journal_attempt(&f, &dispatched.attempt.attempt_id); + let mut stale = attempt.clone(); + stale.runtime_run_id = Some("old-owner-run".into()); + assert!(subjektiv_journal::reconcile_attempt(&f.api, &job, &stale).is_err()); + // Corrupt the external durable binding, then pass the newly read snapshot. + // Comparing only the caller snapshot with the DB would incorrectly settle it. + let connection = rusqlite::Connection::open(&f.api.config.database_path).unwrap(); + connection + .execute( + "UPDATE backend_job_attempts SET runtime_run_id='mismatched-run' + WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3", + rusqlite::params![TEST_WORKSPACE_ID, f.request.job_id, attempt.attempt_id], ) - .is_err() + .unwrap(); + let mismatched = current_journal_attempt(&f, &attempt.attempt_id); + assert!( + subjektiv_journal::reconcile_attempt(&f.api, ¤t_journal_job(&f), &mismatched) + .is_err() + ); + assert_eq!(current_journal_job(&f).state, BackendJobState::Pending); + assert!(current_journal_job(&f).result.is_none()); + assert!( + f.store + .query_journal_operation(&f.host, &f.id, "checkpoint") + .unwrap() + .is_some() ); assert_eq!( - unknown_fixture - .api - .store - .get_backend_job(TEST_WORKSPACE_ID, &unknown_fixture.request.job_id) + current_journal_job(&f) + .load_executable_request() .unwrap() - .unwrap() - .state, - BackendJobState::Unknown + .input, + f.request.input ); - assert!( - unknown_fixture - .store - .active_journal_generation(&unknown_fixture.host, &unknown_fixture.id) - .unwrap() - .is_some(), - "the generation is not closed before an actual lifecycle fence" - ); - assert!(unknown_fixture.execution.take_inputs().is_empty()); } #[tokio::test] @@ -1624,11 +1893,13 @@ async fn journal_profile_name_without_host_job_binding_keeps_normal_archive_poli } #[tokio::test] -async fn saved_checkpoint_unknown_auto_resolves_without_waiting_for_failed_session_cleanup() { - for recovery in ["transition", "restart"] { +async fn saved_checkpoint_recovery_completes_without_proving_failed_cleanup_ended() { + for recovery in ["journal", "backend"] { let f = RuntimeFixture::new().await; + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); + f.checkpoint(worker); // Deliberately do not call finish: checkpoint receipt is durable. f.execution.take_inputs(); f.api @@ -1640,34 +1911,14 @@ async fn saved_checkpoint_unknown_auto_resolves_without_waiting_for_failed_sessi &Utc::now().to_rfc3339(), ) .unwrap(); - if recovery == "transition" { - let result = f - .api - .finish_backend_job_attempt( - &f.request.job_id, - &dispatched.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "result_lost", - "result acknowledgement unavailable", - ) - .unwrap(); - assert_eq!(result.state, BackendJobAttemptState::Completed); + + if recovery == "journal" { + subjektiv_journal::recover_attempts(&f.api).unwrap(); } else { - f.api - .store - .finish_backend_job_attempt( - TEST_WORKSPACE_ID, - &f.request.job_id, - &dispatched.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "result_lost", - "Backend stopped before outcome reconciliation", - &Utc::now().to_rfc3339(), - ) - .unwrap(); - // The generic Job recovery entry must not require a later source notification. + // Generic startup recovery must reach the same domain receipt owner. f.api.recover_backend_jobs().unwrap(); } + let blocked = wait_for_backend_job_worker_cleanup( &f.api, &f.request.job_id, @@ -1675,6 +1926,7 @@ async fn saved_checkpoint_unknown_auto_resolves_without_waiting_for_failed_sessi BackendJobWorkerCleanupState::Failed, ) .await; + assert_eq!(blocked.state, BackendJobAttemptState::Completed); let saved = f .api @@ -1707,7 +1959,9 @@ async fn saved_checkpoint_unknown_auto_resolves_without_waiting_for_failed_sessi &Utc::now().to_rfc3339(), ) .unwrap(); + f.api.recover_backend_jobs().unwrap(); + wait_for_backend_job_worker_cleanup( &f.api, &f.request.job_id, @@ -2666,7 +2920,8 @@ async fn subject_removal_rejects_mismatched_runtime_receipts_before_revocation() } #[tokio::test] -async fn subject_removal_unknown_job_settles_after_cleanup_without_restarting_old_writer() { +async fn subject_removal_failed_job_keeps_unrecorded_input_after_cleanup_without_restarting_writer() +{ let f = SubjectRemovalFixture::new().await; let reserved = f.pending_journal(); let dispatched = f @@ -2693,9 +2948,9 @@ async fn subject_removal_unknown_job_settles_after_cleanup_without_restarting_ol .finish_backend_job_attempt( &reserved.job.request.job_id, &dispatched.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "response_lost", - "test lost result", + BackendJobAttemptState::Failed, + "execution_timeout", + "fixture execution deadline exceeded; shutdown is checked separately", ) .unwrap(); wait_for_backend_job_worker_cleanup( @@ -2706,13 +2961,13 @@ async fn subject_removal_unknown_job_settles_after_cleanup_without_restarting_ol ) .await; let response = f.remove(); - assert_eq!( + assert_ne!( response.status, StatusCode::OK.as_u16(), "{}", response.body ); - f.assert_removed_with_history(); + f.assert_occupied(); assert_eq!( f.api .store @@ -2720,7 +2975,7 @@ async fn subject_removal_unknown_job_settles_after_cleanup_without_restarting_ol .unwrap() .unwrap() .state, - BackendJobState::Unknown + BackendJobState::Failed ); f.api .store @@ -2782,6 +3037,8 @@ async fn subject_removal_unknown_job_settles_after_cleanup_without_restarting_ol reserved.job.request.input ); assert!(f.store.journal_retention_required(&f.host).unwrap()); + assert_ne!(f.remove().status, StatusCode::OK.as_u16()); + f.assert_occupied(); } #[tokio::test] @@ -2900,7 +3157,7 @@ async fn subject_removal_prior_archive_policy_receipt_remains_recoverable() { assert_eq!(prepared.plan.policy_digest, recovered.plan.policy_digest); } -async fn assert_subject_removal_recovers_unknown_snapshot(legacy: bool, completed: bool) { +async fn assert_unbound_pending_snapshot_keeps_source(legacy: bool, completed: bool) { let f = SubjectRemovalFixture::new().await; let reserved = f.pending_journal_input(legacy); let grant = reserved @@ -2960,58 +3217,49 @@ async fn assert_subject_removal_recovers_unknown_snapshot(legacy: bool, complete .unwrap(); subjektiv::journal::service::finish(&f.store, &ctx).unwrap(); } - // With no bound execution Worker, the store records cleanup Completed. The - // unfinished case must remain Unknown (absence is not a Purge receipt). + // Keep the original reservation and diagnosis; map absence proves no cleanup. f.api .store - .finish_backend_job_attempt( + .record_backend_job_reconciliation( TEST_WORKSPACE_ID, - &reserved.job.request.job_id, - &reserved.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "lost-response", + &reserved.attempt, + "response_lost", "fixture retained outcome", + "Reconcile saved Journal operations and the original execution owner", &Utc::now().to_rfc3339(), ) .unwrap(); let response = f.remove(); assert_eq!( response.status, - StatusCode::OK.as_u16(), + StatusCode::CONFLICT.as_u16(), "{}", response.body ); - f.assert_removed_with_history(); + f.assert_occupied(); let job = f .api .store .get_backend_job(TEST_WORKSPACE_ID, &reserved.job.request.job_id) .unwrap() .unwrap(); - assert_eq!( - job.state, - if completed { - BackendJobState::Completed - } else { - BackendJobState::Unknown - } - ); + assert_eq!(job.state, BackendJobState::Pending); assert_eq!(job.current_attempt, 1); } #[tokio::test] -async fn subject_removal_recovers_completed_unknown_without_recursive_subject_lock() { - assert_subject_removal_recovers_unknown_snapshot(false, true).await; +async fn subject_removal_keeps_unbound_checkpoint_without_execution_evidence() { + assert_unbound_pending_snapshot_keeps_source(false, true).await; } #[tokio::test] -async fn subject_removal_recovers_legacy_completed_unknown_without_writer_authority() { - assert_subject_removal_recovers_unknown_snapshot(true, true).await; +async fn subject_removal_keeps_legacy_checkpoint_without_execution_evidence() { + assert_unbound_pending_snapshot_keeps_source(true, true).await; } #[tokio::test] -async fn subject_removal_retains_legacy_unbound_unknown_without_blocking_deletion() { - assert_subject_removal_recovers_unknown_snapshot(true, false).await; +async fn subject_removal_blocks_legacy_unbound_pending_without_execution_evidence() { + assert_unbound_pending_snapshot_keeps_source(true, false).await; } #[tokio::test] @@ -4224,6 +4472,7 @@ mod immutable { dir, api, execution, + job_store, store, host, source, @@ -4246,10 +4495,12 @@ mod immutable { let config = api.config.clone(); drop(api); drop(store); + drop(job_store); // Reopen after input save but before dispatch, with no readable original source. + let job_store = Arc::new(test_control_store(&config)); let api = WorkspaceApi::new_with_execution_backend_and_broker( config.clone(), - Arc::new(test_control_store(&config)), + job_store.clone(), execution.clone(), BackendResourceBroker::default(), HostExecutors::default(), @@ -4262,6 +4513,7 @@ mod immutable { dir, api, execution, + job_store, store, host, source, @@ -4282,15 +4534,35 @@ mod immutable { frozen ); assert_eq!(saved.input_digest, digest); + assert_eq!( + saved.state, + BackendJobState::Pending, + "pre-dispatch reopen failure: {:?}: {:?}", + saved.failure_category, + saved.failure_detail + ); let dispatched = f .api .dispatch_backend_job_reservation(f.reserved.clone()) .unwrap(); + assert_eq!( + dispatched.attempt.state, + BackendJobAttemptState::Dispatched, + "dispatch failure: {:?}: {:?}", + dispatched.attempt.failure_category, + dispatched.attempt.failure_detail + ); assert_eq!( f.store .active_journal_generation(&f.host, &f.id) .unwrap() - .unwrap(), + .unwrap_or_else(|| { + let current = current_journal_job(&f); + let attempt = current_journal_attempt(&f, &dispatched.attempt.attempt_id); + panic!("generation disappeared after dispatch: job={:?} {:?}: {:?}; attempt={:?} {:?}: {:?}", + current.state, current.failure_category, current.failure_detail, + attempt.state, attempt.failure_category, attempt.failure_detail); + }), generation ); let worker = dispatched.attempt.worker.as_ref().unwrap(); @@ -4740,6 +5012,27 @@ async fn explicit_erasure_reclaims_over_100_historical_body_inputs_and_waits_for ); assert_eq!(saved.input_digest, f.request.input_digest().unwrap()); assert!(saved.load_executable_request().is_err()); + for attempt in f + .api + .store + .list_backend_job_attempts(TEST_WORKSPACE_ID, &saved.request.job_id) + .unwrap() + { + if attempt.worker.is_none() { + assert!( + attempt.allocation_precluded_at.is_some(), + "no inference from Failed or missing Worker" + ); + f.api + .require_backend_job_attempt_effects_ended(&attempt) + .unwrap(); + } else { + assert!( + attempt.allocation_precluded_at.is_none(), + "an admitted execution needs its separate termination/Host receipts" + ); + } + } assert!( !serde_json::to_string(&saved.request) .unwrap() @@ -4940,12 +5233,15 @@ mod lifecycle_barrier { assert_no_preaccept_attestation(&f); release.send(()).unwrap(); // Dispatch must independently finish after acceptance, even with the Tool waiting. + let dispatched = joined(dispatch).unwrap(); + let unfinished = joined(early_finish).unwrap_err().to_string(); assert!( !unfinished.contains("trusted Host delivery"), "acceptance attestation now exists" ); + let JournalJobResponse::Saved(record) = joined(tool).unwrap() else { panic!("record receipt") }; @@ -4961,6 +5257,7 @@ mod lifecycle_barrier { 0, "acceptance and mutation alone are not a checkpoint" ); + let position = save_fixture_position(&f.api, &worker, record.state_token); subjektiv_journal::execute( &f.api, @@ -4977,23 +5274,34 @@ mod lifecycle_barrier { }, ) .unwrap(); - let api = f.api.clone(); - let callback_worker = worker.clone(); - let completion = joined(spawn(move || { - subjektiv_journal::finish(&api, &callback_worker) - })) + // Recovery may settle the checkpoint before the Worker's finish callback. + // Do not require a new live write grant after that exact outcome is saved. + let saved = observe_journal_completion(&f).await; + assert_eq!(saved.state, BackendJobState::Completed); + assert_eq!(saved.current_attempt, dispatched.attempt.attempt); + let attempt = current_journal_attempt(&f, &dispatched.attempt.attempt_id); + assert_eq!(attempt.state, BackendJobAttemptState::Completed); + assert_eq!(attempt.worker.as_ref(), Some(&worker)); + assert_eq!(attempt.runtime_run_id, dispatched.attempt.runtime_run_id); + assert_eq!(attempt.result, saved.result); + let completion: JournalJobCompletion = + serde_json::from_value(saved.result.clone().unwrap()).unwrap(); + assert_eq!(completion.process_order, 1); + assert_eq!(completion.process_byte, 0); + let context = JournalJobContext::validated_recovery( + &f.store, + &f.host, + &completion.generation_id, + subjektiv::api::JobAttemptBinding { + job_id: saved.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + }, + ) .unwrap(); - f.api - .accept_backend_job_result( - &worker, - &BackendJobResultSubmission { - job_id: f.request.job_id.clone(), - attempt_id: dispatched.attempt.attempt_id.clone(), - input_digest: f.request.input_digest().unwrap(), - result: serde_json::to_value(completion).unwrap(), - }, - ) - .unwrap(); + assert_eq!( + journal::service::finish(&f.store, &context).unwrap(), + completion + ); wait_for_backend_job_worker_cleanup( &f.api, &f.request.job_id, diff --git a/crates/workspace-server/src/storage_format_tests.rs b/crates/workspace-server/src/storage_format_tests.rs index 3bc3c034b..036f686c9 100644 --- a/crates/workspace-server/src/storage_format_tests.rs +++ b/crates/workspace-server/src/storage_format_tests.rs @@ -110,13 +110,18 @@ fn assert_accepted_without_domain_changes(conn: &Connection, current: i64) { verify_history(conn, current).unwrap(); if current >= 114 { let plan = published_plan(conn, current).unwrap().unwrap(); - assert_eq!(plan.migrations.len(), 1); + assert_eq!(plan.migrations.len(), 2); assert_eq!(plan.migrations[0].version, 122); assert_eq!( plan.migrations[0].name, crate::config_adoption::MIGRATION_NAME ); - assert_eq!(plan.target_schema_version, 122); + assert_eq!(plan.migrations[1].version, 123); + assert_eq!( + plan.migrations[1].name, + super::super::backend_job_upgrade::MIGRATION_NAME + ); + assert_eq!(plan.target_schema_version, 123); } assert_eq!(rows, saved_rows(conn)); assert_eq!(shape, snapshot(conn).unwrap()); @@ -171,59 +176,121 @@ fn converged_shapes_keep_all_real_predecessor_receipts_and_erasure_fences() { } } +// Snapshot every retained value except the three fields the new migration owns. +// Existing published-family history tables remain byte-for-byte in this check. +fn immutable_rows(conn: &Connection) -> Vec<(String, Vec)> { + let mut result = Vec::new(); + for (name, _) in saved_rows(conn) { + if name == "__yoi_schema_migrations" { + continue; + } + let columns = super::super::table_columns(conn, &name) + .unwrap() + .into_iter() + .filter(|column| { + !["backend_jobs", "backend_job_attempts"].contains(&name.as_str()) + || !["state", "failure_category", "failure_detail"].contains(&column.as_str()) + }) + .map(|column| format!("\"{}\"", column.replace('"', "\"\""))) + .collect::>() + .join(","); + let mut statement = conn + .prepare(&format!("SELECT {columns} FROM \"{name}\"")) + .unwrap(); + let count = statement.column_count(); + let mut rows = statement + .query_map([], |row| { + (0..count) + .map(|index| row.get::<_, rusqlite::types::Value>(index)) + .collect::>>() + }) + .unwrap() + .map(|row| format!("{:?}", row.unwrap())) + .collect::>(); + rows.sort(); + result.push((name, rows)); + } + result +} + +fn exercise_published_runner(stage: Stage, baseline: i64, predecessor: Option<(Family, usize)>) { + let source = fixture(stage, baseline, predecessor); + let expected = immutable_rows(&source); + let original_receipts = super::super::workspace_schema_migration_history(&source).unwrap(); + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("catalog.db"); + let mut destination = Connection::open(&path).unwrap(); + Backup::new(&source, &mut destination) + .unwrap() + .run_to_completion(128, std::time::Duration::ZERO, None) + .unwrap(); + drop(destination); + let mut committed_history = None; + for _ in 0..2 { + let conn = Connection::open(&path).unwrap(); + // Actual production open/ordered runner, including the new migration. + super::super::prepare_connection(&conn).unwrap(); + assert_eq!(super::super::current_schema_version(&conn).unwrap(), 123); + let actual = immutable_rows(&conn); + for (table, rows) in &expected { + assert_eq!( + &actual.iter().find(|(name, _)| name == table).unwrap().1, + rows, + "{table}, from {stage:?}, baseline {baseline}" + ); + } + let outcome:(String,String,Option)=conn.query_row("SELECT state,failure_category,completed_at FROM backend_jobs WHERE job_id='saved-job'",[],|row|Ok((row.get(0)?,row.get(1)?,row.get(2)?))).unwrap(); + assert_eq!( + outcome, + ( + "failed".into(), + super::super::backend_job_upgrade::FAILURE_CATEGORY.into(), + None + ) + ); + let mut history = super::super::workspace_schema_migration_history(&conn).unwrap(); + assert_eq!( + history.pop(), + Some(( + 123, + super::super::backend_job_upgrade::MIGRATION_NAME.into() + )) + ); + assert_eq!( + history.pop(), + Some((122, crate::config_adoption::MIGRATION_NAME.into())) + ); + assert_eq!(history, original_receipts); + verify_history(&conn, 123).unwrap(); + let saved = saved_rows(&conn); + if let Some(previous) = &committed_history { + assert_eq!(previous, &saved); + } else { + committed_history = Some(saved); + } + } +} + #[test] fn published_database_runs_normal_store_preparation_and_reopens_without_downgrade() { - for stage in [ - Stage::Published(Family::Permissions, 2), - Stage::Published(Family::Removal, 4), - Stage::Published(Family::Jobs, 5), - Stage::Converged(119), - Stage::Converged(120), - Stage::Converged(121), - ] { - let current = match stage { - Stage::Published(_, count) => 113 + count as i64, - Stage::Converged(version) => version, - Stage::Base => 113, - }; - let source = fixture(stage, 113, None); - let expected = saved_rows(&source); - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("catalog.db"); - let mut destination = Connection::open(&path).unwrap(); - Backup::new(&source, &mut destination) - .unwrap() - .run_to_completion(128, std::time::Duration::ZERO, None) - .unwrap(); - drop(destination); - for _ in 0..2 { - let conn = Connection::open(&path).unwrap(); - // Production store owner, not a separate test migration implementation. - super::super::prepare_connection(&conn).unwrap(); - assert_eq!(super::super::current_schema_version(&conn).unwrap(), 122); - for (table, rows) in &expected { - if table != "__yoi_schema_migrations" { - assert_eq!( - saved_rows(&conn) - .into_iter() - .find(|(name, _)| name == table) - .unwrap() - .1, - *rows, - "{table}, from {current}" - ); - } + for family in FAMILIES { + for count in 1..=family.names().len() { + let stage = Stage::Published(family, count); + for baseline in 113..=113 + count as i64 { + exercise_published_runner(stage, baseline, None); } - let mut history = super::super::workspace_schema_migration_history(&conn).unwrap(); - assert_eq!( - history.pop(), - Some((122, crate::config_adoption::MIGRATION_NAME.into())) - ); - assert_eq!( - history, - super::super::workspace_schema_migration_history(&source).unwrap() - ); - verify_history(&conn, 122).unwrap(); + } + } + for current in 119..=121 { + for baseline in 119..=current { + exercise_published_runner(Stage::Converged(current), baseline, None); + } + for origin in + std::iter::once(None).chain(FAMILIES.into_iter().flat_map(|family| { + (1..=family.names().len()).map(move |count| Some((family, count))) + })) + { + exercise_published_runner(Stage::Converged(current), 113, origin); } } } diff --git a/crates/workspace-server/src/storage_formats.rs b/crates/workspace-server/src/storage_formats.rs index aafe26414..32d187d20 100644 --- a/crates/workspace-server/src/storage_formats.rs +++ b/crates/workspace-server/src/storage_formats.rs @@ -75,6 +75,26 @@ pub(super) fn verify_history(conn: &Connection, current: i64) -> Result<()> { } fn verify_history_records(conn: &Connection, current: i64, rows: &[(i64, String)]) -> Result<()> { + verify_raw_capture_shape(conn, current, rows)?; + if current == 123 { + verify_execution_shape(conn)?; + super::backend_job_upgrade::verify(conn)?; + if rows == [(123, super::SCHEMA_BASELINE_NAME.to_owned())] { + verify_configuration_shape(conn)?; + require_shape(conn, Stage::Base)?; + return verify_103_features(conn, current, rows); + } + if rows.last().map(|(version, name)| (*version, name.as_str())) + != Some((123, super::backend_job_upgrade::MIGRATION_NAME)) + { + return Err(invalid("unrecognized Backend Job execution receipt")); + } + let prefix = &rows[..rows.len() - 1]; + if prefix.last().map(|row| row.0) != Some(122) { + return Err(invalid("missing Backend Job execution predecessor")); + } + return verify_history_records(conn, 122, prefix); + } if current == 122 { verify_configuration_shape(conn)?; if rows == [(122, super::SCHEMA_BASELINE_NAME.to_owned())] { @@ -216,15 +236,18 @@ pub(super) fn published_plan( verify_history(conn, current)?; Ok(Some(super::WorkspaceSchemaMigrationPlan { current_schema_version: current, - target_schema_version: 122, - migrations: if current < 122 { - vec![super::WorkspaceSchemaMigrationStep { - version: 122, - name: crate::config_adoption::MIGRATION_NAME.to_owned(), - }] - } else { - vec![] - }, + target_schema_version: 123, + migrations: [ + (122, crate::config_adoption::MIGRATION_NAME), + (123, super::backend_job_upgrade::MIGRATION_NAME), + ] + .into_iter() + .filter(|(version, _)| *version > current) + .map(|(version, name)| super::WorkspaceSchemaMigrationStep { + version, + name: name.to_owned(), + }) + .collect(), })) } @@ -347,6 +370,139 @@ fn build_references() -> Result { }) } +fn extension_shape(sql: &str) -> Result { + let reference = Connection::open_in_memory()?; + reference.execute_batch("CREATE TABLE backend_jobs(workspace_id TEXT,job_id TEXT,request_json TEXT); CREATE TABLE backend_job_attempts(workspace_id TEXT,job_id TEXT,attempt_id TEXT); CREATE TABLE backend_job_deliveries(workspace_id TEXT,delivery_id TEXT);")?; + reference.execute_batch(sql)?; + let mut shape = snapshot(&reference)?; + for name in [ + "backend_jobs", + "backend_job_attempts", + "backend_job_deliveries", + ] { + shape.remove(name); + } + Ok(shape) +} + +fn require_extension(conn: &Connection, sql: &str) -> Result { + let expected = extension_shape(sql)?; + let actual = snapshot(conn)?; + let tables: BTreeSet<_> = expected + .iter() + .filter(|(_, object)| object.0 == "table") + .map(|(_, object)| object.1.clone()) + .collect(); + for (name, object) in &actual { + if tables.contains(&object.1) && !expected.contains_key(name) { + return Err(invalid(format!("unreceipted Job evidence object: {name}"))); + } + } + for (name, object) in &expected { + if actual.get(name) != Some(object) { + return Err(invalid(format!("Job evidence schema drift: {name}"))); + } + } + Ok(expected) +} + +fn verify_raw_capture_shape(conn: &Connection, current: i64, rows: &[(i64, String)]) -> Result<()> { + let expected = extension_shape(super::backend_job_upgrade::RAW_SCHEMA)?; + let actual = snapshot(conn)?; + let mut execution = extension_shape(super::backend_job_upgrade::SCHEMA)?; + execution.extend(extension_shape(super::backend_job_recovery_store::SCHEMA)?); + if current < 123 + && super::current_schema_version(conn)? < 123 + && execution.keys().any(|name| actual.contains_key(name)) + { + return Err(invalid("Backend Job execution objects before receipt 123")); + } + let present = expected.keys().any(|name| actual.contains_key(name)); + if !present { + return Ok(()); + } + if current < 86 { + return Err(invalid("pre-content capture before its schema-86 boundary")); + } + require_extension(conn, super::backend_job_upgrade::RAW_SCHEMA)?; + let populated: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM backend_job_pre_content_history)", + [], + |row| row.get(0), + )?; + if populated + && !(rows.first().is_some_and(|row| row.0 <= 85) && rows.iter().any(|row| row.0 == 86)) + { + return Err(invalid( + "populated pre-content capture without retained schema-86 history", + )); + } + super::backend_job_upgrade::verify_pre_content(conn) +} + +fn verify_execution_shape(conn: &Connection) -> Result<()> { + require_extension(conn, super::backend_job_upgrade::RAW_SCHEMA)?; + require_extension(conn, super::backend_job_upgrade::SCHEMA)?; + require_extension(conn, super::backend_job_recovery_store::SCHEMA)?; + Ok(()) +} + +// Ignore only separately validated evidence programs, not arbitrary extra Job +// triggers. The retained family's CHECK is transformed only after receipt 123; +// all other retained columns, programs and original receipt names remain exact. +fn legacy_shape(conn: &Connection) -> Result { + let mut shape = snapshot(conn)?; + for sql in [ + super::backend_job_upgrade::RAW_SCHEMA, + super::backend_job_upgrade::SCHEMA, + super::backend_job_recovery_store::SCHEMA, + ] { + for name in extension_shape(sql)?.keys() { + shape.remove(name); + } + } + Ok(shape) +} + +fn upgraded_reference(conn: &Connection, mut shape: Shape) -> Result { + if super::current_schema_version(conn)? != 123 { + return Ok(shape); + } + for table in ["backend_jobs", "backend_job_attempts"] { + if let Some((_, _, tokens)) = shape.get_mut(table) { + if table == "backend_job_attempts" { + // This exact nullable declaration belongs only to receipt 123. + // Canonical declarations are sorted; allocation precedes attempt. + let start = tokens + .iter() + .position(|token| token == "(") + .ok_or_else(|| invalid("missing attempt declarations"))?; + tokens.splice( + start + 1..start + 1, + [ + "word:allocation_precluded_at".into(), + "word:text".into(), + ",".into(), + ], + ); + } + // These exact retained state declarations put the legacy value last. + let start = tokens + .windows(5) + .position(|w| w == ["word:check", "(", "word:state", "word:in", "("]); + if let Some(start) = start { + if let Some(end) = tokens[start + 5..].iter().position(|token| token == ")") { + let end = start + 5 + end; + if end >= 2 && tokens[end - 1] == "literal:unknown" && tokens[end - 2] == "," { + tokens.drain(end - 2..end); + } + } + } + } + } + Ok(shape) +} + fn require_shape(conn: &Connection, stage: Stage) -> Result<()> { let references = references()?; let selected = |shape: Shape| -> Shape { @@ -355,8 +511,8 @@ fn require_shape(conn: &Connection, stage: Stage) -> Result<()> { .filter(|(_, object)| references.tables.contains(&object.1)) .collect() }; - let expected = selected(references.stages[&stage].clone()); - let actual = selected(snapshot(conn)?); + let expected = selected(upgraded_reference(conn, references.stages[&stage].clone())?); + let actual = selected(legacy_shape(conn)?); if actual != expected { let changed: BTreeSet<_> = actual .keys() @@ -373,7 +529,7 @@ fn reject_early_tail(conn: &Connection, current: i64) -> Result<()> { return Ok(()); } let references = references()?; - let actual = snapshot(conn)?; + let actual = legacy_shape(conn)?; let base = &references.stages[&Stage::Base]; for (name, object) in &actual { if references.tables.contains(&object.1) && !base.contains_key(name) { diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 452e7e012..c91ce632d 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -129,10 +129,15 @@ pub use ticket::{ }; const OLDEST_SCHEMA_VERSION: i64 = 50; +#[path = "backend_job_recovery_store.rs"] +mod backend_job_recovery_store; +#[path = "backend_job_upgrade.rs"] +pub(crate) mod backend_job_upgrade; #[path = "storage_formats.rs"] mod storage_formats; +pub use backend_job_recovery_store::BackendJobReconciliation; -const LATEST_SCHEMA_VERSION: i64 = 122; +const LATEST_SCHEMA_VERSION: i64 = 123; const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants"; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; @@ -510,6 +515,11 @@ const MIGRATIONS: &[Migration] = &[ name: crate::config_adoption::MIGRATION_NAME, apply: migrate_configuration_adoption_to_v122, }, + Migration { + version: 123, + name: backend_job_upgrade::MIGRATION_NAME, + apply: migrate_backend_job_execution_to_v123, + }, ]; // Both schema-103 identities shipped. Keep the alternate implementation and accept @@ -1961,7 +1971,8 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { ) -> Result; fn get_backend_job(&self, workspace_id: &str, job_id: &str) -> Result>; - /// Pending/unknown Jobs and terminal Jobs with unfinished Worker cleanup + fn backend_job_was_migrated(&self, workspace_id: &str, job_id: &str) -> Result; + /// Pending Jobs and terminal Jobs with unfinished Worker cleanup /// retain their resource. This is a query, not an independent lock ledger. fn find_active_backend_job_for_resource( &self, @@ -1981,6 +1992,70 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { job_id: &str, attempt_id: &str, ) -> Result>; + /// Full ordered execution history for exact-owner release CAS. + fn list_backend_job_attempts( + &self, + workspace_id: &str, + job_id: &str, + ) -> Result>; + fn record_backend_job_reconciliation( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + failure_category: &str, + failure_detail: &str, + next_action: &str, + now: &str, + ) -> Result; + fn probe_backend_job_recovery( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + ) -> Result; + fn record_backend_job_execution_fence( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + controller_evidence: &str, + host_evidence: &str, + now: &str, + ) -> Result; + fn has_backend_job_execution_fence( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + ) -> Result; + /// Host-only saved non-Journal result reconciliation; no model/finalizer rerun. + fn reconcile_backend_job_domain_result( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + submission: &BackendJobResultSubmission, + now: &str, + ) -> Result; + fn get_backend_job_reconciliation( + &self, + workspace_id: &str, + job_id: &str, + attempt_id: &str, + ) -> Result>; + fn claim_backend_job_worker_cleanup_owned( + &self, + workspace_id: &str, + expected_history: &[BackendJobAttemptRecord], + expected: &BackendJobAttemptRecord, + now: &str, + ) -> Result<(BackendJobAttemptRecord, bool)>; + fn finish_backend_job_worker_cleanup_owned( + &self, + workspace_id: &str, + expected_history: &[BackendJobAttemptRecord], + expected: &BackendJobAttemptRecord, + state: BackendJobWorkerCleanupState, + failure_category: Option<&str>, + failure_detail: Option<&str>, + now: &str, + ) -> Result; fn list_recoverable_backend_job_attempts( &self, workspace_id: &str, @@ -2032,8 +2107,8 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { completion: Option<&BackendJobResultSubmission>, now: &str, ) -> Result; - /// Bounded keyset over current Journal Unknown attempts, independent of source liveness. - fn list_backend_journal_unknown_attempts( + /// Bounded keyset over current Journal Pending attempts, independent of source liveness. + fn list_backend_journal_pending_attempts( &self, workspace_id: &str, after_job_id: Option<&str>, @@ -3786,6 +3861,57 @@ impl SqliteWorkspaceStore { )?)) } + /// Read only: recover the original create reservation, never create/restore a + /// Worker. The workspace-store caller must separately check registry Job binding. + pub(crate) fn lookup_backend_job_allocation( + &self, + workspace_id: &str, + job: &BackendJobRecord, + attempt: &BackendJobAttemptRecord, + ) -> Result> { + if job.workspace_id != workspace_id + || attempt.workspace_id != workspace_id + || job.request.job_id != attempt.job_id + || job.current_attempt != attempt.attempt + || job.input_digest != attempt.input_digest + || job.state != BackendJobState::Pending + || attempt.state.terminal() + || attempt.attempt_id + != crate::backend_job::attempt_id(&job.request.job_id, attempt.attempt) + { + return Err(Error::InvalidInput( + "allocation lookup requires the exact current Job intent".into(), + )); + } + job.load_executable_request()?; + let key = crate::backend_job::allocation_key(&attempt.attempt_id); + self.with_conn(|conn| { + let saved = conn.query_row("SELECT worker_id,runtime_id,request_fingerprint,create_fingerprint,memory_language,state,singleton_key,singleton_generation FROM worker_create_reservations WHERE workspace_id=?1 AND allocation_key=?2",params![workspace_id,key],|row|Ok((row.get::<_,String>(0)?,row.get::<_,String>(1)?,row.get::<_,Option>(2)?,row.get::<_,String>(3)?,row.get::<_,Option>(4)?,row.get::<_,String>(5)?,row.get::<_,Option>(6)?,row.get::<_,Option>(7)?))).optional()?; + let Some((worker_id,runtime,request_fingerprint,create_fingerprint,language,state,singleton,generation))=saved else { return Ok(None); }; + if state == "removed" || singleton.is_some() || generation.is_some() { + return Err(Error::RepositoryConflict("original Job allocation is removed or has different create intent".into())); + } + // The trusted reservation fixes the original create intent, including + // its then-selected Profile. Reading that evidence does not resolve + // today's Profile/default or issue any new execution grant. + let request_fingerprint = request_fingerprint.filter(|fingerprint| !fingerprint.is_empty()) + .ok_or_else(|| Error::Store("original Job create fingerprint missing".into()))?; + let snapshot = manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: workspace_id.into(), + language: language.ok_or_else(||Error::Store("Job allocation Memory snapshot missing".into()))?, + }; + validate_workspace_memory_settings_snapshot(&snapshot,workspace_id)?; + if create_fingerprint != bound_worker_create_fingerprint(&request_fingerprint,&snapshot) { + return Err(Error::RepositoryConflict("Job allocation create fingerprint mismatch".into())); + } + let worker_id = worker_id.parse::().map_err(|_|Error::Store("invalid original Job Worker identity".into()))?; + if attempt.worker.as_ref().is_some_and(|worker| worker.runtime_id != runtime || worker.worker_id != worker_id.to_string()) { + return Err(Error::RepositoryConflict("Job allocation differs from saved execution binding".into())); + } + Ok(Some((WorkerCreateReservation{worker_id,create_fingerprint,memory_settings:snapshot,singleton:None},runtime))) + }) + } + pub(crate) fn reserved_worker_create_runtime( &self, workspace_id: &str, @@ -7968,6 +8094,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?; let mut job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; @@ -8030,6 +8157,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore { self.with_conn(|conn| read_backend_job(conn, workspace_id, job_id)) } + fn backend_job_was_migrated(&self, workspace_id: &str, job_id: &str) -> Result { + self.with_conn(|conn| Ok(conn.query_row( + "SELECT EXISTS(SELECT 1 FROM backend_job_upgrade_history WHERE workspace_id=?1 AND job_id=?2)", + params![workspace_id, job_id], |row| row.get(0), + )?)) + } + fn find_active_backend_job_for_resource( &self, workspace_id: &str, @@ -8069,6 +8203,122 @@ impl ControlPlaneStore for SqliteWorkspaceStore { self.with_conn(|conn| read_backend_job_attempt(conn, workspace_id, job_id, attempt_id)) } + fn list_backend_job_attempts( + &self, + workspace_id: &str, + job_id: &str, + ) -> Result> { + self.with_conn(|conn| backend_job_recovery_store::history(conn, workspace_id, job_id)) + } + fn record_backend_job_reconciliation( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + failure_category: &str, + failure_detail: &str, + next_action: &str, + now: &str, + ) -> Result { + backend_job_recovery_store::record( + self, + workspace_id, + expected, + failure_category, + failure_detail, + next_action, + now, + ) + } + fn probe_backend_job_recovery( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + ) -> Result { + backend_job_recovery_store::probe(self, workspace_id, expected) + } + fn record_backend_job_execution_fence( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + controller_evidence: &str, + host_evidence: &str, + now: &str, + ) -> Result { + backend_job_recovery_store::record_fence( + self, + workspace_id, + expected, + controller_evidence, + host_evidence, + now, + ) + } + fn has_backend_job_execution_fence( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + ) -> Result { + self.with_conn(|conn| backend_job_recovery_store::has_fence(conn, workspace_id, expected)) + } + fn reconcile_backend_job_domain_result( + &self, + workspace_id: &str, + expected: &BackendJobAttemptRecord, + submission: &BackendJobResultSubmission, + now: &str, + ) -> Result { + backend_job_recovery_store::reconcile_domain_result( + self, + workspace_id, + expected, + submission, + now, + ) + } + fn get_backend_job_reconciliation( + &self, + workspace_id: &str, + job_id: &str, + attempt_id: &str, + ) -> Result> { + self.with_conn(|conn| conn.query_row("SELECT cause,detail,next_action,updated_at FROM backend_job_reconciliation WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3",params![workspace_id,job_id,attempt_id],|row|Ok(BackendJobReconciliation{cause:row.get(0)?,detail:row.get(1)?,next_action:row.get(2)?,updated_at:row.get(3)?})).optional().map_err(Error::from)) + } + fn claim_backend_job_worker_cleanup_owned( + &self, + workspace_id: &str, + expected_history: &[BackendJobAttemptRecord], + expected: &BackendJobAttemptRecord, + now: &str, + ) -> Result<(BackendJobAttemptRecord, bool)> { + backend_job_recovery_store::claim_cleanup_owned( + self, + workspace_id, + expected_history, + expected, + now, + ) + } + fn finish_backend_job_worker_cleanup_owned( + &self, + workspace_id: &str, + expected_history: &[BackendJobAttemptRecord], + expected: &BackendJobAttemptRecord, + state: BackendJobWorkerCleanupState, + failure_category: Option<&str>, + failure_detail: Option<&str>, + now: &str, + ) -> Result { + backend_job_recovery_store::finish_cleanup_owned( + self, + workspace_id, + expected_history, + expected, + state, + failure_category, + failure_detail, + now, + ) + } fn list_recoverable_backend_job_attempts( &self, workspace_id: &str, @@ -8076,15 +8326,10 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result> { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - // A dispatch claim has no externally meaningful identity until the - // deterministic Worker allocation is bound. Startup can therefore - // replay an interrupted claim through the same allocation key. - tx.execute( - "UPDATE backend_job_attempts SET state = 'reserved' WHERE workspace_id = ?1 AND state = 'dispatching' AND worker_id IS NULL", - params![workspace_id], - )?; + // Preserve the last confirmed dispatch fact. An unbound claim may + // already own an external allocation; the Server reconciles its ID. let mut statement = tx.prepare( - "SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('reserved', 'dispatched') ORDER BY created_at LIMIT ?2", + "SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j USING(workspace_id,job_id) WHERE a.workspace_id = ?1 AND a.state IN ('reserved', 'dispatching', 'dispatched') AND a.attempt=j.current_attempt AND j.state='pending' AND j.input_disposition='available' AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=a.workspace_id AND h.job_id=a.job_id) ORDER BY a.created_at LIMIT ?2", )?; let identities = statement.query_map(params![workspace_id, limit as i64], |row| { Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) @@ -8108,6 +8353,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result<(BackendJobReservation, bool)> { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?; let job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; @@ -8149,7 +8395,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result> { self.with_conn(|conn| { let mut statement = conn.prepare( - "SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='reserved' AND j.input_disposition='available' ORDER BY a.created_at, a.attempt_id LIMIT ?2", + "SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='reserved' AND j.input_disposition='available' AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=a.workspace_id AND h.job_id=a.job_id) ORDER BY a.created_at, a.attempt_id LIMIT ?2", )?; let identities = statement.query_map(params![workspace_id, limit as i64], |row| { Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) @@ -8172,6 +8418,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result<(WorkerRegistryProjectionCommit, bool)> { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?; let job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; @@ -8237,6 +8484,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?; let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?; subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; job.load_executable_request()?; @@ -8323,6 +8571,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; job.load_executable_request()?; + backend_job_recovery_store::require_current_contract(&tx, workspace_id, &submission.job_id)?; if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending { return Err(Error::InvalidInput( "Backend Job attempt no longer accepts results".to_string(), @@ -8349,16 +8598,16 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } - fn list_backend_journal_unknown_attempts( + fn list_backend_journal_pending_attempts( &self, workspace_id: &str, after_job_id: Option<&str>, limit: usize, ) -> Result> { self.with_conn(|conn| { - let mut stmt = conn.prepare("SELECT a.job_id,a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='unknown' AND j.state='unknown' AND a.attempt=j.current_attempt AND json_type(j.request_json,'$.grants.subjektiv_journal')='object' AND (?2 IS NULL OR a.job_id>?2) ORDER BY a.job_id LIMIT ?3")?; + let mut stmt = conn.prepare("SELECT a.job_id,a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='dispatched' AND j.state='pending' AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=a.workspace_id AND h.job_id=a.job_id) AND a.attempt=j.current_attempt AND json_type(j.request_json,'$.grants.subjektiv_journal')='object' AND (?2 IS NULL OR a.job_id>?2) ORDER BY a.job_id LIMIT ?3")?; let ids = stmt.query_map(params![workspace_id,after_job_id,limit.clamp(1,100) as i64], |r| Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::>>()?; - ids.into_iter().map(|(job,id)| read_backend_job_attempt(conn,workspace_id,&job,&id)?.ok_or_else(|| Error::Store("Journal Unknown attempt disappeared".into()))).collect() + ids.into_iter().map(|(job,id)| read_backend_job_attempt(conn,workspace_id,&job,&id)?.ok_or_else(|| Error::Store("Journal Pending attempt disappeared".into()))).collect() }) } @@ -8385,6 +8634,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + backend_job_recovery_store::require_current_contract(&tx, workspace_id, &expected.job_id)?; let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?; subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?; @@ -8392,7 +8642,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { if &attempt != expected || job.current_attempt != attempt.attempt || job.request.grants.subjektiv_journal.is_none() || attempt.input_digest != job.input_digest - || !matches!(job.state, BackendJobState::Unknown | BackendJobState::Completed | BackendJobState::Failed) { + || !matches!(job.state, BackendJobState::Pending | BackendJobState::Completed | BackendJobState::Failed) { return Err(Error::InvalidInput("Journal reconciliation binding changed".into())); } if let Some(submission) = completion { @@ -8401,25 +8651,25 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } let (digest, json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?; if attempt.state == BackendJobAttemptState::Completed && attempt.result_digest.as_deref() == Some(digest.as_str()) { tx.commit()?; return Ok(attempt); } - if attempt.state != BackendJobAttemptState::Unknown || job.state != BackendJobState::Unknown { + if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending { return Err(Error::InvalidInput("Journal completion cannot replace definitive outcome".into())); } - tx.execute("UPDATE backend_job_attempts SET state='completed',result_json=?4,result_digest=?5,failure_category=NULL,failure_detail=NULL,updated_at=?6,completed_at=?6 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='unknown'",params![workspace_id,expected.job_id,expected.attempt_id,json,digest,now])?; - tx.execute("UPDATE backend_jobs SET state='completed',result_json=?3,result_digest=?4,failure_category=NULL,failure_detail=NULL,updated_at=?5,completed_at=?5 WHERE workspace_id=?1 AND job_id=?2 AND state='unknown'",params![workspace_id,expected.job_id,json,digest,now])?; + tx.execute("UPDATE backend_job_attempts SET state='completed',result_json=?4,result_digest=?5,failure_category=NULL,failure_detail=NULL,worker_cleanup_state=COALESCE(worker_cleanup_state,'pending'),worker_cleanup_updated_at=CASE WHEN worker_cleanup_state IS NULL THEN ?6 ELSE worker_cleanup_updated_at END,updated_at=?6,completed_at=?6 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='dispatched'",params![workspace_id,expected.job_id,expected.attempt_id,json,digest,now])?; + tx.execute("UPDATE backend_jobs SET state='completed',result_json=?3,result_digest=?4,failure_category=NULL,failure_detail=NULL,updated_at=?5,completed_at=?5 WHERE workspace_id=?1 AND job_id=?2 AND state='pending'",params![workspace_id,expected.job_id,json,digest,now])?; notification_routing::jobs::capture(&tx,workspace_id,&expected.job_id,&expected.attempt_id)?; } else { if attempt.state == BackendJobAttemptState::Failed && attempt.failure_category.as_deref() == Some("journal_attempt_fenced") { tx.commit()?; return Ok(attempt); } - if attempt.state != BackendJobAttemptState::Unknown || job.state != BackendJobState::Unknown + if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending || attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) || attempt.worker_cleanup_completed_at.is_none() { - return Err(Error::InvalidInput("partial Journal Unknown requires completed canonical cleanup fence".into())); + return Err(Error::InvalidInput("partial Journal Pending requires completed canonical cleanup fence".into())); } let purged = if let Some(worker) = &attempt.worker { tx.query_row("SELECT EXISTS(SELECT 1 FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='succeeded' AND session_disposition='purge')",params![workspace_id,worker.runtime_id,worker.worker_id],|r|r.get::<_,bool>(0))? } else { false }; - if !purged { return Err(Error::InvalidInput("partial Journal Unknown has no committed stop/Purge removal receipt".into())); } - tx.execute("UPDATE backend_job_attempts SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed; retained operations remain valid',updated_at=?4 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='unknown'",params![workspace_id,expected.job_id,expected.attempt_id,now])?; - tx.execute("UPDATE backend_jobs SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed',updated_at=?3 WHERE workspace_id=?1 AND job_id=?2 AND state='unknown'",params![workspace_id,expected.job_id,now])?; + if !purged { return Err(Error::InvalidInput("partial Journal Pending has no committed stop/Purge removal receipt".into())); } + tx.execute("UPDATE backend_job_attempts SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed; retained operations remain valid',updated_at=?4 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='dispatched'",params![workspace_id,expected.job_id,expected.attempt_id,now])?; + tx.execute("UPDATE backend_jobs SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed',updated_at=?3 WHERE workspace_id=?1 AND job_id=?2 AND state='pending'",params![workspace_id,expected.job_id,now])?; } let result = read_backend_job_attempt(&tx,workspace_id,&expected.job_id,&expected.attempt_id)?.ok_or_else(|| Error::Store("Reconciled Journal attempt missing".into()))?; tx.commit()?; Ok(result) @@ -8436,12 +8686,9 @@ impl ControlPlaneStore for SqliteWorkspaceStore { failure_detail: &str, now: &str, ) -> Result { - if !matches!( - state, - BackendJobAttemptState::Failed | BackendJobAttemptState::Unknown - ) { + if !matches!(state, BackendJobAttemptState::Failed) { return Err(Error::InvalidInput( - "Backend Job failure completion requires failed or unknown state".to_string(), + "Backend Job failure completion requires failed state".to_string(), )); } let detail = bounded_failure_detail(failure_detail); @@ -8465,16 +8712,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore { tx.commit()?; return Ok(attempt); } + backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?; tx.execute( - "UPDATE backend_job_attempts SET state = ?4, failure_category = ?5, failure_detail = ?6, worker_cleanup_state = CASE WHEN worker_id IS NULL THEN 'completed' ELSE 'pending' END, worker_cleanup_failure_category = NULL, worker_cleanup_failure_detail = NULL, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = CASE WHEN worker_id IS NULL THEN ?7 ELSE NULL END, completed_at = ?7, updated_at = ?7 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND state IN ('reserved', 'dispatching', 'dispatched')", + "UPDATE backend_job_attempts SET state = ?4, failure_category = ?5, failure_detail = ?6, worker_cleanup_state = CASE WHEN state='reserved' AND worker_id IS NULL THEN 'completed' ELSE 'pending' END, worker_cleanup_failure_category = NULL, worker_cleanup_failure_detail = NULL, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = CASE WHEN state='reserved' AND worker_id IS NULL THEN ?7 ELSE NULL END, allocation_precluded_at = CASE WHEN state='reserved' AND runtime_id IS NULL AND worker_id IS NULL AND runtime_run_id IS NULL AND dispatched_at IS NULL THEN ?7 ELSE NULL END, completed_at = ?7, updated_at = ?7 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND state IN ('reserved', 'dispatching', 'dispatched')", params![workspace_id, job_id, attempt_id, state.as_str(), failure_category, detail, now], )?; if attempt.attempt == job.current_attempt && job.state == BackendJobState::Pending { - let job_state = match state { - BackendJobAttemptState::Failed => BackendJobState::Failed, - BackendJobAttemptState::Unknown => BackendJobState::Unknown, - _ => unreachable!(), - }; + let job_state = BackendJobState::Failed; tx.execute( "UPDATE backend_jobs SET state = ?3, failure_category = ?4, failure_detail = ?5, completed_at = ?6, updated_at = ?6 WHERE workspace_id = ?1 AND job_id = ?2 AND state = 'pending'", params![workspace_id, job_id, job_state.as_str(), failure_category, detail, now], @@ -8490,35 +8734,12 @@ impl ControlPlaneStore for SqliteWorkspaceStore { fn recover_backend_job_worker_cleanups( &self, workspace_id: &str, - now: &str, + _now: &str, limit: usize, ) -> Result> { - self.with_conn_mut(|conn| { - let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - tx.execute( - "UPDATE backend_job_attempts SET worker_cleanup_state = 'pending', worker_cleanup_failure_category = 'cleanup_interrupted', worker_cleanup_failure_detail = 'Backend restart interrupted Worker cleanup; retrying canonical removal', worker_cleanup_updated_at = ?2 WHERE workspace_id = ?1 AND worker_cleanup_state = 'executing'", - params![workspace_id, now], - )?; - let mut statement = tx.prepare( - "SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('completed', 'failed', 'unknown') AND runtime_id IS NOT NULL AND worker_cleanup_state IN ('pending', 'failed') ORDER BY worker_cleanup_updated_at, created_at LIMIT ?2", - )?; - let identities = statement - .query_map(params![workspace_id, limit as i64], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) - })? - .collect::>>()?; - drop(statement); - let attempts = identities - .into_iter() - .map(|(job_id, attempt_id)| { - read_backend_job_attempt(&tx, workspace_id, &job_id, &attempt_id)?.ok_or_else( - || Error::Store("recoverable Backend Job Worker cleanup disappeared".to_string()), - ) - }) - .collect::>>()?; - tx.commit()?; - Ok(attempts) - }) + // Reopening observes retained owners; it does not revoke an executing + // cleanup or authorize a new one by resetting its state. + self.list_backend_job_worker_cleanups(workspace_id, limit) } fn list_backend_job_worker_cleanups( @@ -8528,7 +8749,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result> { self.with_conn(|conn| { let mut statement = conn.prepare( - "SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('completed', 'failed', 'unknown') AND runtime_id IS NOT NULL AND worker_cleanup_state IN ('pending', 'failed') ORDER BY worker_cleanup_updated_at, created_at LIMIT ?2", + "SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j USING(workspace_id,job_id) WHERE a.workspace_id = ?1 AND a.state IN ('completed', 'failed') AND a.runtime_id IS NOT NULL AND (a.worker_cleanup_state IN ('pending', 'failed', 'executing') OR (a.state='completed' AND a.worker_cleanup_state='completed' AND a.attempt=j.current_attempt AND j.input_disposition='available' AND json_type(j.request_json,'$.grants.subjektiv_journal')='object')) AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=a.workspace_id AND h.job_id=a.job_id) ORDER BY a.worker_cleanup_updated_at, a.created_at LIMIT ?2", )?; let identities = statement .query_map(params![workspace_id, limit as i64], |row| { @@ -8555,6 +8776,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result<(BackendJobAttemptRecord, bool)> { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; if !attempt.state.terminal() || attempt.worker.is_none() { @@ -8607,6 +8829,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let failure_detail = failure_detail.map(bounded_failure_detail); self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?; let current = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; if current.worker_cleanup_state == Some(state) { @@ -8620,7 +8843,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } let completed_at = (state == BackendJobWorkerCleanupState::Completed).then_some(now); let changed = tx.execute( - "UPDATE backend_job_attempts SET worker_cleanup_state = ?4, worker_cleanup_failure_category = ?5, worker_cleanup_failure_detail = ?6, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = ?8 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND worker_cleanup_state = 'executing'", + "UPDATE backend_job_attempts SET worker_cleanup_state = ?4, worker_cleanup_failure_category = ?5, worker_cleanup_failure_detail = ?6, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = ?8 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND worker_cleanup_state = 'executing' AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=backend_job_attempts.workspace_id AND h.job_id=backend_job_attempts.job_id)", params![workspace_id, job_id, attempt_id, state.as_str(), failure_category, failure_detail, now, completed_at], )?; if changed != 1 { @@ -12809,7 +13032,7 @@ fn read_active_backend_job_for_resource( // effects block even after Worker removal; cleanup is not outcome resolution. let id = conn.query_row( "SELECT job.job_id FROM backend_jobs job WHERE job.workspace_id = ?1 AND job.resource_key = ?2 - AND (job.state IN ('pending', 'unknown') OR EXISTS ( + AND (job.state='pending' OR EXISTS ( SELECT 1 FROM backend_job_attempts attempt WHERE attempt.workspace_id = job.workspace_id AND attempt.job_id = job.job_id AND attempt.worker_cleanup_state IS NOT 'completed')) @@ -12921,7 +13144,7 @@ fn read_backend_job_attempt( attempt_id: &str, ) -> Result> { let row = conn.query_row( - "SELECT attempt, input_digest, state, runtime_id, worker_id, runtime_run_id, dispatched_at, deadline_at, result_json, result_digest, failure_category, failure_detail, worker_cleanup_state, worker_cleanup_failure_category, worker_cleanup_failure_detail, worker_cleanup_updated_at, worker_cleanup_completed_at, created_at, updated_at, completed_at FROM backend_job_attempts WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3", + "SELECT attempt, input_digest, state, runtime_id, worker_id, runtime_run_id, dispatched_at, deadline_at, result_json, result_digest, failure_category, failure_detail, worker_cleanup_state, worker_cleanup_failure_category, worker_cleanup_failure_detail, worker_cleanup_updated_at, worker_cleanup_completed_at, created_at, updated_at, completed_at, allocation_precluded_at FROM backend_job_attempts WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3", params![workspace_id, job_id, attempt_id], |row| Ok(( row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?, @@ -12930,7 +13153,7 @@ fn read_backend_job_attempt( row.get::<_, Option>(9)?, row.get::<_, Option>(10)?, row.get::<_, Option>(11)?, row.get::<_, Option>(12)?, row.get::<_, Option>(13)?, row.get::<_, Option>(14)?, row.get::<_, Option>(15)?, row.get::<_, Option>(16)?, row.get::<_, String>(17)?, - row.get::<_, String>(18)?, row.get::<_, Option>(19)?, + row.get::<_, String>(18)?, row.get::<_, Option>(19)?, row.get::<_, Option>(20)?, )), ).optional()?; let Some(( @@ -12954,6 +13177,7 @@ fn read_backend_job_attempt( created_at, updated_at, completed_at, + allocation_precluded_at, )) = row else { return Ok(None); @@ -12987,6 +13211,7 @@ fn read_backend_job_attempt( worker, runtime_run_id, dispatched_at, + allocation_precluded_at, deadline_at, result, result_digest, @@ -16074,6 +16299,36 @@ fn migrate_plugin_services_v111_to_v112(conn: &Connection) -> Result<()> { Ok(()) } +fn migrate_backend_job_execution_to_v123(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 122 { + return Err(Error::Store( + "Backend Job execution migration requires schema 122".into(), + )); + } + storage_formats::verify_history(conn, 122)?; + let foreign_keys: bool = conn.query_row("PRAGMA foreign_keys", [], |row| row.get(0))?; + conn.execute_batch("PRAGMA foreign_keys=OFF;")?; + let migration = (|| -> Result<()> { + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + backend_job_upgrade::install(&tx)?; + tx.execute( + "INSERT INTO __yoi_schema_migrations(version,name) VALUES(123,?1)", + [backend_job_upgrade::MIGRATION_NAME], + )?; + storage_formats::verify_history(&tx, 123)?; + tx.commit()?; + Ok(()) + })(); + let restore = conn + .execute_batch(if foreign_keys { + "PRAGMA foreign_keys=ON;" + } else { + "PRAGMA foreign_keys=OFF;" + }) + .map_err(Error::from); + migration.and(restore) +} + fn migrate_configuration_adoption_to_v122(conn: &Connection) -> Result<()> { let current = current_schema_version(conn)?; if !(113..=121).contains(¤t) { @@ -16192,6 +16447,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { create_workspace_schema_through_v113(conn)?; conn.execute_batch(content_history_store::SCHEMA)?; conn.execute_batch(crate::config_adoption::SCHEMA)?; + backend_job_upgrade::install_fresh(conn)?; Ok(()) } @@ -16229,7 +16485,11 @@ fn migrate_plugin_plans_v106_to_v107(conn: &Connection) -> Result<()> { // Fresh composition uses the posts contract. Retained fixtures pass frozen host // and component DDL to the shared cross-domain constraint composition below. fn create_workspace_schema_structure(conn: &Connection) -> Result<()> { - create_workspace_schema_structure_from_sql(conn, include_str!("latest_schema.sql"), true)?; + create_workspace_schema_structure_from_sql( + conn, + &backend_job_upgrade::retained_checks(include_str!("latest_schema.sql")), + true, + )?; conn.execute_batch(include_str!("activation_schema.sql"))?; conn.execute_batch(include_str!("passkey_results_schema.sql"))?; conn.execute_batch(crate::plugin_plan::SCHEMA_SQL)?; @@ -27498,20 +27758,32 @@ INSERT INTO worker_registry ( } #[tokio::test] - async fn backend_job_input_unknown_retention_and_legacy_invalidation_do_not_reinterpret_outcome() + async fn backend_job_input_pending_retention_and_legacy_invalidation_do_not_reinterpret_outcome() { use crate::backend_job::{JobInputDisposition, JobInputRetirementReason}; let store = SqliteWorkspaceStore::in_memory_fixture().unwrap(); seed_resource_job_workspace(&store).await; for (id, state) in [ ("unfinished", None), - ("unknown-input", Some(BackendJobAttemptState::Unknown)), + ("reconciling-input", None), ("failed-input", Some(BackendJobAttemptState::Failed)), ] { let original = resource_job_request(id, id, "candidate"); let reservation = store .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) .unwrap(); + if id == "reconciling-input" { + store + .record_backend_job_reconciliation( + "resource-workspace", + &reservation.attempt, + "receipt_missing", + "acceptance lookup temporarily unavailable", + "reconcile_saved_acceptance", + RESOURCE_JOB_NOW, + ) + .unwrap(); + } if let Some(state) = state { store .finish_backend_job_attempt( @@ -27685,12 +27957,24 @@ INSERT INTO worker_registry ( migrate_job_input_retention_v102_to_v103(&conn).unwrap(); drop(conn); let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let migrated = reopened + .get_backend_job("resource-workspace", "upgrade") + .unwrap() + .unwrap(); + assert_eq!(migrated.state, BackendJobState::Failed); assert_eq!( + migrated.failure_category.as_deref(), + Some("version_upgrade_execution_aborted") + ); + let mut preserved = migrated.clone(); + preserved.state = before.job.state; + preserved.failure_category = before.job.failure_category.clone(); + preserved.failure_detail = before.job.failure_detail.clone(); + assert_eq!(preserved, before.job); + assert!( reopened - .get_backend_job("resource-workspace", "upgrade") + .backend_job_was_migrated("resource-workspace", "upgrade") .unwrap() - .unwrap(), - before.job ); let erased = reopened .reclaim_backend_job_input( @@ -27701,7 +27985,8 @@ INSERT INTO worker_registry ( ) .unwrap(); assert_eq!(erased.input_disposition, JobInputDisposition::Erased); - assert_eq!(erased.state, BackendJobState::Pending); + assert_eq!(erased.state, BackendJobState::Failed); + assert_eq!(erased.failure_category, migrated.failure_category); assert_eq!( reopened .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) @@ -27926,18 +28211,50 @@ INSERT INTO worker_registry ( assert_eq!(second.job.request, request); assert_eq!(second.attempt.attempt, 2); bind_resource_job_worker(&store, &second, "unknown-worker"); - store - .finish_backend_job_attempt( + let expected = store + .get_backend_job_attempt( "resource-workspace", &request.job_id, &second.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "unknown_effects", - "domain outcome unknown", + ) + .unwrap() + .unwrap(); + let pending = store + .record_backend_job_reconciliation( + "resource-workspace", + &expected, + "receipt_unavailable", + "domain completion must be reconciled", + "read_saved_domain_completion", + RESOURCE_JOB_NOW, + ) + .unwrap(); + let history = store + .list_backend_job_attempts("resource-workspace", &request.job_id) + .unwrap(); + let (executing, claimed) = store + .claim_backend_job_worker_cleanup_owned( + "resource-workspace", + &history, + &pending, + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!(claimed); + let history = store + .list_backend_job_attempts("resource-workspace", &request.job_id) + .unwrap(); + store + .finish_backend_job_worker_cleanup_owned( + "resource-workspace", + &history, + &executing, + BackendJobWorkerCleanupState::Completed, + None, + None, RESOURCE_JOB_NOW, ) .unwrap(); - finish_resource_job_cleanup(&store, &second); drop(store); let restarted = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let active = restarted @@ -27947,7 +28264,7 @@ INSERT INTO worker_registry ( ) .unwrap() .unwrap(); - assert_eq!(active.state, BackendJobState::Unknown); + assert_eq!(active.state, BackendJobState::Pending); assert_eq!(active.request, request); assert!( restarted @@ -28946,13 +29263,12 @@ INSERT INTO worker_registry ( .unwrap(); assert_eq!(second.attempt.attempt, 2); store - .finish_backend_job_attempt( + .record_backend_job_reconciliation( workspace_id, - &retry_request.job_id, - &second.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "worker_unknown", - "worker unavailable", + &second.attempt, + "worker_unavailable", + "worker acceptance must be reconciled", + "reconcile_saved_acceptance", "2026-09-01T00:01:05Z", ) .unwrap(); diff --git a/crates/workspace-server/src/subject_deletion_store.rs b/crates/workspace-server/src/subject_deletion_store.rs index 2bbdef024..a1d9a5eb6 100644 --- a/crates/workspace-server/src/subject_deletion_store.rs +++ b/crates/workspace-server/src/subject_deletion_store.rs @@ -872,13 +872,16 @@ mod tests { &crate::backend_job::allocation_key(&second.attempt.attempt_id), None, ); - s.finish_backend_job_attempt( + let current = s + .get_backend_job_attempt("w", "job", &second.attempt.attempt_id) + .unwrap() + .unwrap(); + s.record_backend_job_reconciliation( "w", - "job", - &second.attempt.attempt_id, - BackendJobAttemptState::Unknown, - "unknown", - "unknown outcome", + ¤t, + "submission_receipt_unconfirmed", + "fixture dispatch reply lost", + "Read the exact original allocation receipt", NOW, ) .unwrap(); diff --git a/crates/workspace-server/src/ticket_item_checker.rs b/crates/workspace-server/src/ticket_item_checker.rs index b1f683e77..2087f9764 100644 --- a/crates/workspace-server/src/ticket_item_checker.rs +++ b/crates/workspace-server/src/ticket_item_checker.rs @@ -442,6 +442,7 @@ mod tests { state: BackendJobAttemptState::Completed, worker: Some(RuntimeWorkerRef::new("runtime", "checker")), runtime_run_id: Some("run".into()), + allocation_precluded_at: None, dispatched_at: Some("1".into()), deadline_at: "3".into(), result: Some(result.clone()), diff --git a/tools/state-contract-name-exceptions.json b/tools/state-contract-name-exceptions.json index 25187bccc..218126dcc 100644 --- a/tools/state-contract-name-exceptions.json +++ b/tools/state-contract-name-exceptions.json @@ -1,4 +1,97 @@ [ + { + "path": "crates/standalone/src/job_store/recovery.rs", + "kind": "saved_data", + "reason": "旧standalone Job schema 1/2の実migration fixtureに必要な保存列・JSON key。旧入力tokenの保持と新digestへの既存移行を検証する境界だけに残し、現行実行の状態・CASへ流用しない。", + "lines": [ + { + "sha256": "969de6ed2a496597ba3d80cc06e76d273e73a641e438eee10cb8b708696f29ad", + "count": 1 + }, + { + "sha256": "c69cf30db18c35ff26a854bf47ecd4f67bc70513ab41a7b74b78e232e2e34a68", + "count": 1 + }, + { + "sha256": "f88b9c3ac64944b6e957f9d85a54081925f70cbd2563d936f2a5f9d746093940", + "count": 2 + } + ] + }, + { + "path": "crates/workspace-server/src/backend_job_upgrade.rs", + "kind": "saved_data", + "reason": "schema85以前のJob/全attemptの原保存列をschema86の本文変換前に同transactionで捕捉・検証する読み取り/移行境界。旧tokenを履歴として保ち、raw本文は既Job消去ownerと連動してretireする。現行Job入力digestの別名や新しい実行状態を作らない。", + "lines": [ + { + "sha256": "b80515c50a007075bcde4268cd5362d4f03e8e1b768b3f5d7d9400b408252de1", + "count": 2 + }, + { + "sha256": "3476b390e729656e898805367f93d86368a1660d7fc59142f588767470eb6b4f", + "count": 1 + }, + { + "sha256": "1fe9f975d2ae4ee3186faefdbde6f29a93c43423a1993b7def62cd757c903472", + "count": 1 + }, + { + "sha256": "37c5715b8ac1eb001ebd75d90078751313b1c0bff0717446a4a17481efd7267a", + "count": 1 + }, + { + "sha256": "acf5ef6e7e4fef26678263c18fe4bf29fbac42046dc94cc0a727e94e7942da32", + "count": 1 + }, + { + "sha256": "f121db497084c3889dba8545ee27e0225db94cfde2d662ea922c723a1fea31ff", + "count": 1 + }, + { + "sha256": "656f37d2121e4c564555684926b6917d273d9ca97e9b3112c6733925397410f6", + "count": 1 + }, + { + "sha256": "3265977c091a93b4f70af0c9f93fa19f15b0022ec5c90400b29e885dd4b941e0", + "count": 1 + }, + { + "sha256": "f621d8c70aecdc042ed8dfa68bcf6f7d69b23eb3f2dcc32464dbba25175d5c40", + "count": 1 + } + ] + }, + { + "path": "crates/workspace-server/src/backend_job_upgrade_tests.rs", + "kind": "saved_data", + "reason": "実旧schemaのJob入力列・request JSONと全attempt原値をfixtureに挿入し、正規runner前後の保持・rollback/retry/reopenを比較する保存互換試験。現行producer/実行型には旧keyを導入しない。", + "lines": [ + { + "sha256": "430e40af773fc80bc14662b74fa4cd960aaf30b78b9b1b02be32004ada56d886", + "count": 1 + }, + { + "sha256": "030aed0d240185b411ef3a190dbd718e00df8e91f0ac799ff39fa524a6f704de", + "count": 2 + }, + { + "sha256": "d03c3392bb2443aa9384c3340960c07aacc605af847dc3525ea934d928bc9844", + "count": 1 + }, + { + "sha256": "2cf9adfc96b54eecc5545a04e5d651c8a03f9d78eb1f8bf9dc981a5d91a76e2e", + "count": 1 + }, + { + "sha256": "7253a381f932f6fce62caae83f6c1cc25553dbb662a795638d83f19101790f00", + "count": 1 + }, + { + "sha256": "c72df513101625c8bff4c62b7826fc9f4ef02cb207a6d302fe1ccecb09583ec9", + "count": 1 + } + ] + }, { "path": "crates/subjektiv/src/storage_format_tests.rs", "kind": "saved_data",