Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: checkpoint owned Job recovery and one-time upgrade failure

Browse Source 
Incomplete, unapproved T841 checkpoint. Separate normal same-attempt
reconciliation from the version-upgrade conversion of legacy unfinished
Jobs. Preserve exact allocation/submission identity, immutable input,
independent Controller/Host fences and cleanup ownership; add
same-transaction preallocation rejection evidence for historical consumers.
Journal reconciliation uses persisted operations/checkpoints, never live
input reselection.

Validated root cargo check, 34 focused Server/store/Host tests, 24 upgrade
tests, historical erasure over 100 Jobs, formatting, state-contract names
and diff checks. Full acceptance remains unverified: Executing cleanup
restart without retained Host/removal evidence, frozen-source race,
remaining Journal/NonJournal failures and full crate/generated validation.
Not a completed reviewed handoff or target integration.
This commit is contained in:
Hare
2026-11-04 20:38:33 +09:00
parent c6756bba2e
commit cf2c39f808
53 changed files with 16894 additions and 3457 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/job/src/lib.rs
-2
View File
@@ -191,7 +191,6 @@ pub enum JobState {
Pending,
Completed,
Failed,
Unknown,
Cancelled,
}
@@ -203,7 +202,6 @@ pub enum JobAttemptState {
Dispatched,
Completed,
Failed,
Unknown,
Cancelled,
}
crates/job/tests/contract.rs
+3 -9
View File
@@ -39,7 +39,6 @@ fn shared_lifecycle_enums_round_trip_all_host_states() {
(JobState::Pending, "pending"),
(JobState::Completed, "completed"),
(JobState::Failed, "failed"),
(JobState::Unknown, "unknown"),
(JobState::Cancelled, "cancelled"),
] {
assert_eq!(serde_json::to_value(state).unwrap(), json!(name));
@@ -54,7 +53,6 @@ fn shared_lifecycle_enums_round_trip_all_host_states() {
(JobAttemptState::Dispatched, "dispatched"),
(JobAttemptState::Completed, "completed"),
(JobAttemptState::Failed, "failed"),
(JobAttemptState::Unknown, "unknown"),
(JobAttemptState::Cancelled, "cancelled"),
] {
assert_eq!(serde_json::to_value(state).unwrap(), json!(name));
@@ -63,6 +61,9 @@ fn shared_lifecycle_enums_round_trip_all_host_states() {
state
);
}
// Previous uncertain lifecycle values belong only to versioned migration input.
assert!(serde_json::from_value::<JobState>(json!("unknown")).is_err());
assert!(serde_json::from_value::<JobAttemptState>(json!("unknown")).is_err());
assert!(serde_json::from_value::<JobState>(json!("running")).is_err());
assert!(serde_json::from_value::<JobAttemptState>(json!("idle")).is_err());
}
@@ -111,13 +112,6 @@ fn outcome_is_a_neutral_read_snapshot_with_optional_result_and_failure() {
Some("execution"),
Some("known failure"),
),
(
JobState::Unknown,
JobAttemptState::Unknown,
None,
Some("interrupted"),
Some("requires reconciliation"),
),
(
JobState::Cancelled,
JobAttemptState::Cancelled,
crates/standalone/src/host.rs
+98 -14
View File
@@ -53,6 +53,7 @@ pub struct StandaloneHost {
record: StandaloneWorkerRecord,
lease: Option<StandaloneWorkerLease>,
jobs: crate::jobs::StandaloneJobs,
result_recovery: Option<tokio::task::JoinHandle<Result<(), crate::jobs::JobServiceError>>>,
subject: Option<crate::subjektiv::SubjectConnection>,
subject_release: Option<worker::SubjectReleaseReservation>,
}
@@ -101,7 +102,7 @@ pub enum StandaloneStartupError {
InitialContextPending { worker_id: WorkerId },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
#[derive(Debug, Clone, PartialEq, Eq, Error)]
pub enum StandaloneShutdownError {
#[error("standalone Job execution cleanup or terminal persistence could not be confirmed")]
JobCleanup,
@@ -109,6 +110,10 @@ pub enum StandaloneShutdownError {
DeadlineExceeded,
#[error("the standalone Worker shutdown confirmation was lost")]
ConfirmationLost,
#[error("the standalone Worker Feature cleanup failed")]
FeatureCleanup {
failures: Vec<worker::hook::HookError>,
},
#[error("the standalone Worker final state could not be committed")]
StateStore,
}
@@ -178,7 +183,11 @@ impl StandaloneHost {
.as_ref()
.map(|s| s.jobs_path.clone())
.unwrap_or_else(|| store.jobs_path(worker_id));
let jobs = match crate::jobs::StandaloneJobs::open(&jobs_path, launch.cwd.clone()) {
let jobs = match crate::jobs::StandaloneJobs::open_for_host(
&jobs_path,
launch.cwd.clone(),
allocation.lease(),
) {
Ok(jobs) => jobs,
Err(_) => {
cancel_unpublished_start(&store, &mut subject, allocation)?;
@@ -402,8 +411,12 @@ impl StandaloneHost {
.as_ref()
.map(|s| s.jobs_path.clone())
.unwrap_or_else(|| store.jobs_path(worker_id));
let jobs = crate::jobs::StandaloneJobs::open(&jobs_path, record.cwd.canonical_path.clone())
.map_err(|_| StandaloneStartupError::StateStore)?;
let jobs = crate::jobs::StandaloneJobs::open_for_host(
&jobs_path,
record.cwd.canonical_path.clone(),
&lease,
)
.map_err(|_| StandaloneStartupError::StateStore)?;
if let Some(client) = &model_client {
jobs.bind_model_client(client.clone_boxed());
}
@@ -482,6 +495,13 @@ impl StandaloneHost {
if let Some(connection) = &subject {
connection.host.recover_jobs();
}
let reserved_result_jobs = jobs.reserved_result_job_ids();
let result_jobs = jobs.clone();
let result_recovery = tokio::spawn(async move {
result_jobs
.resume_reserved_result_jobs(reserved_result_jobs?)
.await
});
Self {
handle: started.handle,
shutdown: Some(started.shutdown),
@@ -491,6 +511,7 @@ impl StandaloneHost {
record,
lease: Some(lease),
jobs,
result_recovery: Some(result_recovery),
subject,
subject_release: None,
}
@@ -594,6 +615,7 @@ impl StandaloneHost {
}
pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> {
self.stop_result_recovery().await;
let jobs_result = self.jobs.shutdown().await;
let command = protocol::WorkerCommandEnvelope::new(u64::MAX);
let _ = self.handle.send(Method::Shutdown { command }).await;
@@ -601,16 +623,9 @@ impl StandaloneHost {
self.retain_lease();
return Err(StandaloneShutdownError::ConfirmationLost);
};
match tokio::time::timeout(self.shutdown_timeout, shutdown).await {
Ok(Ok(())) => {}
Ok(Err(_)) => {
self.retain_lease();
return Err(StandaloneShutdownError::ConfirmationLost);
}
Err(_) => {
self.retain_lease();
return Err(StandaloneShutdownError::DeadlineExceeded);
}
if let Err(error) = confirm_controller_shutdown(shutdown, self.shutdown_timeout).await {
self.retain_lease();
return Err(error);
}
if jobs_result.is_err() {
self.retain_lease();
@@ -650,6 +665,16 @@ impl StandaloneHost {
Ok(())
}
async fn stop_result_recovery(&mut self) {
self.jobs.close();
if let Some(task) = self.result_recovery.take() {
// This joins only the dispatcher. Every admitted Worker/Host effect
// remains owned by jobs.shutdown; cancelling a waiter proves no cleanup.
task.abort();
let _ = task.await;
}
}
fn retain_lease(&mut self) {
if let Some(subject) = &mut self.subject {
subject.retain();
@@ -660,9 +685,29 @@ impl StandaloneHost {
}
}
async fn confirm_controller_shutdown(
shutdown: worker::controller::ShutdownReceiver,
timeout: Duration,
) -> Result<(), StandaloneShutdownError> {
let report = tokio::time::timeout(timeout, shutdown)
.await
.map_err(|_| StandaloneShutdownError::DeadlineExceeded)?
.map_err(|_| StandaloneShutdownError::ConfirmationLost)?;
if report.feature_failures.is_empty() {
Ok(())
} else {
Err(StandaloneShutdownError::FeatureCleanup {
failures: report.feature_failures,
})
}
}
impl Drop for StandaloneHost {
fn drop(&mut self) {
self.jobs.close();
if let Some(task) = self.result_recovery.take() {
task.abort();
}
// A dropped Host cannot synchronously confirm either controller or Job
// cleanup. Keep its lease until explicit shutdown or process recovery,
// preventing another Host from reopening a still-live execution store.
@@ -931,6 +976,45 @@ mod local_input_tests {
use super::*;
use protocol::{AuthenticatedInputSource, Method};
// Production report consumer over a real Controller completion channel. The report sender
// is replaced here; actual stopping-hook production is covered by Worker Job tests.
#[tokio::test]
async fn controller_report_preserves_cleanup_failure_instead_of_accepting_scope_return() {
let failure = worker::hook::HookError::new(
worker::hook::HookErrorCategory::Dependency,
"retained Host cleanup",
);
let (sender, receiver) = tokio::sync::oneshot::channel();
sender
.send(worker::controller::ControllerShutdownReport {
feature_failures: vec![failure.clone()],
})
.unwrap();
assert_eq!(
confirm_controller_shutdown(receiver, Duration::from_secs(1)).await,
Err(StandaloneShutdownError::FeatureCleanup {
failures: vec![failure]
})
);
let (sender, receiver) = tokio::sync::oneshot::channel();
sender
.send(worker::controller::ControllerShutdownReport::default())
.unwrap();
assert_eq!(
confirm_controller_shutdown(receiver, Duration::from_secs(1)).await,
Ok(())
);
let (sender, receiver) =
tokio::sync::oneshot::channel::<worker::controller::ControllerShutdownReport>();
drop(sender);
assert_eq!(
confirm_controller_shutdown(receiver, Duration::from_secs(1)).await,
Err(StandaloneShutdownError::ConfirmationLost)
);
}
#[test]
fn local_transport_stamps_operator_without_backend_account() {
let source = AuthenticatedInputSource::Account {
crates/standalone/src/job_store.rs
+372 -146
View File
@@ -6,9 +6,10 @@
//! Job state is not stored in Memory or interactive Sessions.
//!
//! Identical intent under the same ID returns the current snapshot; changes conflict.
//! Only known failures can create new attempts through explicit retry. Reopening preserves
//! Reserved and makes unaccepted Dispatched attempts Unknown. Unknown does not prove no effects;
//! cancellation does not erase that attempt's uncertainty or serialization fence.
//! Version cutover fails only old unfinished work and permanently forbids old Job continuation.
//! Ordinary reopen preserves execution bindings: neither a terminal outcome nor process absence
//! releases serialization/capacity. Retry requires independent owned shutdown evidence; granted
//! Jobs also require authoritative domain reconciliation of the original input and operations.
//!
//! Results commit before releasing the Internal Worker. Replaying the same result is idempotent;
//! different results and stale attempts are rejected. Later cancellation/execution failure does
@@ -25,7 +26,10 @@ use thiserror::Error;
#[path = "job_store/legacy_migrations.rs"]
mod legacy_migrations;
const SCHEMA_VERSION: i64 = 5;
#[path = "job_store/recovery.rs"]
mod recovery;
const SCHEMA_VERSION: i64 = 6;
/// Availability of the immutable input body, independent of execution outcome.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -72,7 +76,6 @@ pub enum JobState {
Pending,
Completed,
Failed,
Unknown,
Cancelled,
}
@@ -83,7 +86,6 @@ pub enum JobAttemptState {
Dispatched,
Completed,
Failed,
Unknown,
Cancelled,
}
@@ -98,7 +100,6 @@ impl JobAttemptState {
Self::Dispatched => "dispatched",
Self::Completed => "completed",
Self::Failed => "failed",
Self::Unknown => "unknown",
Self::Cancelled => "cancelled",
}
}
@@ -117,6 +118,40 @@ pub struct JobSnapshot {
pub attempt: JobAttempt,
pub result: Option<Value>,
pub acknowledged: bool,
/// A version cutover is final even when input/checkpoint remains incomplete.
pub version_stopped: bool,
/// All historical execution owners, independent of Job outcome/checkpoint.
pub executions: Vec<JobExecutionOwnership>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobExecutionOwnership {
pub attempt_id: String,
pub owner_id: String,
pub run_id: String,
pub writes_revoked: bool,
pub cleanup_receipt: Option<String>,
}
impl JobSnapshot {
/// Terminal failure/completion without the attempt's admission binding is not
/// an empty set of successfully joined executions. Keep that missing evidence
/// distinct from an unstarted reservation and from checkpoint coverage.
pub fn executions_released(&self) -> bool {
let needs_binding = matches!(
self.attempt.state,
JobAttemptState::Dispatched | JobAttemptState::Completed | JobAttemptState::Failed
);
(!needs_binding
|| self
.executions
.iter()
.any(|execution| execution.attempt_id == self.attempt.attempt_id))
&& self
.executions
.iter()
.all(|e| e.writes_revoked && e.cleanup_receipt.is_some())
}
}
impl JobSnapshot {
@@ -147,7 +182,6 @@ impl JobSnapshot {
JobState::Pending => job::JobState::Pending,
JobState::Completed => job::JobState::Completed,
JobState::Failed => job::JobState::Failed,
JobState::Unknown => job::JobState::Unknown,
JobState::Cancelled => job::JobState::Cancelled,
},
attempt_state: match self.attempt.state {
@@ -155,14 +189,12 @@ impl JobSnapshot {
JobAttemptState::Dispatched => job::JobAttemptState::Dispatched,
JobAttemptState::Completed => job::JobAttemptState::Completed,
JobAttemptState::Failed => job::JobAttemptState::Failed,
JobAttemptState::Unknown => job::JobAttemptState::Unknown,
JobAttemptState::Cancelled => job::JobAttemptState::Cancelled,
},
result: self.result.clone(),
failure_category: self.attempt.failure.as_ref().map(|_| {
match self.attempt.state {
JobAttemptState::Cancelled => "cancelled",
JobAttemptState::Unknown => "unknown_outcome",
_ => "execution_failed",
}
.to_string()
@@ -216,6 +248,7 @@ pub enum JobStoreError {
pub(crate) struct JobStore {
conn: Connection,
owner_id: String,
}
impl JobStore {
@@ -239,61 +272,12 @@ impl JobStore {
}
let mut conn = Connection::open(path)?;
feature_storage::configure_connection(&conn)?;
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let version: i64 = tx.pragma_query_value(None, "user_version", |row| row.get(0))?;
match version {
0 => {
tx.execute_batch(SCHEMA)?;
tx.execute_batch(GRANT_SCHEMA)?;
tx.execute_batch(RECONCILIATION_SCHEMA)?;
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
1 | 2 => {
if version == 1 {
tx.execute_batch(GRANT_SCHEMA)?;
}
legacy_migrations::migrate_legacy_jobs(&tx)?;
tx.execute_batch(RECONCILIATION_SCHEMA)?;
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
3 => {
tx.execute_batch(RECONCILIATION_SCHEMA)?;
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
4 | SCHEMA_VERSION => {}
other => return Err(JobStoreError::UnsupportedSchema(other)),
}
if version < 5 {
tx.execute_batch(INPUT_RETENTION_SCHEMA)?;
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
if has_legacy_archive {
tx.execute_batch("DROP TRIGGER legacy_job_archive_immutable;
CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END;
CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive
WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available')
OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;")?;
}
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
// An accepted result is the only evidence of success. Reserved attempts
// were never started and remain pending; cancelled attempts never resume.
tx.execute_batch(
"UPDATE job_attempts SET state = 'completed', failure = NULL
WHERE state = 'dispatched' AND result_json IS NOT NULL;
UPDATE job_attempts SET state = 'unknown',
failure = 'Host interrupted before an accepted result'
WHERE state = 'dispatched' AND result_json IS NULL;
UPDATE job_intents SET state = (
SELECT state FROM job_attempts
WHERE job_id = job_intents.job_id AND number = current_attempt)
WHERE state = 'pending' AND EXISTS (
SELECT 1 FROM job_attempts WHERE job_id = job_intents.job_id
AND number = current_attempt AND state IN ('completed', 'unknown'));",
)?;
tx.commit()?;
Ok(Self { conn })
recovery::initialize(&mut conn)?;
// Opening never infers shutdown from a missing task or a process exit.
Ok(Self {
conn,
owner_id: uuid::Uuid::now_v7().to_string(),
})
}
pub(crate) fn reserve(&mut self, request: JobRequest) -> Result<JobSnapshot, JobStoreError> {
@@ -351,9 +335,9 @@ impl JobStore {
Ok(snapshot)
}
/// Domain Host attests that this fenced attempt's only effects are reconciled in its
/// authoritative durable consumer. This releases serialization, not the original Unknown
/// history, and cannot authorize retry or change its result. The service checks cleanup.
/// Domain Host attests that this fenced attempt's effects are reconciled in its
/// authoritative durable consumer. Preserve the original attempt/cause; this receipt
/// neither proves execution shutdown nor permits an old-version Job to continue.
pub(crate) fn reconcile_granted_attempt(
&mut self,
job_id: &str,
@@ -374,11 +358,16 @@ impl JobStore {
let snapshot = require_snapshot(&tx, job_id)?;
check_attempt(&snapshot, attempt_id)?;
if snapshot.domain_grant.is_none()
|| snapshot.attempt.state != JobAttemptState::Unknown
|| !matches!(snapshot.state, JobState::Unknown | JobState::Cancelled)
|| snapshot.attempt.state != JobAttemptState::Failed
|| !matches!(snapshot.state, JobState::Failed | JobState::Cancelled)
{
return Err(JobStoreError::InvalidTransition(
"domain reconciliation requires a fenced unknown granted attempt",
"domain reconciliation requires a failed granted attempt",
));
}
if !snapshot.executions_released() {
return Err(JobStoreError::InvalidTransition(
"execution cleanup is not proven",
));
}
let old: Option<String> = tx.query_row("SELECT domain_receipt FROM job_domain_reconciliations WHERE job_id=?1 AND attempt_id=?2",
@@ -394,14 +383,24 @@ impl JobStore {
Ok(())
}
/// Only definitive failure can create a new immutable attempt. Unknown
/// outcomes require caller reconciliation, never an automatic replay.
/// Only result-only work with independent shutdown proof can use generic retry.
/// Side-effecting domain work must reconcile its authoritative operation receipts.
pub(crate) fn retry(&mut self, job_id: &str) -> Result<JobSnapshot, JobStoreError> {
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let previous = require_snapshot(&tx, job_id)?;
previous.load_executable_request()?;
if previous.version_stopped || !previous.executions_released() {
return Err(JobStoreError::InvalidTransition(
"version stop or unreleased execution forbids retry",
));
}
if previous.domain_grant.is_some() {
return Err(JobStoreError::InvalidTransition(
"domain Jobs require same-Job reconciliation, not generic retry",
));
}
if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed {
return Err(JobStoreError::InvalidTransition(
"retry requires known failure",
@@ -463,7 +462,8 @@ impl JobStore {
return Ok(snapshot);
}
if reason == JobInputRetirementReason::Completed
&& (snapshot.state != JobState::Completed
&& (!snapshot.executions_released()
|| snapshot.state != JobState::Completed
|| snapshot.attempt.state != JobAttemptState::Completed
|| snapshot.result.is_none())
{
@@ -497,6 +497,164 @@ impl JobStore {
Ok(snapshot)
}
/// Settlement belongs to the current attempt and the exact Host connection
/// that dispatched it. Reopen/map absence cannot transfer that authority.
pub(crate) fn check_execution_binding(
&self,
job_id: &str,
attempt_id: &str,
input_digest: &str,
execution: &JobExecutionOwnership,
) -> Result<(), JobStoreError> {
let saved = self.get(job_id)?;
check_attempt(&saved, attempt_id)?;
if saved.input_digest != input_digest {
return Err(JobStoreError::DigestMismatch);
}
if saved.version_stopped
|| execution.attempt_id != attempt_id
|| execution.owner_id != self.owner_id
|| !saved.executions.iter().any(|bound| {
bound.attempt_id == attempt_id
&& bound.owner_id == execution.owner_id
&& bound.run_id == execution.run_id
})
{
return Err(JobStoreError::AttemptMismatch);
}
Ok(())
}
/// Only a bound runner proof can discharge ownership. Terminal state, process
/// absence, or a successful result do not call this method implicitly.
pub(crate) fn confirm_execution_cleanup(
&mut self,
job_id: &str,
execution: &JobExecutionOwnership,
receipt: &str,
) -> Result<(), JobStoreError> {
if execution.owner_id != self.owner_id
|| receipt.is_empty()
|| receipt.len() > 256
|| receipt.chars().any(char::is_control)
{
return Err(JobStoreError::AttemptMismatch);
}
let changed = self.conn.execute("UPDATE job_execution_ownership SET writes_revoked=1,cleanup_receipt=?5 WHERE job_id=?1 AND attempt_id=?2 AND owner_id=?3 AND run_id=?4",
params![job_id,execution.attempt_id,execution.owner_id,execution.run_id,receipt])?;
if changed != 1 {
return Err(JobStoreError::AttemptMismatch);
}
Ok(())
}
pub(crate) fn execution_obligations(
&self,
) -> Result<Vec<JobExecutionOwnership>, JobStoreError> {
// A failed/dispatched row without its admission binding cannot disappear
// from the shutdown barrier merely because there is no ownership row.
let missing: Option<String> = self.conn.query_row(
"SELECT a.attempt_id FROM job_intents i JOIN job_attempts a
ON a.job_id=i.job_id AND a.number=i.current_attempt
WHERE a.state IN ('dispatched','failed','completed')
AND NOT EXISTS(SELECT 1 FROM job_execution_ownership e WHERE e.attempt_id=a.attempt_id)
AND NOT EXISTS(SELECT 1 FROM job_version_stops s WHERE s.job_id=i.job_id)
LIMIT 1", [], |row|row.get(0)).optional()?;
if let Some(attempt) = missing {
return Err(JobStoreError::Corrupt(format!(
"execution admission/cleanup evidence missing for attempt {attempt}"
)));
}
let mut stmt = self.conn.prepare("SELECT attempt_id,owner_id,run_id,writes_revoked,cleanup_receipt FROM job_execution_ownership WHERE cleanup_receipt IS NULL OR writes_revoked=0 ORDER BY rowid")?;
Ok(stmt
.query_map([], |r| {
Ok(JobExecutionOwnership {
attempt_id: r.get(0)?,
owner_id: r.get(1)?,
run_id: r.get(2)?,
writes_revoked: r.get(3)?,
cleanup_receipt: r.get(4)?,
})
})?
.collect::<Result<Vec<_>, _>>()?)
}
/// Same immutable Journal input, bounded new attempt, never a replacement Job.
/// The caller has reconciled/closed the old generation's saved operations.
pub(crate) fn resume_granted(
&mut self,
job_id: &str,
attempt_id: &str,
) -> Result<JobSnapshot, JobStoreError> {
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let saved = require_snapshot(&tx, job_id)?;
check_attempt(&saved, attempt_id)?;
saved.load_executable_request()?;
let reconciled: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM job_domain_reconciliations WHERE job_id=?1 AND attempt_id=?2)", params![job_id,attempt_id], |row|row.get(0))?;
if saved.version_stopped
|| saved.domain_grant.is_none()
|| !saved.executions_released()
|| !reconciled
|| saved.state != JobState::Failed
|| saved.attempt.state != JobAttemptState::Failed
|| saved.attempt.number >= saved.request.limits.max_attempts
{
return Err(JobStoreError::InvalidTransition(
"same-Job recovery requires reconciled execution and available attempt budget",
));
}
let next = saved.attempt.number + 1;
insert_attempt(&tx, &saved.request, next)?;
tx.execute(
"UPDATE job_intents SET state='pending',current_attempt=?2 WHERE job_id=?1",
params![job_id, next],
)?;
let snapshot = require_snapshot(&tx, job_id)?;
tx.commit()?;
Ok(snapshot)
}
/// A domain's saved complete checkpoint may settle the current interrupted
/// attempt without invoking the model or claiming that execution has ended.
pub(crate) fn accept_reconciled(
&mut self,
submission: &JobResultSubmission,
) -> Result<JobSnapshot, JobStoreError> {
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let saved = require_snapshot(&tx, &submission.job_id)?;
check_attempt(&saved, &submission.attempt_id)?;
if saved.version_stopped
|| saved.domain_grant.is_none()
|| saved.state == JobState::Cancelled
{
return Err(JobStoreError::InvalidTransition(
"domain completion is unavailable for this Job",
));
}
if saved.input_digest != submission.input_digest {
return Err(JobStoreError::DigestMismatch);
}
if saved.state == JobState::Completed {
drop(tx);
return self.accept(submission);
}
saved.load_executable_request()?;
let (digest, encoded) =
job::result_digest(&submission.result, saved.request.limits.max_result_bytes)?;
tx.execute("UPDATE job_attempts SET state='completed',result_json=?3,result_digest=?4 WHERE job_id=?1 AND attempt_id=?2",params![submission.job_id,submission.attempt_id,encoded,digest])?;
tx.execute(
"UPDATE job_intents SET state='completed' WHERE job_id=?1",
[&submission.job_id],
)?;
let snapshot = require_snapshot(&tx, &submission.job_id)?;
tx.commit()?;
Ok(snapshot)
}
pub(crate) fn get(&self, job_id: &str) -> Result<JobSnapshot, JobStoreError> {
require_snapshot(&self.conn, job_id)
}
@@ -511,8 +669,8 @@ impl JobStore {
.collect()
}
pub(crate) fn unknown_granted(&self) -> Result<Vec<JobSnapshot>, JobStoreError> {
let mut statement = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_attempts a ON a.job_id=i.job_id AND a.number=i.current_attempt JOIN job_domain_grants g ON g.job_id=i.job_id WHERE a.state='unknown' AND NOT EXISTS (SELECT 1 FROM job_domain_reconciliations r WHERE r.job_id=a.job_id AND r.attempt_id=a.attempt_id) ORDER BY i.rowid")?;
pub(crate) fn unreleased_granted(&self) -> Result<Vec<JobSnapshot>, JobStoreError> {
let mut statement = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_attempts a ON a.job_id=i.job_id AND a.number=i.current_attempt JOIN job_domain_grants g ON g.job_id=i.job_id WHERE (EXISTS(SELECT 1 FROM job_execution_ownership e WHERE e.job_id=i.job_id AND e.cleanup_receipt IS NULL) OR (a.state='failed' AND NOT EXISTS (SELECT 1 FROM job_domain_reconciliations r WHERE r.job_id=a.job_id AND r.attempt_id=a.attempt_id))) ORDER BY i.rowid")?;
let ids = statement
.query_map([], |row| row.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
@@ -526,7 +684,7 @@ impl JobStore {
let mut statement = self.conn.prepare(
"SELECT i.job_id FROM job_intents i JOIN job_attempts a
ON a.job_id = i.job_id AND a.number = i.current_attempt
WHERE i.state = 'pending' AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid",
WHERE i.state = 'pending' AND NOT EXISTS (SELECT 1 FROM job_version_stops s WHERE s.job_id=i.job_id) AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid",
)?;
let ids = statement
.query_map([], |row| row.get::<_, String>(0))?
@@ -550,7 +708,9 @@ impl JobStore {
let snapshot = require_snapshot(&tx, job_id)?;
check_attempt(&snapshot, attempt_id)?;
snapshot.load_executable_request()?;
if snapshot.state != JobState::Pending
if snapshot.version_stopped
|| !snapshot.executions_released()
|| snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Reserved
{
return Err(JobStoreError::InvalidTransition(
@@ -559,9 +719,9 @@ impl JobStore {
}
check_serialization(&tx, snapshot.request.serialization_key.as_deref(), job_id)?;
let (active, active_limit): (i64, Option<u16>) = tx.query_row(
"SELECT COUNT(*), MIN(i.max_concurrent) FROM job_attempts a JOIN job_intents i
ON i.job_id = a.job_id AND i.current_attempt = a.number
WHERE a.state = 'dispatched'",
"SELECT COUNT(*), MIN(i.max_concurrent) FROM job_execution_ownership e JOIN job_intents i
ON i.job_id = e.job_id
WHERE e.writes_revoked=0 OR e.cleanup_receipt IS NULL",
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)?;
@@ -577,6 +737,8 @@ impl JobStore {
"UPDATE job_attempts SET state = 'dispatched' WHERE job_id = ?1 AND attempt_id = ?2",
params![job_id, attempt_id],
)?;
tx.execute("INSERT INTO job_execution_ownership(job_id,attempt_id,owner_id,run_id) VALUES(?1,?2,?3,?4)",
params![job_id,attempt_id,self.owner_id,worker::job::execution_run_id(&snapshot.request, attempt_id)?.to_string()])?;
let snapshot = require_snapshot(&tx, job_id)?;
tx.commit()?;
Ok(snapshot)
@@ -613,9 +775,14 @@ impl JobStore {
snapshot.load_executable_request()?;
if snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Dispatched
|| !snapshot.executions.iter().any(|execution| {
execution.attempt_id == bound.attempt_id
&& execution.owner_id == self.owner_id
&& !execution.writes_revoked
})
{
return Err(JobStoreError::InvalidTransition(
"result requires the active dispatched attempt",
"result requires the active dispatched attempt owned by this Host",
));
}
tx.execute(
@@ -670,20 +837,17 @@ impl JobStore {
Ok(snapshot)
}
/// Cancellation is durable. Cancelling unknown intent must not erase the
/// attempt's uncertain execution evidence or release its serialization fence.
/// A previously accepted result wins and remains success.
/// Cancellation preserves execution ownership and the serialization fence until
/// independent cleanup proof exists. A previously accepted result remains success.
pub(crate) fn cancel(&mut self, job_id: &str) -> Result<JobSnapshot, JobStoreError> {
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let snapshot = require_snapshot(&tx, job_id)?;
if snapshot.state == JobState::Unknown {
tx.execute(
"UPDATE job_intents SET state = 'cancelled' WHERE job_id = ?1",
[job_id],
)?;
} else if !matches!(snapshot.state, JobState::Completed | JobState::Cancelled) {
if !matches!(
snapshot.state,
JobState::Completed | JobState::Cancelled | JobState::Failed
) {
set_terminal(
&tx,
&snapshot,
@@ -732,9 +896,11 @@ fn check_serialization(
let conflicting: bool = conn.query_row(
"SELECT EXISTS(SELECT 1 FROM job_intents i WHERE serialization_key = ?1
AND job_id <> ?2 AND (state = 'pending' OR EXISTS
(SELECT 1 FROM job_attempts a WHERE a.job_id = i.job_id AND a.state = 'unknown'
AND NOT EXISTS (SELECT 1 FROM job_domain_reconciliations r
WHERE r.job_id=a.job_id AND r.attempt_id=a.attempt_id))))",
(SELECT 1 FROM job_execution_ownership e WHERE e.job_id=i.job_id
AND (e.writes_revoked=0 OR e.cleanup_receipt IS NULL))
OR (state='failed' AND EXISTS(SELECT 1 FROM job_domain_grants g WHERE g.job_id=i.job_id)
AND NOT EXISTS(SELECT 1 FROM job_domain_reconciliations r JOIN job_attempts a ON a.attempt_id=r.attempt_id
WHERE r.job_id=i.job_id AND a.number=i.current_attempt))))",
params![key, job_id],
|row| row.get(0),
)?;
@@ -832,7 +998,6 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
"pending" => JobState::Pending,
"completed" => JobState::Completed,
"failed" => JobState::Failed,
"unknown" => JobState::Unknown,
"cancelled" => JobState::Cancelled,
_ => {
return Err(JobStoreError::Corrupt(format!(
@@ -845,7 +1010,6 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
"dispatched" => JobAttemptState::Dispatched,
"completed" => JobAttemptState::Completed,
"failed" => JobAttemptState::Failed,
"unknown" => JobAttemptState::Unknown,
"cancelled" => JobAttemptState::Cancelled,
_ => {
return Err(JobStoreError::Corrupt(format!(
@@ -875,6 +1039,11 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
.transpose()?,
state,
acknowledged,
version_stopped: conn.query_row("SELECT EXISTS(SELECT 1 FROM job_version_stops WHERE job_id=?1)", [job_id], |r|r.get(0))?,
executions: {
let mut stmt = conn.prepare("SELECT attempt_id,owner_id,run_id,writes_revoked,cleanup_receipt FROM job_execution_ownership WHERE job_id=?1 ORDER BY rowid")?;
stmt.query_map([job_id], |r| Ok(JobExecutionOwnership { attempt_id:r.get(0)?,owner_id:r.get(1)?,run_id:r.get(2)?,writes_revoked:r.get(3)?,cleanup_receipt:r.get(4)? }))?.collect::<Result<Vec<_>,_>>()?
},
attempt: JobAttempt {
attempt_id,
number,
@@ -953,8 +1122,10 @@ CREATE TABLE job_intents (
fingerprint TEXT NOT NULL,
serialization_key TEXT,
max_concurrent INTEGER NOT NULL CHECK (max_concurrent BETWEEN 1 AND 65535),
state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown', 'cancelled')),
state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'cancelled')),
current_attempt INTEGER NOT NULL CHECK (current_attempt BETWEEN 1 AND 255),
input_disposition TEXT NOT NULL DEFAULT 'available'
CHECK (input_disposition IN ('available','reclaimed','erased','invalidated')),
acknowledged INTEGER NOT NULL DEFAULT 0 CHECK (acknowledged IN (0, 1)),
CHECK (acknowledged = 0 OR state = 'completed'),
FOREIGN KEY (job_id, current_attempt) REFERENCES job_attempts(job_id, number)
@@ -965,7 +1136,7 @@ CREATE TABLE job_attempts (
number INTEGER NOT NULL CHECK (number BETWEEN 1 AND 255),
attempt_id TEXT NOT NULL UNIQUE,
input_digest TEXT NOT NULL,
state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatched', 'completed', 'failed', 'unknown', 'cancelled')),
state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatched', 'completed', 'failed', 'cancelled')),
result_json TEXT,
result_digest TEXT,
failure TEXT,
@@ -974,10 +1145,10 @@ CREATE TABLE job_attempts (
CHECK (state <> 'completed' OR result_json IS NOT NULL)
);
CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key)
WHERE serialization_key IS NOT NULL AND state IN ('pending', 'unknown');
WHERE serialization_key IS NOT NULL AND state = 'pending';
CREATE INDEX job_attempt_state ON job_attempts(state);
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF
job_id, request_json, fingerprint, serialization_key, max_concurrent ON job_intents
job_id, fingerprint, serialization_key, max_concurrent ON job_intents
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
CREATE TRIGGER job_attempt_immutable BEFORE UPDATE OF
job_id, number, attempt_id, input_digest ON job_attempts
@@ -986,6 +1157,14 @@ CREATE TRIGGER job_result_immutable BEFORE UPDATE OF result_json, result_digest
WHEN OLD.result_json IS NOT NULL AND
(NEW.result_json IS NOT OLD.result_json OR NEW.result_digest IS NOT OLD.result_digest)
BEGIN SELECT RAISE(ABORT, 'immutable accepted Job result'); END;
CREATE TRIGGER job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON job_intents
WHEN (
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
AND json_type(NEW.request_json,'$.input') = 'null'
) IS NOT TRUE
BEGIN SELECT RAISE(ABORT, 'immutable Job input retirement'); END;
";
#[cfg(test)]
@@ -996,7 +1175,7 @@ mod tests {
use std::sync::{Arc, Barrier};
use tempfile::TempDir;
fn request(id: &str) -> JobRequest {
pub(super) fn request(id: &str) -> JobRequest {
JobRequest {
job_id: id.into(),
purpose: "fixture".into(),
@@ -1024,7 +1203,7 @@ mod tests {
.finish(
"first",
&reserved.attempt.attempt_id,
JobAttemptState::Unknown,
JobAttemptState::Failed,
Some("response lost"),
)
.unwrap();
@@ -1039,6 +1218,13 @@ mod tests {
.reconcile_granted_attempt("first", "foreign-attempt", "domain-fenced")
.is_err()
);
assert!(
store
.reconcile_granted_attempt("first", &unknown.attempt.attempt_id, "domain-fenced")
.is_err()
);
record_runner_cleanup(&mut store, "first");
let unknown = store.get("first").unwrap();
store
.reconcile_granted_attempt("first", &unknown.attempt.attempt_id, "domain-fenced")
.unwrap();
@@ -1056,7 +1242,7 @@ mod tests {
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
assert_eq!(reopened.get("first").unwrap(), unknown);
assert!(reopened.unknown_granted().unwrap().is_empty());
assert!(reopened.unreleased_granted().unwrap().is_empty());
assert!(
reopened
.conn
@@ -1067,42 +1253,53 @@ mod tests {
}
#[test]
fn schema_three_migration_preserves_unknown_grant_and_its_serialization_fence() {
let (dir, mut store) = fixture();
fn schema_three_cutover_preserves_grant_history_and_execution_fence() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("jobs.sqlite3");
let mut conn = Connection::open(&path).unwrap();
feature_storage::configure_connection(&conn).unwrap();
let tx = conn.transaction().unwrap();
tx.execute_batch(legacy_migrations::SCHEMA).unwrap();
tx.execute_batch(GRANT_SCHEMA).unwrap();
let mut first = request("first");
first.serialization_key = Some("journal".into());
let reserved = store
.reserve_granted(first, Some(json!({"domain":"journal"})))
.unwrap();
store
.dispatch("first", &reserved.attempt.attempt_id, 8)
.unwrap();
let unknown = store
.finish(
"first",
&reserved.attempt.attempt_id,
JobAttemptState::Unknown,
Some("unconfirmed"),
)
.unwrap();
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
ALTER TABLE job_intents DROP COLUMN input_disposition;
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
assert_eq!(reopened.get("first").unwrap(), unknown);
tx.execute("INSERT INTO job_intents(job_id,request_json,fingerprint,serialization_key,max_concurrent,state,current_attempt) VALUES('first',?1,?2,'journal',8,'unknown',1)", params![serde_json::to_string(&first).unwrap(),first.fingerprint().unwrap()]).unwrap();
tx.execute("INSERT INTO job_attempts(job_id,number,attempt_id,input_digest,state,failure) VALUES('first',1,?1,?2,'unknown','unconfirmed')", params![job::attempt_id("first",1),first.input_digest().unwrap()]).unwrap();
tx.execute(
"INSERT INTO job_domain_grants VALUES('first',?1)",
[serde_json::to_string(&json!({"domain":"journal"})).unwrap()],
)
.unwrap();
tx.pragma_update(None, "user_version", 3).unwrap();
tx.commit().unwrap();
drop(conn);
let mut reopened = JobStore::open(&path).unwrap();
let saved = reopened.get("first").unwrap();
assert_eq!(saved.state, JobState::Failed);
assert_eq!(
saved.attempt.failure.as_deref(),
Some(recovery::UPDATE_INTERRUPTION)
);
assert_eq!(saved.domain_grant, Some(json!({"domain":"journal"})));
assert!(saved.version_stopped);
assert!(!saved.executions_released());
let mut next = request("next");
next.serialization_key = Some("journal".into());
assert!(matches!(
reopened.reserve(next.clone()),
reopened.reserve(next),
Err(JobStoreError::SerializationConflict(_))
));
reopened
.reconcile_granted_attempt("first", &unknown.attempt.attempt_id, "domain-fenced")
.unwrap();
reopened.reserve(next).unwrap();
assert!(
reopened
.reconcile_granted_attempt("first", &saved.attempt.attempt_id, "not-a-stop-proof")
.is_err()
);
assert!(
reopened
.resume_granted("first", &saved.attempt.attempt_id)
.is_err()
);
assert!(reopened.retry("first").is_err());
}
fn fixture() -> (TempDir, JobStore) {
@@ -1120,6 +1317,15 @@ mod tests {
}
}
// Store-level fixture only: the runner/gate proof is exercised by Jobs tests.
fn record_runner_cleanup(store: &mut JobStore, id: &str) {
for execution in store.get(id).unwrap().executions {
store
.confirm_execution_cleanup(id, &execution, "fixture-owned-run-joined")
.unwrap();
}
}
fn dispatched(store: &mut JobStore, request: JobRequest) -> JobSnapshot {
let reserved = store.reserve(request).unwrap();
store
@@ -1139,7 +1345,9 @@ mod tests {
.dispatch("retained", &reserved.attempt.attempt_id, 8)
.unwrap();
let result = submission(&running, json!({"saved":true}));
let completed = store.accept(&result).unwrap();
store.accept(&result).unwrap();
record_runner_cleanup(&mut store, "retained");
let completed = store.get("retained").unwrap();
store.ack("retained").unwrap();
let retired = store
.reclaim_input(
@@ -1219,7 +1427,7 @@ mod tests {
let (_dir, mut store) = fixture();
for (id, state) in [
("pending", None),
("unknown", Some(JobAttemptState::Unknown)),
("unknown", Some(JobAttemptState::Failed)),
("failed", Some(JobAttemptState::Failed)),
] {
let original = request(id);
@@ -1285,7 +1493,8 @@ mod tests {
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
let recovered = reopened.get("erase-running").unwrap();
assert_eq!(recovered.state, JobState::Unknown);
assert_eq!(recovered, erased);
assert!(!recovered.executions_released());
assert_eq!(recovered.input_disposition, JobInputDisposition::Erased);
assert_eq!(recovered.input_digest, running.input_digest);
assert!(reopened.accept(&result).is_err());
@@ -1588,6 +1797,8 @@ mod tests {
)
.unwrap();
assert_eq!(store.reserve(original.clone()).unwrap(), failed);
assert!(store.retry("retry").is_err());
record_runner_cleanup(&mut store, "retry");
let second = store.retry("retry").unwrap();
assert_eq!(second.request, original);
assert_eq!(second.attempt.number, 2);
@@ -1631,6 +1842,7 @@ mod tests {
None,
)
.unwrap();
record_runner_cleanup(&mut store, "retry");
assert!(matches!(
store.retry("retry"),
Err(JobStoreError::InvalidTransition("attempt limit reached"))
@@ -1736,7 +1948,6 @@ mod tests {
.unwrap();
for state in [
JobAttemptState::Failed,
JobAttemptState::Unknown,
JobAttemptState::Cancelled,
JobAttemptState::Completed,
] {
@@ -1803,10 +2014,11 @@ mod tests {
assert_eq!(reopened.pending().unwrap(), vec![pending.clone()]);
assert_eq!(reopened.get("pending").unwrap(), pending);
let unknown = reopened.get("interrupted").unwrap();
assert_eq!(unknown.state, JobState::Unknown);
assert_eq!(unknown.attempt.state, JobAttemptState::Unknown);
assert_eq!(unknown, active);
assert_eq!(unknown.state, JobState::Pending);
assert_eq!(unknown.attempt.state, JobAttemptState::Dispatched);
assert_eq!(unknown.result, None);
assert!(unknown.attempt.failure.is_some());
assert!(!unknown.executions_released());
assert_eq!(reopened.reserve(request("interrupted")).unwrap(), unknown);
assert!(reopened.retry("interrupted").is_err());
assert!(
@@ -1840,10 +2052,11 @@ mod tests {
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
let retained = reopened.get("accepted-fixture").unwrap();
assert_eq!(retained.state, JobState::Completed);
assert_eq!(retained.attempt.state, JobAttemptState::Completed);
assert_eq!(retained.state, JobState::Pending);
assert_eq!(retained.attempt.state, JobAttemptState::Dispatched);
assert_eq!(retained.result, Some(bound.result.clone()));
assert_eq!(reopened.accept(&bound).unwrap(), retained);
assert!(!retained.executions_released());
assert!(reopened.accept(&bound).is_err());
assert!(reopened.pending().unwrap().is_empty());
}
@@ -1911,7 +2124,8 @@ mod tests {
));
let cancelled_unknown = reopened.cancel("first").unwrap();
assert_eq!(cancelled_unknown.state, JobState::Cancelled);
assert_eq!(cancelled_unknown.attempt.state, JobAttemptState::Unknown);
assert_eq!(cancelled_unknown.attempt.state, JobAttemptState::Cancelled);
assert!(!cancelled_unknown.executions_released());
assert!(matches!(
reopened.reserve(second.clone()),
Err(JobStoreError::SerializationConflict(_))
@@ -1971,6 +2185,11 @@ mod tests {
None,
)
.unwrap();
assert!(matches!(
store.dispatch("other", &other.attempt.attempt_id, 8),
Err(JobStoreError::ConcurrentLimit)
));
record_runner_cleanup(&mut store, "strict");
store
.dispatch("other", &other.attempt.attempt_id, 8)
.unwrap();
@@ -1989,6 +2208,11 @@ mod tests {
JobAttemptState::Reserved
);
store.accept(&submission(&other, json!({}))).unwrap();
assert!(matches!(
store.dispatch("host-bound", &host_bound.attempt.attempt_id, 1),
Err(JobStoreError::ConcurrentLimit)
));
record_runner_cleanup(&mut store, "other");
store
.dispatch("host-bound", &host_bound.attempt.attempt_id, 1)
.unwrap();
@@ -2105,7 +2329,7 @@ mod tests {
.finish(
"failure",
&reserved.attempt.attempt_id,
JobAttemptState::Unknown,
JobAttemptState::Cancelled,
None
)
.is_err()
@@ -2122,8 +2346,8 @@ mod tests {
);
assert!(store.accept(&submission(&reserved, json!({}))).is_err());
assert!(store.ack("failure").is_err());
assert_eq!(store.cancel("failure").unwrap().state, JobState::Cancelled);
assert!(store.retry("failure").is_err());
assert_eq!(store.cancel("failure").unwrap(), failed);
assert_eq!(store.retry("failure").unwrap().attempt.number, 2);
}
#[test]
@@ -2176,6 +2400,8 @@ mod tests {
Some("known failure"),
)
.unwrap();
record_runner_cleanup(&mut store, "atomic");
let failed = store.get("atomic").unwrap();
store
.conn
.execute_batch(
crates/standalone/src/job_store/legacy_migrations.rs
+50 -2
View File
@@ -1,5 +1,48 @@
use super::*;
// Frozen pre-cutover schema, including the retired persisted state spelling.
pub(super) const SCHEMA: &str = "
CREATE TABLE job_intents (
job_id TEXT PRIMARY KEY NOT NULL,
request_json TEXT NOT NULL,
fingerprint TEXT NOT NULL,
serialization_key TEXT,
max_concurrent INTEGER NOT NULL CHECK (max_concurrent BETWEEN 1 AND 65535),
state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown', 'cancelled')),
current_attempt INTEGER NOT NULL CHECK (current_attempt BETWEEN 1 AND 255),
acknowledged INTEGER NOT NULL DEFAULT 0 CHECK (acknowledged IN (0, 1)),
CHECK (acknowledged = 0 OR state = 'completed'),
FOREIGN KEY (job_id, current_attempt) REFERENCES job_attempts(job_id, number)
DEFERRABLE INITIALLY DEFERRED
);
CREATE TABLE job_attempts (
job_id TEXT NOT NULL REFERENCES job_intents(job_id),
number INTEGER NOT NULL CHECK (number BETWEEN 1 AND 255),
attempt_id TEXT NOT NULL UNIQUE,
input_digest TEXT NOT NULL,
state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatched', 'completed', 'failed', 'unknown', 'cancelled')),
result_json TEXT,
result_digest TEXT,
failure TEXT,
PRIMARY KEY (job_id, number),
CHECK ((result_json IS NULL) = (result_digest IS NULL)),
CHECK (state <> 'completed' OR result_json IS NOT NULL)
);
CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key)
WHERE serialization_key IS NOT NULL AND state IN ('pending', 'unknown');
CREATE INDEX job_attempt_state ON job_attempts(state);
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF
job_id, request_json, fingerprint, serialization_key, max_concurrent ON job_intents
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
CREATE TRIGGER job_attempt_immutable BEFORE UPDATE OF
job_id, number, attempt_id, input_digest ON job_attempts
BEGIN SELECT RAISE(ABORT, 'immutable Job attempt identity'); END;
CREATE TRIGGER job_result_immutable BEFORE UPDATE OF result_json, result_digest ON job_attempts
WHEN OLD.result_json IS NOT NULL AND
(NEW.result_json IS NOT OLD.result_json OR NEW.result_digest IS NOT OLD.result_digest)
BEGIN SELECT RAISE(ABORT, 'immutable accepted Job result'); END;
";
/// Frozen schema-1/2 adapter. Preserve original intent and attempt bindings
/// before replacing counters with identities derived from immutable input.
pub(super) fn migrate_legacy_jobs(conn: &Connection) -> Result<(), JobStoreError> {
@@ -244,10 +287,15 @@ mod tests {
.collect::<Result<_, _>>()
.unwrap();
assert_eq!(digests, vec![request("done").input_digest().unwrap(); 2]);
assert_eq!(store.get("running").unwrap().state, JobState::Unknown);
assert_eq!(store.get("running").unwrap().state, JobState::Failed);
assert_eq!(
store.get("running").unwrap().attempt.failure.as_deref(),
Some(recovery::UPDATE_INTERRUPTION)
);
assert!(store.get("running").unwrap().version_stopped);
assert_eq!(
store.reserve(request("pending")).unwrap().attempt.state,
JobAttemptState::Reserved
JobAttemptState::Failed
);
assert!(
store
crates/standalone/src/job_store/recovery.rs
+1076
View File
File diff suppressed because it is too large Load Diff
crates/standalone/src/jobs.rs
+549 -71
View File
@@ -11,24 +11,27 @@
//! Dropping a wait future does not release Host cancel/cleanup ownership. Consumer acknowledgement
//! is separate from result lifetime; delivery failures never trigger model reexecution.
//!
//! Timeout/shutdown makes unaccepted attempts Unknown before cancellation. Unconfirmed cleanup
//! keeps Host leases held. Clones cannot reopen the store after service closure or carry
//! authority over to another Host.
//! Timeout/shutdown fences unaccepted attempts before cancellation. Only owned runner shutdown
//! plus joined/revoked Host callbacks discharge execution reservations. Clones cannot reopen
//! the store after service closure or carry authority over to another Host.
#[cfg(test)]
mod tests;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
};
use std::time::Duration;
use agen::llm_client::LlmClient;
use job::{JobRequest, JobResultSubmission};
use tokio::sync::watch;
use tokio::sync::{Notify, watch};
use crate::job_store::JobStore;
pub use crate::job_store::{
JobAttempt, JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot,
JobState, JobStoreError,
JobAttempt, JobAttemptState, JobExecutionOwnership, JobInputDisposition,
JobInputRetirementReason, JobSnapshot, JobState, JobStoreError,
};
#[derive(Debug, thiserror::Error)]
@@ -45,12 +48,35 @@ pub enum JobServiceError {
Cleanup(String),
#[error("standalone Job shutdown is incomplete; execution resources are still owned")]
ShutdownIncomplete,
#[error(
"Job settlement storage recovery exhausted after {attempts} tries; execution retained; next action: await a successful write/recovery probe in this Host store, then reconcile the same attempt: {source}"
)]
SettlementRecoveryExhausted {
attempts: usize,
#[source]
source: Box<JobServiceError>,
},
}
pub(crate) trait JobEffectGuard {}
impl<T> JobEffectGuard for T {}
pub(crate) trait JobAttemptFence: Send + Sync {
fn ensure_live(&self) -> Result<(), String>;
fn enter(&self) -> Result<Box<dyn JobEffectGuard + '_>, String> {
self.ensure_live()?;
Ok(Box::new(()))
}
}
pub(crate) trait JobDomainProvider: Send + Sync {
/// Opt in only when every external callback holds `JobAttemptFence::enter`
/// through its complete synchronous effect and no ungated background work exists.
/// The service joins that gate after runner/finalizer shutdown; an ordinary
/// liveness check alone cannot provide this guarantee.
fn callbacks_use_attempt_fence(&self) -> bool {
false
}
fn grant(
&self,
snapshot: &JobSnapshot,
@@ -64,10 +90,21 @@ struct Execution {
attempt_id: String,
resource_key: Option<String>,
max_concurrent: u16,
effect_gate: Arc<Mutex<()>>,
recovery_issue: Arc<Mutex<Option<JobServiceError>>>,
task: tokio::task::JoinHandle<Result<(), JobServiceError>>,
// Retain typed Worker failures and their cleanup resources until the owner can resolve them.
_owned: Arc<
Mutex<
Option<Result<worker::job::OwnedInternalJobExecution, worker::job::JobExecutionError>>,
>,
>,
}
struct Executions {
closed: bool,
// Set only after this service joined owned Tasks, checked durable obligations
// and dropped its stores/adapters. Empty maps or closed slots are not proof.
shutdown_confirmed: bool,
jobs: HashMap<String, Execution>,
}
@@ -90,6 +127,7 @@ pub struct StandaloneJobs {
capacity: u16,
domain: Arc<Mutex<Option<Arc<dyn JobDomainProvider>>>>,
model_client: Arc<Mutex<Option<Box<dyn LlmClient>>>>,
storage_recovered: Arc<Notify>,
}
impl StandaloneJobs {
@@ -111,6 +149,12 @@ impl StandaloneJobs {
"delegated Job execution/cleanup remains running or unconfirmed".into(),
));
}
if !self
.with_store(|store| store.execution_obligations())?
.is_empty()
{
return Err(JobServiceError::ShutdownIncomplete);
}
Ok(QuiescentJobClosure { executions })
}
@@ -119,15 +163,117 @@ impl StandaloneJobs {
store: Arc::new(Mutex::new(Some(JobStore::open(path)?))),
executions: Arc::new(Mutex::new(Executions {
closed: false,
shutdown_confirmed: false,
jobs: HashMap::new(),
})),
cwd,
capacity: job::DEFAULT_MAX_CONCURRENT_JOBS,
domain: Arc::default(),
model_client: Arc::default(),
storage_recovered: Arc::default(),
})
}
/// Production Host entry: only a freshly revalidated Worker lease authorizes
/// restart reconciliation. The lease must remain owned until actual shutdown.
pub(crate) fn open_for_host(
path: &Path,
cwd: PathBuf,
lease: &crate::store::StandaloneWorkerLease,
) -> Result<Self, JobServiceError> {
let owner = lease
.verify_current_id()
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
let jobs = Self::open(path, cwd)?;
jobs.with_store_write(|store| {
store.bind_host_lease(&owner.to_string())?;
// Re-read after opening/migration, immediately before mutating attempts.
let current = lease.verify_current_id().map_err(|_| {
JobStoreError::InvalidTransition("Worker lease ownership changed during Job open")
})?;
if current != owner {
return Err(JobStoreError::AttemptMismatch);
}
store.reconcile_host_restart().map(|_| ())
})?;
Ok(jobs)
}
pub(crate) fn reserved_result_job_ids(&self) -> Result<Vec<String>, JobServiceError> {
// Retain IDs, not a second queue of immutable bodies outside the owned
// execution barrier while awaiting an earlier Job's cleanup.
Ok(self
.pending()?
.into_iter()
.filter(|saved| {
saved.domain_grant.is_none()
&& !saved.version_stopped
&& saved.attempt.state == JobAttemptState::Reserved
})
.map(|saved| saved.request.job_id)
.collect())
}
/// The restore caller captures IDs before returning the Host to clients, then
/// runs this once after binding adapters. Granted Jobs keep domain recovery.
pub(crate) async fn resume_reserved_result_jobs(
&self,
ids: Vec<String>,
) -> Result<(), JobServiceError> {
for id in ids {
self.start(&id).await?;
self.wait(&id).await?;
}
Ok(())
}
/// Storage exhaustion is owned by the same execution Task, not a new pending
/// Job state. The diagnostic retains its typed source until a successful Host
/// store write wakes reconciliation; no polling or model retry is scheduled.
pub fn settlement_recovery_diagnostic(&self, job_id: &str) -> Option<String> {
let executions = self.executions.lock().expect("Job executions poisoned");
let execution = executions.jobs.get(job_id)?;
let issue = execution
.recovery_issue
.lock()
.expect("Job recovery issue poisoned");
issue.as_ref().map(|original| {
let completion = execution.complete.borrow();
match completion.as_ref() {
Some(Err(final_error)) => format!("{original}; settlement ended: {final_error}; next action: resolve the retained typed execution/cleanup failure, not another storage retry"),
_ => original.to_string(),
}
})
}
/// Called by the existing Host/domain recovery hook when storage is available
/// again. A failed probe does not schedule another budget; a successful write
/// wakes only retained settlement Tasks, never starts another execution.
pub(crate) fn recover_storage(&self) -> Result<(), JobServiceError> {
let attempts: Vec<_> = {
let executions = self.executions.lock().expect("Job executions poisoned");
executions
.jobs
.iter()
.filter_map(|(id, execution)| {
if execution.task.is_finished() {
return None;
}
execution
.recovery_issue
.lock()
.expect("Job recovery issue poisoned")
.is_some()
.then(|| (id.clone(), execution.attempt_id.clone()))
})
.collect()
};
for (id, attempt) in attempts {
self.with_store_write(|store| store.probe_settlement_write(&id, &attempt))?;
}
Ok(())
}
pub(crate) fn bind_model_client(&self, client: Box<dyn LlmClient>) {
*self
.model_client
@@ -157,7 +303,7 @@ impl StandaloneJobs {
if executions.closed {
return Err(JobServiceError::Closed);
}
self.with_store(|store| store.reserve_granted(request, Some(grant)))
self.with_store_write(|store| store.reserve_granted(request, Some(grant)))
}
fn with_store<T>(
@@ -168,6 +314,17 @@ impl StandaloneJobs {
Ok(operation(slot.as_mut().ok_or(JobServiceError::Closed)?)?)
}
fn with_store_write<T>(
&self,
operation: impl FnOnce(&mut JobStore) -> Result<T, JobStoreError>,
) -> Result<T, JobServiceError> {
let value = self.with_store(operation)?;
// A successful commit in this same Host/store is a recovery event, unlike
// an arbitrary read, UI click, timer, or a newly issued replacement Job.
self.storage_recovered.notify_waiters();
Ok(value)
}
/// Persist intent without starting a model. Exact resend returns the same
/// current attempt, including after acknowledgement or cancellation.
pub fn request(&self, request: JobRequest) -> Result<JobSnapshot, JobServiceError> {
@@ -175,7 +332,7 @@ impl StandaloneJobs {
if executions.closed {
return Err(JobServiceError::Closed);
}
Ok(self.with_store(|store| store.reserve(request))?)
Ok(self.with_store_write(|store| store.reserve(request))?)
}
/// Enumerate historical domain snapshots for a Session, including retired input receipts.
@@ -205,7 +362,7 @@ impl StandaloneJobs {
{
return Err(JobServiceError::Capacity);
}
self.with_store(|store| store.reclaim_input(job_id, expected_input_digest, reason))
self.with_store_write(|store| store.reclaim_input(job_id, expected_input_digest, reason))
}
pub fn get(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
@@ -222,12 +379,12 @@ impl StandaloneJobs {
self.with_store(|store| store.unacknowledged())
}
pub(crate) fn unknown_granted(&self) -> Result<Vec<JobSnapshot>, JobServiceError> {
self.with_store(|store| store.unknown_granted())
pub(crate) fn unreleased_granted(&self) -> Result<Vec<JobSnapshot>, JobServiceError> {
self.with_store(|store| store.unreleased_granted())
}
/// Release only an Unknown attempt's serialization reservation after the bound domain
/// has reconciled its receipts and durably fenced further effects. Never a generic retry.
/// Release a failed granted attempt's domain reservation only after authoritative
/// reconciliation and independent execution cleanup. Never a generic retry.
pub(crate) fn reconcile_granted_attempt(
&self,
job_id: &str,
@@ -238,29 +395,63 @@ impl StandaloneJobs {
if executions.closed {
return Err(JobServiceError::Closed);
}
if executions.jobs.contains_key(job_id) {
if executions.jobs.get(job_id).is_some_and(|run| {
!run.task.is_finished() || !matches!(run.complete.borrow().as_ref(), Some(Ok(())))
}) {
return Err(JobServiceError::Cleanup(
"domain reconciliation requires observed execution cleanup".into(),
));
}
self.with_store(|store| store.reconcile_granted_attempt(job_id, attempt_id, domain_receipt))
self.with_store_write(|store| {
store.reconcile_granted_attempt(job_id, attempt_id, domain_receipt)
})
}
/// Explicit new attempt, bounded by immutable intent limits. Unknown
/// outcomes cannot be retried without domain reconciliation/new intent.
pub(crate) fn resume_granted(
&self,
job_id: &str,
attempt_id: &str,
) -> Result<JobSnapshot, JobServiceError> {
let executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
}
if executions.jobs.get(job_id).is_some_and(|run| {
!run.task.is_finished() || !matches!(run.complete.borrow().as_ref(), Some(Ok(())))
}) {
return Err(JobServiceError::ShutdownIncomplete);
}
self.with_store_write(|store| store.resume_granted(job_id, attempt_id))
}
pub(crate) fn accept_reconciled(
&self,
submission: &JobResultSubmission,
) -> Result<JobSnapshot, JobServiceError> {
let executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
}
let _effects = executions
.jobs
.get(&submission.job_id)
.map(|run| run.effect_gate.lock().expect("Job effects poisoned"));
self.with_store_write(|store| store.accept_reconciled(submission))
}
/// Explicit new result-only attempt, bounded by immutable intent limits and
/// independent shutdown proof. Granted Jobs use domain-owned recovery instead.
pub fn retry(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
let executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
}
if executions
.jobs
.get(job_id)
.is_some_and(|run| !run.task.is_finished())
{
if executions.jobs.get(job_id).is_some_and(|run| {
!run.task.is_finished() || !matches!(run.complete.borrow().as_ref(), Some(Ok(())))
}) {
return Err(JobServiceError::Capacity);
}
Ok(self.with_store(|store| store.retry(job_id))?)
Ok(self.with_store_write(|store| store.retry(job_id))?)
}
/// Successful model processing and consumer postprocessing have independent
@@ -270,7 +461,7 @@ impl StandaloneJobs {
if executions.closed {
return Err(JobServiceError::Closed);
}
Ok(self.with_store(|store| store.ack(job_id))?)
Ok(self.with_store_write(|store| store.ack(job_id))?)
}
pub fn cancel(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
@@ -278,7 +469,11 @@ impl StandaloneJobs {
if executions.closed {
return Err(JobServiceError::Closed);
}
let snapshot = self.with_store(|store| store.cancel(job_id))?;
let _effect = executions
.jobs
.get(job_id)
.map(|run| run.effect_gate.lock().expect("Job effects poisoned"));
let snapshot = self.with_store_write(|store| store.cancel(job_id))?;
if let Some(execution) = executions.jobs.get(job_id) {
execution.cancel.send_replace(true);
}
@@ -324,15 +519,35 @@ impl StandaloneJobs {
});
let snapshot = self.get(job_id)?;
snapshot.load_executable_request()?;
if snapshot.version_stopped
|| snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Reserved
{
return Err(JobStoreError::InvalidTransition(
"start requires an unstarted current attempt",
)
.into());
}
let effect_gate = Arc::new(Mutex::new(()));
let revoked = Arc::new(AtomicBool::new(false));
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
let deadline = tokio::time::Instant::now() + timeout;
let fence: Arc<dyn JobAttemptFence> = Arc::new(BoundAttemptFence {
revoked: revoked.clone(),
effect_gate: effect_gate.clone(),
store: self.store.clone(),
job_id: job_id.into(),
attempt_id: snapshot.attempt.attempt_id.clone(),
input_digest: snapshot.input_digest.clone(),
deadline,
});
let domain_fenced = snapshot.domain_grant.is_none()
|| self
.domain
.lock()
.expect("Job domain poisoned")
.as_ref()
.is_some_and(|provider| provider.callbacks_use_attempt_fence());
let grant = match (
&snapshot.domain_grant,
self.domain.lock().expect("Job domain poisoned").as_ref(),
@@ -356,10 +571,9 @@ impl StandaloneJobs {
// Count live resources, not only result states: acceptance can precede
// the Internal Worker's cleanup and must not free a concurrency slot.
let live = || {
executions
.jobs
.values()
.filter(|run| !run.task.is_finished())
executions.jobs.values().filter(|run| {
!run.task.is_finished() || !matches!(run.complete.borrow().as_ref(), Some(Ok(())))
})
};
let count = live().count();
let active_limit = live()
@@ -377,28 +591,32 @@ impl StandaloneJobs {
}
let resource_key = snapshot.request.serialization_key.clone();
if let Some(key) = &resource_key {
if executions
.jobs
.values()
.any(|run| !run.task.is_finished() && run.resource_key.as_ref() == Some(key))
{
if executions.jobs.values().any(|run| {
run.resource_key.as_ref() == Some(key)
&& (!run.task.is_finished()
|| !matches!(run.complete.borrow().as_ref(), Some(Ok(()))))
}) {
return Err(JobServiceError::Capacity);
}
}
// Retain errors from finished tasks until a caller observes them rather
// than silently swallowing store/cleanup failures.
if let Some(previous) = executions.jobs.get(job_id) {
if !previous.task.is_finished() {
if !previous.task.is_finished()
|| !matches!(previous.complete.borrow().as_ref(), Some(Ok(())))
{
return Err(JobServiceError::Capacity);
}
}
let snapshot = self.with_store(|store| {
let snapshot = self.with_store_write(|store| {
store.dispatch(job_id, &snapshot.attempt.attempt_id, self.capacity)
})?;
let store = self.store.clone();
let id = job_id.to_string();
let attempt_id = snapshot.attempt.attempt_id.clone();
let sink = Arc::new(BoundResultSink {
storage_recovered: self.storage_recovered.clone(),
revoked: revoked.clone(),
store: store.clone(),
job_id: id.clone(),
attempt_id: attempt_id.clone(),
@@ -408,40 +626,92 @@ impl StandaloneJobs {
let (cancel, mut cancellation) = watch::channel(false);
let (complete_tx, complete) = watch::channel(None);
let execution_attempt = attempt_id.clone();
let execution_gate = effect_gate.clone();
let ownership = snapshot
.executions
.iter()
.find(|run| run.attempt_id == attempt_id)
.cloned()
.ok_or(JobStoreError::AttemptMismatch)?;
let expected_digest = snapshot.input_digest.clone();
let owned = Arc::new(Mutex::new(None));
let retained = owned.clone();
let recovery_issue = Arc::new(Mutex::new(None));
let task_issue = recovery_issue.clone();
let storage_recovered = self.storage_recovered.clone();
let task = tokio::spawn(async move {
let outcome: Result<(), JobServiceError> = async {
let (sender_tx, mut sender_rx) = tokio::sync::oneshot::channel();
let run = prepared.run(attempt_id.clone(), sink, move |sender| { let _ = sender_tx.send(sender); });
let run = prepared.run_owned(attempt_id.clone(), sink, move |sender| {
let _ = sender_tx.send(sender);
});
tokio::pin!(run);
let reason = tokio::select! {
result = &mut run => {
let (state, detail) = match result {
Ok(_) => (JobAttemptState::Failed, "Worker finished without an accepted structured result".to_string()),
Err(error @ worker::job::JobExecutionError::ResultUnconfirmed(_)) => (JobAttemptState::Unknown, error.to_string()),
Err(error) => (JobAttemptState::Failed, error.to_string()),
};
finish_if_pending(&store, &id, &attempt_id, state, &detail)?;
return Ok(());
let mut stop_reason = None;
let result = tokio::select! {
result = &mut run => result,
reason = async {
tokio::select! {
_ = tokio::time::sleep(timeout) => "timeout",
_ = cancellation.wait_for(|cancelled| *cancelled) => "Host cancellation/shutdown",
}
} => {
// The persisted terminal state revokes future domain calls, while the gate
// joins any synchronous call already inside the Host boundary.
{
let _effects = execution_gate.lock().expect("Job effects poisoned");
revoked.store(true, Ordering::Release);
stop_reason = Some(reason);
// A transient store failure does not cancel ownership of settlement.
// The local revocation and effect gate still fence callbacks;
// retry the exact terminal write after the owned runner returns.
let _ = finish_if_pending(&store, &id, &attempt_id, JobAttemptState::Failed, reason);
}
if let Ok(Ok(sender)) = tokio::time::timeout(Duration::from_secs(1), &mut sender_rx).await {
let _ = sender.try_send(());
}
match tokio::time::timeout(Duration::from_secs(2), &mut run).await {
Ok(result) => result,
Err(_) => {
// Report the overdue cancellation, but never drop the owned future.
// A later return can supply real shutdown evidence; until then both
// the task and durable execution reservation remain owned.
eprintln!("standalone Job {id} cancellation is still running; ownership retained");
// Keep waiters attached to this recovery owner. An interim
// cleanup error would stop the Journal dispatcher before a
// later owned return/storage recovery can settle this Job.
run.await
}
}
}
_ = tokio::time::sleep(timeout) => "timeout",
_ = cancellation.wait_for(|cancelled| *cancelled) => "Host cancellation/shutdown",
};
// Fence timeout/shutdown before cancelling so a late tool call can
// never turn an interrupted unaccepted execution into success.
let fence_result = finish_if_pending(&store, &id, &attempt_id, JobAttemptState::Unknown, reason);
// Cancellation is routed through Engine; keep the future alive to
// allow normal Feature cleanup. Hard stop is a fenced unknown
// outcome, never an automatic retry. Result acceptance wins races.
if let Ok(sender) = sender_rx.try_recv() {
let _ = sender.try_send(());
} else if let Ok(Ok(sender)) = tokio::time::timeout(Duration::from_secs(1), &mut sender_rx).await {
let _ = sender.try_send(());
}
let cleanup = tokio::time::timeout(Duration::from_secs(2), &mut run).await;
fence_result?;
if cleanup.is_err() { return Err(JobServiceError::Cleanup("Internal Worker cancellation deadline exceeded; outcome fenced unknown".into())); }
Ok(())
}.await;
let allow_unconfirmed = stop_reason.is_none() && tokio::time::Instant::now() < deadline;
*retained.lock().expect("Job owned execution poisoned") = Some(result);
let outcome = recover_owned_settlement(
&storage_recovered,
&task_issue,
|| {
let _effects = execution_gate.lock().expect("Job effects poisoned");
revoked.store(true, Ordering::Release);
let owned = retained.lock().expect("Job owned execution poisoned");
settle_owned_execution(
&store,
&id,
&attempt_id,
&expected_digest,
&ownership,
domain_fenced,
stop_reason,
allow_unconfirmed,
owned.as_ref().expect("owned execution retained"),
)
},
|| {
let mut slot = store.lock().expect("Job store poisoned");
let store = slot.as_mut().ok_or(JobServiceError::Closed)?;
store.probe_settlement_write(&id, &attempt_id)?;
Ok(())
},
)
.await;
complete_tx.send_replace(Some(
outcome.as_ref().map(|_| ()).map_err(ToString::to_string),
));
@@ -453,6 +723,9 @@ impl StandaloneJobs {
cancel,
complete,
task,
_owned: owned,
effect_gate,
recovery_issue,
attempt_id: execution_attempt,
resource_key,
max_concurrent: snapshot.request.limits.max_concurrent_jobs,
@@ -515,7 +788,11 @@ impl StandaloneJobs {
/// incomplete/error result retains the service resources and must retain the Host lease.
pub(crate) fn try_shutdown(&self) -> Result<(), JobServiceError> {
self.close();
self.recover_storage()?;
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions.shutdown_confirmed {
return Ok(());
}
for execution in executions.jobs.values() {
if !execution.task.is_finished() {
return Err(JobServiceError::ShutdownIncomplete);
@@ -530,6 +807,12 @@ impl StandaloneJobs {
}
}
}
if !self
.with_store(|store| store.execution_obligations())?
.is_empty()
{
return Err(JobServiceError::ShutdownIncomplete);
}
executions.jobs.clear();
// Retained external handles share these slots; they cannot retain a live store,
// provider or model client after successful owner shutdown.
@@ -539,11 +822,13 @@ impl StandaloneJobs {
.lock()
.expect("Job model transport poisoned")
.take();
executions.shutdown_confirmed = true;
Ok(())
}
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
self.close();
self.recover_storage()?;
let completions: Vec<_> = self
.executions
.lock()
@@ -563,13 +848,188 @@ impl StandaloneJobs {
loop {
match self.try_shutdown() {
// Only the task's final return follows its completion notification.
Err(JobServiceError::ShutdownIncomplete) => tokio::task::yield_now().await,
Err(JobServiceError::ShutdownIncomplete) => {
let has_running_task = self
.executions
.lock()
.expect("Job executions poisoned")
.jobs
.values()
.any(|execution| !execution.task.is_finished());
if !has_running_task {
// Saved obligations cannot be discharged by yielding or an absent map.
return Err(JobServiceError::ShutdownIncomplete);
}
tokio::task::yield_now().await;
}
result => return result,
}
}
}
}
// Retry only persistence, never dispatch, domain operations, or Worker shutdown.
// The owning execution Task retains the exact typed result throughout this finite
// recovery window, including when all observers drop their wait futures.
pub(crate) async fn recover_settlement(
mut settle: impl FnMut() -> Result<(), JobServiceError>,
) -> Result<(), JobServiceError> {
const DELAYS_MS: [u64; 5] = [100, 200, 400, 800, 1600];
for pass in 0..=DELAYS_MS.len() {
match settle() {
Ok(()) => return Ok(()),
Err(error) if transient_settlement_storage_error(&error) => {
let Some(delay) = DELAYS_MS.get(pass) else {
return Err(JobServiceError::SettlementRecoveryExhausted {
attempts: pass + 1,
source: Box::new(error),
});
};
tokio::time::sleep(Duration::from_millis(*delay)).await;
}
Err(error) => return Err(error),
}
}
unreachable!("finite settlement retry returns on its final pass")
}
pub(crate) async fn recover_owned_settlement(
storage_recovered: &Notify,
issue: &Mutex<Option<JobServiceError>>,
mut settle: impl FnMut() -> Result<(), JobServiceError>,
mut probe: impl FnMut() -> Result<(), JobServiceError>,
) -> Result<(), JobServiceError> {
loop {
match recover_settlement(&mut settle).await {
Err(error @ JobServiceError::SettlementRecoveryExhausted { .. }) => {
{
let mut saved = issue.lock().expect("Job recovery issue poisoned");
if saved.is_none() {
*saved = Some(error);
}
}
// The execution owner also owns recovery detection; no user action,
// other Job or detached monitor is needed. A timer only schedules an
// exact-owner storage probe, NEVER another settlement budget or effect.
// Only a committed probe permits settlement to resume. Permanent
// errors and stale ownership stop probing with the typed cause intact.
let mut delay = Duration::from_secs(1);
loop {
tokio::select! {
_ = storage_recovered.notified() => {},
_ = tokio::time::sleep(delay) => {},
}
match probe() {
Ok(()) => break,
Err(error) if transient_settlement_storage_error(&error) => {
delay = (delay * 2).min(Duration::from_secs(30));
}
Err(error) => return Err(error),
}
}
}
result => {
if result.is_ok() {
issue.lock().expect("Job recovery issue poisoned").take();
}
return result;
}
}
}
}
fn transient_settlement_storage_error(error: &JobServiceError) -> bool {
matches!(error,
JobServiceError::Store(JobStoreError::Sqlite(rusqlite::Error::SqliteFailure(error, _)))
if matches!(error.code,
rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked
| rusqlite::ErrorCode::SystemIoFailure | rusqlite::ErrorCode::DiskFull
| rusqlite::ErrorCode::CannotOpen))
}
fn settle_owned_execution(
store: &Mutex<Option<JobStore>>,
id: &str,
attempt: &str,
input_digest: &str,
ownership: &JobExecutionOwnership,
domain_fenced: bool,
stop_reason: Option<&str>,
allow_unconfirmed: bool,
result: &Result<worker::job::OwnedInternalJobExecution, worker::job::JobExecutionError>,
) -> Result<(), JobServiceError> {
let detail = match result {
Ok(execution) => execution
.result
.as_ref()
.err()
.map(ToString::to_string)
.unwrap_or_else(|| "Worker finished without an accepted structured result".into()),
Err(error) => error.to_string(),
};
// Validate the saved owner before any terminal write. A stale result/cleanup
// must not finish a later attempt, even if that later attempt is still pending.
let mut slot = store.lock().expect("Job store poisoned");
let store = slot.as_mut().ok_or(JobServiceError::Closed)?;
store.check_execution_binding(id, attempt, input_digest, ownership)?;
let snapshot = store.get(id)?;
if let Ok(execution) = result {
if execution.job_id != id
|| execution.attempt_id != attempt
|| execution.input_digest != input_digest
|| execution.run_id.to_string() != ownership.run_id
{
return Err(JobStoreError::AttemptMismatch.into());
}
let submission = execution
.result
.as_ref()
.ok()
.map(|accepted| &accepted.submission)
.or_else(|| {
(allow_unconfirmed || snapshot.state == JobState::Completed)
.then(|| execution.unconfirmed_submission())
.flatten()
});
if let Some(submission) = submission {
if submission.job_id != id
|| submission.attempt_id != attempt
|| submission.input_digest != input_digest
{
return Err(JobStoreError::AttemptMismatch.into());
}
// Reconcile only the sealed, already-finalized payload. In particular,
// never rerun a non-idempotent domain finalizer after sink response loss.
store.accept(submission)?;
}
}
let snapshot = store.get(id)?;
if snapshot.state == JobState::Pending {
store.finish(
id,
attempt,
JobAttemptState::Failed,
Some(stop_reason.unwrap_or(&detail)),
)?;
}
let execution = result
.as_ref()
.map_err(|error| JobServiceError::Cleanup(error.to_string()))?;
if !domain_fenced
|| !execution.feature_shutdown_complete()
|| execution.worker_failure().is_some()
{
return Err(JobServiceError::Cleanup(format!(
"owned Worker shutdown unconfirmed ({} Feature shutdown failures); typed execution retained",
execution.feature_shutdown_failures().len()
)));
}
// The caller holds the domain effect gate, run_owned has joined/revoked every
// result/finalizer clone, and terminal state rejects all future domain entries.
store.confirm_execution_cleanup(id, ownership, &format!("owned-run:{}", execution.run_id))?;
Ok(())
}
fn finish_if_pending(
store: &Mutex<Option<JobStore>>,
id: &str,
@@ -587,6 +1047,8 @@ fn finish_if_pending(
}
struct BoundResultSink {
storage_recovered: Arc<Notify>,
revoked: Arc<AtomicBool>,
store: Arc<Mutex<Option<JobStore>>>,
job_id: String,
attempt_id: String,
@@ -607,17 +1069,22 @@ impl worker::job::JobResultSink for BoundResultSink {
.map_err(|_| "Job store poisoned".to_string())?;
let store = slot.as_mut().ok_or_else(|| "Job Host closed".to_string())?;
let snapshot = store.get(&self.job_id).map_err(|error| error.to_string())?;
if tokio::time::Instant::now() >= self.deadline && snapshot.state != JobState::Completed {
return Err("Job result deadline expired".into());
if (self.revoked.load(Ordering::Acquire) || tokio::time::Instant::now() >= self.deadline)
&& snapshot.state != JobState::Completed
{
return Err("Job result capability revoked or deadline expired".into());
}
store
.accept(&submission)
.map(|_| ())
.map_err(|error| error.to_string())
.map_err(|error| error.to_string())?;
self.storage_recovered.notify_waiters();
Ok(())
}
}
struct BoundAttemptFence {
revoked: Arc<AtomicBool>,
effect_gate: Arc<Mutex<()>>,
store: Arc<Mutex<Option<JobStore>>>,
job_id: String,
attempt_id: String,
@@ -625,7 +1092,18 @@ struct BoundAttemptFence {
deadline: tokio::time::Instant,
}
impl JobAttemptFence for BoundAttemptFence {
fn enter(&self) -> Result<Box<dyn JobEffectGuard + '_>, String> {
let guard = self
.effect_gate
.lock()
.map_err(|_| "Job effects poisoned".to_string())?;
self.ensure_live()?;
Ok(Box::new(guard))
}
fn ensure_live(&self) -> Result<(), String> {
if self.revoked.load(Ordering::Acquire) {
return Err("Job domain capability revoked".into());
}
if tokio::time::Instant::now() >= self.deadline {
return Err("Job domain deadline expired".into());
}
crates/standalone/src/jobs/tests.rs
+605 -26
View File
@@ -21,6 +21,8 @@ fn request(id: &str) -> JobRequest {
}
fn sink(jobs: &StandaloneJobs, id: &str, attempt: &str) -> BoundResultSink {
BoundResultSink {
storage_recovered: jobs.storage_recovered.clone(),
revoked: Arc::new(AtomicBool::new(false)),
store: jobs.store.clone(),
job_id: id.into(),
attempt_id: attempt.into(),
@@ -75,7 +77,9 @@ impl LlmClient for DomainTestClient {
) -> Result<Pin<Box<dyn Stream<Item = Result<Event, ClientError>> + Send>>, ClientError> {
self.requests.lock().unwrap().push(request);
self.entered.notify_one();
if let Some(release) = &self.failure_release {
if let Some(release) = &self.failure_release
&& self.scripts.lock().unwrap().is_empty()
{
release.notified().await;
return Err(ClientError::Config(
"known scripted provider failure".into(),
@@ -106,6 +110,10 @@ fn domain_request(id: &str) -> JobRequest {
intent
}
impl JobDomainProvider for RecordingDomainProvider {
fn callbacks_use_attempt_fence(&self) -> bool {
true
}
fn grant(
&self,
snapshot: &JobSnapshot,
@@ -133,6 +141,269 @@ async fn await_job(jobs: &StandaloneJobs, id: &str) -> JobSnapshot {
.unwrap()
}
// The model and stopping hook are the only substituted boundaries. Dispatch, result
// persistence, the owned Internal Worker and service cleanup remain production code.
#[derive(Clone, Default)]
struct ControlledShutdown {
entered: Arc<tokio::sync::Notify>,
release: Arc<tokio::sync::Notify>,
block: bool,
fail: bool,
}
impl worker::feature::FeatureModule for ControlledShutdown {
fn descriptor(&self) -> worker::feature::FeatureDescriptor {
worker::feature::FeatureDescriptor::builtin("job-cleanup-test", "Controlled shutdown")
.with_hook(worker::feature::HookDeclaration::new(
"stop",
worker::feature::FeatureHookPoint::WorkerStopping,
))
}
fn install(
&self,
context: &mut worker::feature::FeatureInstallContext<'_>,
) -> Result<(), worker::feature::FeatureInstallError> {
context.hooks().add_worker_stopping(
"stop",
worker::hook::HookExecutionPolicy::fail_closed(),
self.clone(),
)
}
}
#[async_trait]
impl worker::hook::Hook<worker::hook::WorkerStopping> for ControlledShutdown {
async fn call(
&self,
_: &worker::hook::WorkerStoppingContext,
) -> Result<(), worker::hook::HookError> {
self.entered.notify_one();
if self.block {
self.release.notified().await;
}
if self.fail {
Err(worker::hook::HookError::new(
worker::hook::HookErrorCategory::Internal,
"scripted Feature shutdown failure",
))
} else {
Ok(())
}
}
}
impl JobDomainProvider for ControlledShutdown {
fn callbacks_use_attempt_fence(&self) -> bool {
true
}
fn grant(
&self,
_: &JobSnapshot,
_: Arc<dyn JobAttemptFence>,
) -> Result<worker::job::JobFeatureGrant, String> {
Ok(worker::job::JobFeatureGrant {
features: worker::feature::FeatureRegistryBuilder::new().with_module(self.clone()),
satisfied_requirements: vec!["feature.subjektiv"],
..Default::default()
})
}
}
struct ReleaseShutdownOnDrop(Arc<tokio::sync::Notify>);
impl Drop for ReleaseShutdownOnDrop {
fn drop(&mut self) {
self.0.notify_one();
}
}
#[tokio::test]
async fn overdue_timeout_keeps_owned_future_and_capacity_until_feature_shutdown_returns() {
let temp = tempfile::tempdir().unwrap();
let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap();
let hook = ControlledShutdown {
block: true,
..Default::default()
};
let _release_on_failure = ReleaseShutdownOnDrop(hook.release.clone());
jobs.bind_domain(Arc::new(hook.clone())).unwrap();
let mut intent = domain_request("overdue");
intent.limits.timeout_seconds = 1;
intent.limits.max_concurrent_jobs = 1;
intent.serialization_key = Some("owned-resource".into());
let reserved = jobs
.request_granted(intent.clone(), domain_grant("overdue"))
.unwrap();
jobs.request(request("next")).unwrap();
let model = DomainTestClient {
stall: true,
..Default::default()
};
jobs.start_with_model_client("overdue", model.clone())
.await
.unwrap();
await_model(&model).await;
let dispatched = jobs.get("overdue").unwrap();
assert_eq!(dispatched.executions.len(), 1);
let ownership = dispatched.executions[0].clone();
assert_eq!(
ownership.run_id,
worker::job::execution_run_id(&intent, &reserved.attempt.attempt_id)
.unwrap()
.to_string()
);
assert!(!ownership.writes_revoked);
assert!(ownership.cleanup_receipt.is_none());
tokio::time::timeout(Duration::from_secs(5), hook.entered.notified())
.await
.unwrap();
// Grace expiry is diagnostic only: a dropped wait must not detach the owner
// or make the Journal dispatcher abandon the eventual settled result.
assert!(
tokio::time::timeout(Duration::from_secs(3), jobs.wait("overdue"))
.await
.is_err()
);
let mut completion = {
let executions = jobs.executions.lock().unwrap();
let execution = &executions.jobs["overdue"];
assert!(!execution.task.is_finished());
assert!(execution._owned.lock().unwrap().is_none());
execution.complete.clone()
};
let failed = jobs.get("overdue").unwrap();
assert_eq!(failed.state, JobState::Failed);
assert_eq!(failed.attempt.failure.as_deref(), Some("timeout"));
assert_eq!(failed.executions, vec![ownership.clone()]);
assert!(jobs.reserve_quiescent_closure().is_err());
assert!(jobs.retry("overdue").is_err());
assert!(
jobs.resume_granted("overdue", &failed.attempt.attempt_id)
.is_err()
);
assert!(matches!(
jobs.start_with_model_client("next", DomainTestClient::success())
.await,
Err(JobServiceError::Capacity)
));
let mut conflicting = request("conflicting");
conflicting.serialization_key = intent.serialization_key.clone();
assert!(jobs.request(conflicting).is_err());
hook.release.notify_one();
tokio::time::timeout(
Duration::from_secs(5),
completion.wait_for(|state| matches!(state, Some(Ok(())))),
)
.await
.unwrap()
.unwrap();
let cleaned = await_job(&jobs, "overdue").await;
assert!(cleaned.executions_released());
assert_eq!(cleaned.request, reserved.request);
assert_eq!(cleaned.attempt, failed.attempt);
assert_eq!(cleaned.executions[0].owner_id, ownership.owner_id);
assert_eq!(cleaned.executions[0].run_id, ownership.run_id);
assert_eq!(
cleaned.executions[0].cleanup_receipt,
Some(format!("owned-run:{}", ownership.run_id))
);
jobs.start_with_model_client("next", DomainTestClient::success())
.await
.unwrap();
assert_eq!(await_job(&jobs, "next").await.state, JobState::Completed);
jobs.shutdown().await.unwrap();
}
#[tokio::test]
async fn accepted_result_with_feature_shutdown_failure_retains_typed_owner_and_blocks_release() {
let temp = tempfile::tempdir().unwrap();
let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap();
jobs.bind_domain(Arc::new(ControlledShutdown {
fail: true,
..Default::default()
}))
.unwrap();
jobs.request_granted(
domain_request("cleanup-error"),
domain_grant("cleanup-error"),
)
.unwrap();
jobs.start_with_model_client("cleanup-error", DomainTestClient::success())
.await
.unwrap();
assert!(matches!(
tokio::time::timeout(Duration::from_secs(5), jobs.wait("cleanup-error"))
.await
.unwrap(),
Err(JobServiceError::Cleanup(_))
));
let accepted = jobs.get("cleanup-error").unwrap();
assert_eq!(accepted.state, JobState::Completed);
assert_eq!(accepted.result, Some(json!({"valid":true})));
assert!(!accepted.executions_released());
{
let executions = jobs.executions.lock().unwrap();
let owned = executions.jobs["cleanup-error"]._owned.lock().unwrap();
let owned = owned.as_ref().unwrap().as_ref().unwrap();
assert!(!owned.feature_shutdown_complete());
assert!(!owned.feature_shutdown_failures().is_empty());
assert_eq!(owned.run_id.to_string(), accepted.executions[0].run_id);
}
assert!(jobs.reserve_quiescent_closure().is_err());
assert!(
jobs.reclaim_input(
"cleanup-error",
&accepted.input_digest,
JobInputRetirementReason::Completed
)
.is_err()
);
assert!(matches!(
jobs.shutdown().await,
Err(JobServiceError::Cleanup(_))
));
assert!(jobs.store.lock().unwrap().is_some());
assert_eq!(jobs.get("cleanup-error").unwrap(), accepted);
}
#[tokio::test]
async fn accepted_result_with_worker_failure_does_not_confirm_execution_cleanup() {
let temp = tempfile::tempdir().unwrap();
let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap();
jobs.request(request("post-result-error")).unwrap();
let release = Arc::new(tokio::sync::Notify::new());
let model = DomainTestClient {
failure_release: Some(release.clone()),
..DomainTestClient::success()
};
model.scripts.lock().unwrap().pop_back();
release.notify_one();
jobs.start_with_model_client("post-result-error", model)
.await
.unwrap();
assert!(matches!(
tokio::time::timeout(Duration::from_secs(5), jobs.wait("post-result-error"))
.await
.unwrap(),
Err(JobServiceError::Cleanup(_))
));
let accepted = jobs.get("post-result-error").unwrap();
assert_eq!(accepted.state, JobState::Completed);
assert_eq!(accepted.result, Some(json!({"valid":true})));
assert!(!accepted.executions_released());
{
let executions = jobs.executions.lock().unwrap();
let owned = executions.jobs["post-result-error"]._owned.lock().unwrap();
let owned = owned.as_ref().unwrap().as_ref().unwrap();
assert!(owned.feature_shutdown_complete());
assert!(owned.worker_failure().is_some());
assert!(owned.result.is_ok());
}
assert!(jobs.retry("post-result-error").is_err());
assert!(jobs.reserve_quiescent_closure().is_err());
assert!(matches!(
jobs.shutdown().await,
Err(JobServiceError::Cleanup(_))
));
assert!(jobs.store.lock().unwrap().is_some());
}
#[tokio::test]
async fn pending_domain_grants_survive_service_reopen_and_remain_immutable() {
let temp = tempfile::tempdir().unwrap();
@@ -285,12 +556,13 @@ async fn domain_fence_is_live_only_for_current_dispatched_attempt_and_revoked_on
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().to_path_buf()).unwrap();
let provider = Arc::new(RecordingDomainProvider::default());
jobs.bind_domain(provider.clone()).unwrap();
let mut intent = domain_request("fenced");
intent.limits.timeout_seconds = 1;
let reserved = jobs
.request_granted(domain_request("fenced"), domain_grant("fenced"))
.request_granted(intent, domain_grant("fenced"))
.unwrap();
let failure_release = Arc::new(tokio::sync::Notify::new());
let first = DomainTestClient {
failure_release: Some(failure_release.clone()),
stall: true,
..Default::default()
};
jobs.start_with_model_client("fenced", first.clone())
@@ -299,13 +571,27 @@ async fn domain_fence_is_live_only_for_current_dispatched_attempt_and_revoked_on
await_model(&first).await;
let old_fence = provider.0.lock().unwrap()[0].1.clone();
old_fence.ensure_live().unwrap();
failure_release.notify_one();
let failed = await_job(&jobs, "fenced").await;
assert_eq!(failed.state, JobState::Failed);
assert!(failed.result.is_none());
assert!(old_fence.ensure_live().is_err());
// Only known failure permits a retry; cancellation remains terminal.
let retried = jobs.retry("fenced").unwrap();
assert!(failed.executions_released());
assert!(
jobs.retry("fenced").is_err(),
"generic retry cannot bypass the domain"
);
jobs.reconcile_granted_attempt(
"fenced",
&failed.attempt.attempt_id,
"test-domain-reconciled",
)
.unwrap();
let retried = jobs
.resume_granted("fenced", &failed.attempt.attempt_id)
.unwrap();
assert_eq!(retried.request, reserved.request);
assert_eq!(retried.input_digest, reserved.input_digest);
assert_eq!(retried.attempt.number, 2);
assert_ne!(retried.attempt.attempt_id, reserved.attempt.attempt_id);
assert_eq!(retried.domain_grant, reserved.domain_grant);
assert!(old_fence.ensure_live().is_err());
@@ -376,6 +662,263 @@ async fn bound_domain_does_not_change_root_generic_result_only_execution() {
jobs.shutdown().await.unwrap();
}
#[tokio::test]
async fn host_restart_rejects_live_lease_then_fails_only_unconfirmed_dispatch_without_cleanup() {
// Real Worker lease admission and Job SQLite; no Worker/process is spawned
// for the interrupted rows. Releasing a lease is NOT execution drain proof.
use crate::store::{StaleLeasePolicy, StandaloneStoreError, StandaloneWorkerStore};
let temp = tempfile::tempdir().unwrap();
let workers = StandaloneWorkerStore::open(&temp.path().join("state")).unwrap();
let allocation = workers
.allocate(temp.path(), StaleLeasePolicy::Reject)
.unwrap();
let worker = allocation.worker_id();
let path = workers.jobs_path(worker);
let lease = allocation.into_lease();
let prior = StandaloneJobs::open_for_host(&path, temp.path().into(), &lease).unwrap();
let reserved = prior.request(request("interrupted")).unwrap();
let dispatched = prior
.with_store_write(|store| store.dispatch("interrupted", &reserved.attempt.attempt_id, 8))
.unwrap();
let receipt = prior.request(request("accepted")).unwrap();
prior
.with_store_write(|store| store.dispatch("accepted", &receipt.attempt.attempt_id, 8))
.unwrap();
let accepted = prior
.with_store_write(|store| store.accept(&result("accepted", &receipt.attempt.attempt_id)))
.unwrap();
let unstarted = prior.request(request("unstarted")).unwrap();
assert!(matches!(
workers.acquire_lease(worker, StaleLeasePolicy::Recover),
Err(StandaloneStoreError::WorkerLeased(_))
));
assert!(
StandaloneJobs::open_for_host(&path, temp.path().into(), &lease).is_err(),
"same live lease/empty local map is not a restart"
);
assert_eq!(prior.get("interrupted").unwrap(), dispatched);
let old_sink = sink(&prior, "interrupted", &reserved.attempt.attempt_id);
lease.release().unwrap();
let replacement_lease = workers
.acquire_lease(worker, StaleLeasePolicy::Recover)
.unwrap();
let reopened =
StandaloneJobs::open_for_host(&path, temp.path().into(), &replacement_lease).unwrap();
let failed = reopened.get("interrupted").unwrap();
assert_eq!(failed.state, JobState::Failed);
assert_eq!(failed.attempt.state, JobAttemptState::Failed);
assert_eq!(failed.attempt.attempt_id, reserved.attempt.attempt_id);
assert!(
failed
.attempt
.failure
.as_deref()
.unwrap()
.starts_with("host_restart_execution_unconfirmed:")
);
assert!(!failed.version_stopped);
assert_eq!(failed.request, reserved.request);
assert_eq!(failed.executions, dispatched.executions);
assert!(!failed.executions_released());
assert!(
old_sink
.submit(result("interrupted", &reserved.attempt.attempt_id))
.is_err()
);
assert!(reopened.retry("interrupted").is_err());
assert_eq!(reopened.get("accepted").unwrap(), accepted);
assert_eq!(reopened.get("unstarted").unwrap(), unstarted);
reopened.bind_model_client(Box::new(DomainTestClient::success()));
tokio::time::timeout(
Duration::from_secs(5),
reopened.resume_reserved_result_jobs(reopened.reserved_result_job_ids().unwrap()),
)
.await
.unwrap()
.unwrap();
let resumed = reopened.get("unstarted").unwrap();
assert_eq!(resumed.state, JobState::Completed);
assert_eq!(resumed.attempt.attempt_id, unstarted.attempt.attempt_id);
assert_eq!(resumed.attempt.number, 1);
assert!(!resumed.version_stopped);
assert_eq!(reopened.get("interrupted").unwrap(), failed);
assert!(matches!(
reopened.try_shutdown(),
Err(JobServiceError::ShutdownIncomplete)
));
// The fixture never had a real prior runner to join; do not forge its receipt.
}
struct UnconfirmedSink {
bound: BoundResultSink,
commit_before_error: bool,
}
impl JobResultSink for UnconfirmedSink {
fn submit(&self, submission: JobResultSubmission) -> Result<(), String> {
if self.commit_before_error {
self.bound.submit(submission)?;
}
Err("injected result persistence/acknowledgement outage".into())
}
}
#[tokio::test]
async fn sealed_non_journal_result_recovers_store_or_ack_loss_without_running_model_again() {
// Real owned Worker, result Feature and temporary SQLite store; only the
// model and sink acknowledgement boundary are substituted. No process death.
for committed in [false, true] {
let temp = tempfile::tempdir().unwrap();
let jobs =
StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap();
let intent = request("receipt");
let reserved = jobs.request(intent.clone()).unwrap();
let dispatched = jobs
.with_store(|store| store.dispatch("receipt", &reserved.attempt.attempt_id, 8))
.unwrap();
let model = DomainTestClient::success();
let prepared =
worker::job::prepare_job(&intent, temp.path(), Some(Box::new(model.clone()))).unwrap();
let owned = tokio::time::timeout(
Duration::from_secs(5),
prepared.run_owned(
reserved.attempt.attempt_id.clone(),
Arc::new(UnconfirmedSink {
bound: sink(&jobs, "receipt", &reserved.attempt.attempt_id),
commit_before_error: committed,
}),
|_| {},
),
)
.await
.unwrap()
.unwrap();
assert!(matches!(
owned.result,
Err(worker::job::JobExecutionError::ResultUnconfirmed(_))
));
assert!(owned.feature_shutdown_complete());
assert_eq!(
owned.unconfirmed_submission(),
Some(&result("receipt", &reserved.attempt.attempt_id))
);
assert_eq!(
jobs.get("receipt").unwrap().state,
if committed {
JobState::Completed
} else {
JobState::Pending
}
);
let model_calls = model.requests.lock().unwrap().len();
let owned = Ok(owned);
recover_settlement(|| {
settle_owned_execution(
&jobs.store,
"receipt",
&reserved.attempt.attempt_id,
&reserved.input_digest,
&dispatched.executions[0],
true,
None,
true,
&owned,
)
})
.await
.unwrap();
let saved = jobs.get("receipt").unwrap();
assert_eq!(saved.state, JobState::Completed);
assert_eq!(saved.result, Some(json!({"valid":true})));
assert_eq!(saved.attempt.attempt_id, reserved.attempt.attempt_id);
assert_eq!(saved.attempt.number, 1);
assert!(saved.executions_released());
assert_eq!(model.requests.lock().unwrap().len(), model_calls);
jobs.shutdown().await.unwrap();
}
}
#[tokio::test]
async fn stale_settlement_is_not_retried_and_cannot_fail_a_later_attempt() {
let temp = tempfile::tempdir().unwrap();
let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap();
let reserved = jobs.request(request("stale")).unwrap();
let dispatched = jobs
.with_store(|store| store.dispatch("stale", &reserved.attempt.attempt_id, 8))
.unwrap();
let owner = dispatched.executions[0].clone();
jobs.with_store(|store| {
store.finish(
"stale",
&reserved.attempt.attempt_id,
JobAttemptState::Failed,
Some("first failure"),
)?;
store.confirm_execution_cleanup("stale", &owner, "fixture-owned-run-joined")
})
.unwrap();
let next = jobs.retry("stale").unwrap();
let mut calls = 0;
let outcome = recover_settlement(|| {
calls += 1;
settle_owned_execution(
&jobs.store,
"stale",
&reserved.attempt.attempt_id,
&reserved.input_digest,
&owner,
true,
None,
false,
&Err(worker::job::JobExecutionError::MissingResult),
)
})
.await;
assert!(matches!(
outcome,
Err(JobServiceError::Store(JobStoreError::AttemptMismatch))
));
assert_eq!(calls, 1);
assert_eq!(jobs.get("stale").unwrap(), next);
jobs.shutdown().await.unwrap();
}
#[tokio::test]
async fn local_revocation_fences_callbacks_and_results_even_when_terminal_write_is_unavailable() {
let temp = tempfile::tempdir().unwrap();
let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap();
let reserved = jobs.request(request("revoked")).unwrap();
let dispatched = jobs
.with_store(|store| store.dispatch("revoked", &reserved.attempt.attempt_id, 8))
.unwrap();
let revoked = Arc::new(AtomicBool::new(false));
let fence = BoundAttemptFence {
revoked: revoked.clone(),
effect_gate: Arc::new(Mutex::new(())),
store: jobs.store.clone(),
job_id: "revoked".into(),
attempt_id: reserved.attempt.attempt_id.clone(),
input_digest: reserved.input_digest.clone(),
deadline: tokio::time::Instant::now() + Duration::from_secs(10),
};
let mut sink = sink(&jobs, "revoked", &reserved.attempt.attempt_id);
sink.revoked = revoked.clone();
assert!(fence.enter().is_ok());
// Fault boundary: the cancellation's terminal write has not committed.
// No DB terminal state or elapsed deadline is available as revocation proof.
revoked.store(true, Ordering::Release);
assert!(fence.enter().is_err());
assert!(
sink.submit(result("revoked", &reserved.attempt.attempt_id))
.is_err()
);
assert_eq!(jobs.get("revoked").unwrap(), dispatched);
assert!(!dispatched.executions_released());
assert!(matches!(
jobs.shutdown().await,
Err(JobServiceError::ShutdownIncomplete)
));
}
#[tokio::test]
async fn host_result_capability_cannot_cross_job_attempt_digest_or_deadline() {
let temp = tempfile::tempdir().unwrap();
@@ -418,12 +961,13 @@ async fn host_result_capability_cannot_cross_job_attempt_digest_or_deadline() {
changed.result = serde_json::json!({"valid":false});
assert!(expired.submit(changed).is_err());
assert_eq!(jobs.get("second").unwrap().state, JobState::Pending);
jobs.shutdown().await.unwrap();
assert!(
bound
.submit(result("first", &first.attempt.attempt_id))
.is_err()
);
assert!(matches!(
jobs.shutdown().await,
Err(JobServiceError::ShutdownIncomplete)
));
assert!(jobs.store.lock().unwrap().is_some());
assert!(!jobs.get("first").unwrap().executions_released());
assert!(!jobs.get("second").unwrap().executions_released());
}
#[tokio::test]
@@ -437,17 +981,18 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han
&jobs.store,
"cancelled",
&cancelled.attempt.attempt_id,
JobAttemptState::Unknown,
JobAttemptState::Failed,
"cleanup",
)
.unwrap();
assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled);
let accepted = jobs.request(request("accepted")).unwrap();
jobs.with_store(|store| store.dispatch("accepted", &accepted.attempt.attempt_id, 8))
.unwrap();
sink(&jobs, "accepted", &accepted.attempt.attempt_id)
.submit(result("accepted", &accepted.attempt.attempt_id))
jobs.start_with_model_client("accepted", DomainTestClient::success())
.await
.unwrap();
let completed = await_job(&jobs, "accepted").await;
assert!(completed.executions_released());
let bound = sink(&jobs, "accepted", &accepted.attempt.attempt_id);
finish_if_pending(
&jobs.store,
"accepted",
@@ -459,6 +1004,15 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han
assert_eq!(jobs.get("accepted").unwrap().state, JobState::Completed);
jobs.shutdown().await.unwrap();
assert!(jobs.store.lock().unwrap().is_none());
// Repeated close uses the retained proof of this service's real shutdown,
// not an absent store/map as a newly inferred completion.
jobs.try_shutdown().unwrap();
jobs.shutdown().await.unwrap();
assert!(
bound
.submit(result("accepted", &accepted.attempt.attempt_id))
.is_err()
);
for operation in [
jobs.cancel("cancelled"),
jobs.acknowledge("accepted"),
@@ -469,8 +1023,17 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han
}
}
#[test]
fn absent_store_and_empty_execution_map_do_not_create_shutdown_confirmation() {
let temp = tempfile::tempdir().unwrap();
let jobs = StandaloneJobs::open(&temp.path().join("jobs.sqlite3"), temp.path().into()).unwrap();
jobs.close();
jobs.store.lock().unwrap().take();
assert!(matches!(jobs.try_shutdown(), Err(JobServiceError::Closed)));
}
#[tokio::test]
async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() {
async fn cancelled_shutdown_waiter_keeps_execution_barrier_and_failed_input_on_reopen() {
// Actual generic runner + SQLite; only the provider stream is scripted/blocked.
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("jobs.sqlite3");
@@ -505,12 +1068,21 @@ async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() {
.unwrap();
assert!(jobs.store.lock().unwrap().is_none());
let reopened = StandaloneJobs::open(&path, temp.path().into()).unwrap();
let unknown = reopened.get(&reserved.request.job_id).unwrap();
assert_eq!(unknown.attempt.state, JobAttemptState::Unknown);
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
assert_eq!(unknown.request, reserved.request);
assert_eq!(unknown.input_digest, reserved.input_digest);
assert!(reopened.retry(&unknown.request.job_id).is_err());
let stopped = reopened.get(&reserved.request.job_id).unwrap();
assert_eq!(stopped.state, JobState::Failed);
assert_eq!(stopped.attempt.state, JobAttemptState::Failed);
assert_eq!(
stopped.attempt.failure.as_deref(),
Some("Host cancellation/shutdown")
);
assert_eq!(stopped.input_disposition, JobInputDisposition::Available);
assert_eq!(stopped.request, reserved.request);
assert_eq!(stopped.input_digest, reserved.input_digest);
assert!(stopped.executions_released());
assert!(!stopped.version_stopped);
let retry = reopened.retry(&stopped.request.job_id).unwrap();
assert_eq!(retry.attempt.number, 2);
assert_eq!(retry.request, reserved.request);
reopened.shutdown().await.unwrap();
}
@@ -528,6 +1100,8 @@ async fn input_retirement_service_fences_live_attempt_and_preserves_session_rece
jobs.with_store(|store| store.dispatch("input-retirement", &reserved.attempt.attempt_id, 8))
.unwrap();
let fence = BoundAttemptFence {
revoked: Arc::new(AtomicBool::new(false)),
effect_gate: Arc::new(Mutex::new(())),
store: jobs.store.clone(),
job_id: "input-retirement".into(),
attempt_id: reserved.attempt.attempt_id.clone(),
@@ -567,5 +1141,10 @@ async fn input_retirement_service_fences_live_attempt_and_preserves_session_rece
.is_empty()
);
assert_eq!(jobs.request_granted(original, grant).unwrap(), erased);
jobs.shutdown().await.unwrap();
assert!(matches!(
jobs.shutdown().await,
Err(JobServiceError::ShutdownIncomplete)
));
assert!(jobs.store.lock().unwrap().is_some());
assert!(!jobs.get("input-retirement").unwrap().executions_released());
}
crates/standalone/src/store.rs
+70
View File
@@ -523,6 +523,10 @@ impl StandaloneWorkerAllocation {
&self.cwd
}
pub(crate) fn lease(&self) -> &StandaloneWorkerLease {
&self.lease
}
pub fn into_lease(self) -> StandaloneWorkerLease {
self.lease
}
@@ -536,6 +540,30 @@ pub struct StandaloneWorkerLease {
}
impl StandaloneWorkerLease {
/// Revalidate this already-held lease before Job recovery. This neither acquires a lease
/// nor proves any previous execution or Host effect has ended.
pub(crate) fn verify_current_id(&self) -> Result<Uuid, StandaloneStoreError> {
if self.released {
return Err(StandaloneStoreError::LeaseOwnershipLost);
}
let parent = self.path.parent().expect("lease parent");
let _guard = LeaseMutationGuard::acquire(parent)?;
let metadata = fs::symlink_metadata(&self.path).map_err(StandaloneStoreError::Io)?;
if !metadata.is_file() || metadata.file_type().is_symlink() {
return Err(StandaloneStoreError::LeaseOwnershipLost);
}
let bytes = fs::read(&self.path).map_err(StandaloneStoreError::Io)?;
let current: LeaseRecord =
serde_json::from_slice(&bytes).map_err(StandaloneStoreError::Json)?;
if current.lease_id != self.lease_id
|| current.pid != std::process::id()
|| current.liveness() != LeaseLiveness::Live
{
return Err(StandaloneStoreError::LeaseOwnershipLost);
}
Ok(current.lease_id)
}
pub fn release(mut self) -> Result<(), StandaloneStoreError> {
self.release_inner()
}
@@ -900,6 +928,48 @@ permission = "write"
);
}
#[test]
fn lease_verification_is_read_only_and_rejects_replacement_or_changed_process_identity() {
let root = tempfile::tempdir().unwrap();
let store = StandaloneWorkerStore::open(root.path().join("workers")).unwrap();
let allocation = store
.allocate(root.path(), StaleLeasePolicy::Reject)
.unwrap();
let lease = allocation.lease();
let original = fs::read(&lease.path).unwrap();
assert_eq!(lease.verify_current_id().unwrap(), lease.lease_id);
assert_eq!(fs::read(&lease.path).unwrap(), original);
let mut record: LeaseRecord = serde_json::from_slice(&original).unwrap();
record.lease_id = Uuid::now_v7();
let successor = serde_json::to_vec(&record).unwrap();
fs::write(&lease.path, &successor).unwrap();
assert!(matches!(
lease.verify_current_id(),
Err(StandaloneStoreError::LeaseOwnershipLost)
));
assert_eq!(fs::read(&lease.path).unwrap(), successor);
record.lease_id = lease.lease_id;
record.pid = std::process::id().wrapping_add(1);
fs::write(&lease.path, serde_json::to_vec(&record).unwrap()).unwrap();
assert!(matches!(
lease.verify_current_id(),
Err(StandaloneStoreError::LeaseOwnershipLost)
));
record.pid = std::process::id();
record.process_start_marker = record
.process_start_marker
.map(|marker| marker.wrapping_add(1));
fs::write(&lease.path, serde_json::to_vec(&record).unwrap()).unwrap();
assert!(matches!(
lease.verify_current_id(),
Err(StandaloneStoreError::LeaseOwnershipLost)
));
fs::write(&lease.path, original).unwrap();
assert_eq!(lease.verify_current_id().unwrap(), lease.lease_id);
}
#[test]
fn metadata_write_compares_observed_content_even_with_equal_timestamps() {
let root = tempfile::tempdir().unwrap();
crates/standalone/src/subjektiv.rs
+142 -27
View File
@@ -14,8 +14,9 @@
//! The selected Subject's Job store is shared across Workers; the generic Jobs service owns execution
//! and cleanup. Consolidation grants allow only immutable candidate batches and limited operations,
//! not body leases or parent filesystems. Startup pending processing follows Host connections.
//! Unknown attempts are not rerun; delivery failures for accepted results do not trigger model
//! reexecution. Memory application and surface repair use shared-domain CAS/receipts.
//! Execution receipts and domain results are reconciled without blindly replaying effects;
//! delivery failures for accepted results do not trigger model reexecution. Memory application
//! and surface repair use shared-domain CAS/receipts.
mod journal;
pub(crate) use journal::prepare_worker_deletion;
@@ -712,20 +713,135 @@ impl LocalSubjectHost {
}
}
}
if let Ok(pending) = jobs.pending() {
for job in pending {
if job
.domain_grant
.as_ref()
.and_then(|v| serde_json::from_value::<LocalConsolidationGrant>(v.clone()).ok())
.is_some_and(|g| g.binding == self.binding)
{
schedule_job(jobs.clone(), job.request.job_id, self.self_weak.clone());
// Restart reconciliation can leave a failed attempt with a saved domain
// result. Pending alone lists only unstarted reservations, not those cuts.
let candidates = jobs.pending().and_then(|mut pending| {
pending.extend(jobs.unreleased_granted()?);
Ok(pending)
});
match candidates {
Ok(candidates) => {
let mut seen = std::collections::HashSet::new();
for job in candidates {
if job.request.purpose != "subjektiv_consolidation"
|| !seen.insert(job.request.job_id.clone())
|| !job
.domain_grant
.as_ref()
.and_then(|v| {
serde_json::from_value::<LocalConsolidationGrant>(v.clone()).ok()
})
.is_some_and(|g| g.binding == self.binding)
{
continue;
}
match self.recover_consolidation_job(&jobs, &job) {
Ok(true) => {}
Ok(false)
if job.state == crate::jobs::JobState::Pending
&& job.attempt.state == crate::jobs::JobAttemptState::Reserved
&& job.executions_released() =>
{
schedule_job(jobs.clone(), job.request.job_id, self.self_weak.clone());
}
Ok(false) => {} // Missing receipts never authorize replay of effects.
Err(error) => eprintln!(
"standalone consolidation Job {} reconciliation blocked: {error}",
job.request.job_id
),
}
}
}
Err(error) => eprintln!("standalone consolidation recovery lookup failed: {error}"),
}
self.recover_journals();
}
/// Reconstructs a result from domain receipts, never by reentering the finalizer.
/// Acceptance does not assert prior owner shutdown or discharge cleanup ownership.
fn recover_consolidation_job(
&self,
jobs: &crate::jobs::StandaloneJobs,
observed: &crate::jobs::JobSnapshot,
) -> Result<bool, SubjektivHostError> {
let service_error = |error: crate::jobs::JobServiceError| {
SubjektivHostError::Unavailable(error.to_string())
};
self.ensure_live()?;
let current = jobs.get(&observed.request.job_id).map_err(service_error)?;
if current.attempt.attempt_id != observed.attempt.attempt_id
|| current.input_digest != observed.input_digest
{
return Err(SubjektivHostError::InvalidArgument(
"consolidation recovery attempt/input changed".into(),
));
}
if current.version_stopped
|| current.input_disposition != crate::jobs::JobInputDisposition::Available
|| current.state == crate::jobs::JobState::Cancelled
{
return Err(SubjektivHostError::InvalidArgument(
"stopped or retired consolidation cannot be recovered".into(),
));
}
current
.load_executable_request()
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))?;
let grant = self.consolidation_grant(&current)?;
let Some(result) = subjektiv::job::lookup_result(
&self.catalog.store,
&grant.binding.subject_id,
&grant.candidate_ids,
&current.request.job_id,
&current.attempt.attempt_id,
)
.map_err(operation_error)?
else {
return Ok(false);
};
self.ensure_live()?;
let settled = jobs
.accept_reconciled(&job::JobResultSubmission {
job_id: current.request.job_id,
attempt_id: current.attempt.attempt_id,
input_digest: current.input_digest,
result,
})
.map_err(service_error)?;
jobs.acknowledge(&settled.request.job_id)
.map_err(service_error)?;
Ok(true)
}
fn consolidation_grant(
&self,
snapshot: &crate::jobs::JobSnapshot,
) -> Result<LocalConsolidationGrant, SubjektivHostError> {
let grant: LocalConsolidationGrant =
serde_json::from_value(snapshot.domain_grant.clone().ok_or_else(|| {
SubjektivHostError::InvalidArgument("missing explicit Subject delegation".into())
})?)
.map_err(|error| SubjektivHostError::InvalidArgument(error.to_string()))?;
if grant.binding != self.binding
|| snapshot.request.purpose != "subjektiv_consolidation"
|| snapshot.request.input
!= serde_json::json!({"subject_id":self.binding.subject_id,"candidate_ids":grant.candidate_ids})
{
return Err(SubjektivHostError::InvalidArgument(
"persisted Job grant/input does not match local Subject capability".into(),
));
}
let subject = self
.catalog
.store
.subject(&self.binding.subject_id)
.map_err(domain_error)?
.ok_or_else(|| SubjektivHostError::Unavailable("Subject missing".into()))?;
if subject.state != subjektiv::SubjectState::Active {
return Err(SubjektivHostError::Unavailable("Subject is retired".into()));
}
Ok(grant)
}
fn validate_evidence(
&self,
session: &str,
@@ -1125,7 +1241,7 @@ impl SubjektivHost for LocalSubjectHost {
.store
.pending_staging_backlog(&self.binding.subject_id)
.map_err(domain_error)?;
return Ok(memory::backend::MemoryConsolidationOutput { status:"resumed_pending".into(),summary:"Resumed the existing immutable reserved Subject Job; unknown attempts are never replayed".into(),candidate_count:count,total_bytes:bytes });
return Ok(memory::backend::MemoryConsolidationOutput { status:"resumed_pending".into(),summary:"Resumed the existing immutable reserved Subject Job; prior execution receipts remain independently fenced".into(),candidate_count:count,total_bytes:bytes });
}
let (count, bytes) = self
.catalog
@@ -1195,6 +1311,12 @@ struct LocalConsolidationGrant {
}
struct LocalDomainProvider(std::sync::Weak<LocalSubjectHost>);
impl crate::jobs::JobDomainProvider for LocalDomainProvider {
fn callbacks_use_attempt_fence(&self) -> bool {
// Consolidation memory effects and Journal execute/finish hold the attempt gate.
// Result finalizers are independently revoked and joined by the owned Job runner.
true
}
fn grant(
&self,
snapshot: &crate::jobs::JobSnapshot,
@@ -1205,20 +1327,9 @@ impl crate::jobs::JobDomainProvider for LocalDomainProvider {
if snapshot.request.purpose == journal::PURPOSE {
return journal::grant(host, snapshot, fence);
}
let grant: LocalConsolidationGrant = serde_json::from_value(
snapshot
.domain_grant
.clone()
.ok_or("missing explicit Subject delegation")?,
)
.map_err(|e| e.to_string())?;
if grant.binding != host.binding
|| snapshot.request.purpose != "subjektiv_consolidation"
|| snapshot.request.input
!= serde_json::json!({"subject_id":host.binding.subject_id,"candidate_ids":grant.candidate_ids})
{
return Err("persisted Job grant/input does not match local Subject capability".into());
}
let grant = host
.consolidation_grant(snapshot)
.map_err(|error| error.to_string())?;
let adapter = Arc::new(LocalConsolidationHost {
host,
grant,
@@ -1311,7 +1422,11 @@ impl SubjektivHost for LocalConsolidationHost {
_context: &SubjektivHostContext,
operation: SubjektivMemoryBackendOperation,
) -> Result<SubjektivMemoryBackendResponse, SubjektivHostError> {
self.live()?;
let _effects = self
.fence
.enter()
.map_err(SubjektivHostError::Unavailable)?;
self.host.ensure_live()?;
let authority = subjektiv::api::HostOperationContext::validated_consolidation(
&self.host.catalog.store,
&self.grant.binding.subject_id,
crates/standalone/src/subjektiv/journal.rs
+166 -257
View File
@@ -305,11 +305,11 @@ impl LocalSubjectHost {
.pending()
.map_err(service_error)?
.into_iter()
.chain(jobs.unknown_granted().map_err(service_error)?)
.chain(jobs.unreleased_granted().map_err(service_error)?)
.chain(jobs.unacknowledged_results().map_err(service_error)?)
{
let _ = job;
return Err(SubjektivHostError::Conflict { code: "journal_outcome_unconfirmed".into(), message: "Subject has queued/running or Unknown/unacknowledged delegated work; binding retained".into() });
return Err(SubjektivHostError::Conflict { code: "journal_outcome_unconfirmed".into(), message: "Subject has queued/running, unreleased or unacknowledged delegated work; binding retained".into() });
}
let released = self
.catalog
@@ -474,6 +474,7 @@ impl LocalSubjectHost {
pub(super) fn recover_journals(&self) {
let result = (|| {
self.ensure_live()?;
self.jobs()?.recover_storage().map_err(service_error)?;
for attribution in self
.catalog
.store
@@ -565,9 +566,16 @@ impl LocalSubjectHost {
// Continue even without a further user turn. A later admission has
// remained in the ledger, never in this running Job's frozen input.
}
Ok(snapshot)
if snapshot.state == JobState::Failed
&& snapshot.executions_released()
&& !snapshot.version_stopped =>
{
// The next pass reconciles saved operations/checkpoints and resumes
// this exact Job/input within its attempt budget, without resampling.
}
Ok(_) | Err(_) => {
// Failure is not permission to consume sources or blindly rerun the model.
// Reopen/reconnect performs domain reconciliation before replacement.
// Neither failure nor a missing task proves resource shutdown.
*host
.journal_dispatch
.lock()
@@ -593,54 +601,8 @@ impl LocalSubjectHost {
if pending.request.purpose != PURPOSE {
continue;
}
self.journal_grant_scope(&pending)?;
if pending.request.input.get("capture").is_some() {
self.journal_grant(&pending)?;
return Ok(Some((jobs, pending.request.job_id)));
}
// Old descriptor jobs never acquire new execution authority. Their immutable
// identity/input is retired, not upgraded in place; saved checkpoint wins.
let replacement = self.replace_legacy_journal_job(&jobs, &pending)?;
if let Some(id) = replacement {
return Ok(Some((jobs, id)));
}
}
for unknown in jobs.unknown_granted().map_err(service_error)? {
let Ok(grant) = self.journal_grant_scope(&unknown) else {
continue;
};
let context = authority(self, &grant.session_id)?;
if let Some(active) = self
.catalog
.store
.active_journal_generation(&context, &grant.journal_id)
.map_err(journal_error)?
{
if active.job.job_id != unknown.request.job_id
|| active.job.attempt_id != unknown.attempt.attempt_id
{
continue;
}
self.catalog
.store
.close_journal_generation(&context, &active.id, &active.job)
.map_err(journal_error)?;
}
// This grant has no ambient side effects: every successful mutation and exact
// receipt is in the shared store. Its closed generation fences delayed tools.
// Observe actual saved current/open/cursor before freeing generic serialization;
// the replacement reads these receipts, never resubmits an aggregate old result.
let _ = self
.catalog
.store
.journal_resume_state(&context, &grant.journal_id)
.map_err(journal_error)?;
jobs.reconcile_granted_attempt(
&unknown.request.job_id,
&unknown.attempt.attempt_id,
&format!("journal-fenced:{}", unknown.attempt.attempt_id),
)
.map_err(service_error)?;
self.journal_grant(&pending)?;
return Ok(Some((jobs, pending.request.job_id)));
}
let reader =
JournalHostContext::validated_reader(&self.catalog.store, &self.binding.subject_id)
@@ -662,112 +624,63 @@ impl LocalSubjectHost {
continue;
}
let context = authority(self, &journal.source_session_id)?;
if let Some(active) = self
.catalog
.store
.active_journal_generation(&context, &journal.id)
.map_err(journal_error)?
{
let existing = jobs.get(&active.job.job_id).map_err(service_error)?;
if existing.attempt.attempt_id != active.job.attempt_id {
return Err(SubjektivHostError::Unavailable(
"Journal Job attempt changed without reconciliation".into(),
));
}
match existing.attempt.state {
JobAttemptState::Reserved | JobAttemptState::Dispatched => continue,
JobAttemptState::Completed => {
self.complete_journal_job(&jobs, &existing)?
}
JobAttemptState::Unknown
| JobAttemptState::Failed
| JobAttemptState::Cancelled => {
// The generic Host has durably fenced this attempt. Close its domain
// generation before replacement; saved record receipts remain visible.
self.catalog
.store
.close_journal_generation(&context, &active.id, &active.job)
.map_err(journal_error)?;
}
}
}
let mut retained = None;
// Saved body snapshots always win over descriptor migration. In particular,
// a failed continuation must not recapture its original text even if an
// invalidated legacy grant still bounds a larger unprocessed suffix.
for legacy_pass in [false, true] {
let mut after_job = None;
loop {
let page = jobs
.list_session_snapshots(
&journal.source_session_id,
after_job.as_deref(),
100,
)
.map_err(service_error)?;
if page.is_empty() {
break;
}
for saved in &page {
if saved.request.purpose != PURPOSE {
continue;
}
let legacy = saved.input_disposition
== JobInputDisposition::Invalidated
|| (saved.input_disposition == JobInputDisposition::Available
&& saved.request.input.get("capture").is_none());
if legacy != legacy_pass
|| (!legacy
&& saved.input_disposition != JobInputDisposition::Available)
{
continue;
}
let saved_grant = self.journal_grant_scope(saved)?;
if saved_grant.journal_id != journal.id {
continue;
}
if legacy {
if let Some(id) = self.replace_legacy_journal_job(&jobs, saved)? {
return Ok(Some((jobs, id)));
}
continue;
}
self.journal_grant(saved)?;
let original = JournalInput::decode(&saved.request.input)
.map_err(journal_error)?;
let continuation = self.build_journal_input(
&context,
&journal.id,
Some(&original.capture),
)?;
if !continuation.selected_ranges.is_empty() {
retained = Some(continuation);
break;
}
if saved.state == JobState::Completed && saved.acknowledged {
// Recover the acknowledgement-before-reclamation gap without
// turning a retained receipt into an executable empty input.
jobs.reclaim_input(
&saved.request.job_id,
&saved.input_digest,
JobInputRetirementReason::Completed,
)
.map_err(service_error)?;
}
}
if retained.is_some() {
break;
}
after_job = page.last().map(|job| job.request.job_id.clone());
}
if retained.is_some() {
// Resolve every saved owner before admitting a new prefix. A failed
// Job is not permission to resample its input into a replacement Job.
let mut after_job = None;
loop {
let saved_page = jobs
.list_session_snapshots(
&journal.source_session_id,
after_job.as_deref(),
100,
)
.map_err(service_error)?;
if saved_page.is_empty() {
break;
}
for saved in &saved_page {
if saved.request.purpose != PURPOSE {
continue;
}
let grant = self.journal_grant_scope(saved)?;
if grant.journal_id != journal.id {
continue;
}
if saved.version_stopped {
// Migration is not normal recovery or Journal completion.
// Keep the original ranges/input, even when fully checkpointed.
if !self.saved_ranges_reflected(&context, &grant)?
|| !saved.executions_released()
{
return Err(SubjektivHostError::Conflict { code: "journal_version_stopped".into(), message: "Version-stopped Journal retains input/checkpoint and unproven execution obligations".into() });
}
continue;
}
match saved.state {
JobState::Completed => self.complete_journal_job(&jobs, saved)?,
JobState::Pending
if saved.attempt.state == JobAttemptState::Reserved =>
{
self.journal_grant(saved)?;
return Ok(Some((jobs, saved.request.job_id.clone())));
}
JobState::Pending | JobState::Failed => {
if let Some(resumed) = self.recover_journal_job(&jobs, saved)? {
return Ok(Some((jobs, resumed.request.job_id)));
}
}
JobState::Cancelled => {
if !saved.executions_released()
|| !self.saved_ranges_reflected(&context, &grant)?
{
return Err(SubjektivHostError::Conflict {code:"journal_cancelled_input_retained".into(),message:"Cancelled Journal has unreleased execution or unreflected saved input".into()});
}
}
}
}
after_job = saved_page.last().map(|job| job.request.job_id.clone());
}
let input = match retained {
Some(input) => input,
None => self.build_journal_input(&context, &journal.id, None)?,
};
let input = self.build_journal_input(&context, &journal.id, None)?;
if input.selected_ranges.is_empty() {
continue;
@@ -797,120 +710,105 @@ impl LocalSubjectHost {
}
}
fn replace_legacy_journal_job(
fn saved_ranges_reflected(
&self,
context: &JournalHostContext,
grant: &LocalJournalGrant,
) -> Result<bool, SubjektivHostError> {
for range in &grant.ranges {
let status = self
.catalog
.store
.read_journal_source(context, &grant.journal_id, range.source.order)
.map_err(journal_error)?;
if status.source != range.source || status.reflected_bytes < range.bytes.end {
return Ok(false);
}
}
Ok(true)
}
fn recover_journal_job(
&self,
jobs: &crate::jobs::StandaloneJobs,
old: &JobSnapshot,
) -> Result<Option<String>, SubjektivHostError> {
let grant = self.journal_grant_scope(old)?;
if old.input_disposition == JobInputDisposition::Available {
old.load_executable_request()
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
if old.request.input
!= serde_json::json!({"selected_ranges":grant.ranges,"origins":grant.origins})
{
return Err(SubjektivHostError::InvalidArgument(
"invalid legacy Journal descriptor input".into(),
));
}
} else if old.input_disposition != JobInputDisposition::Invalidated {
return Err(SubjektivHostError::InvalidArgument(
"legacy Journal input is unavailable".into(),
));
saved: &JobSnapshot,
) -> Result<Option<JobSnapshot>, SubjektivHostError> {
let current = jobs.get(&saved.request.job_id).map_err(service_error)?;
if current.attempt.attempt_id != saved.attempt.attempt_id
|| current.input_digest != saved.input_digest
{
return Err(SubjektivHostError::Conflict {
code: "journal_attempt_changed".into(),
message: "Journal recovery snapshot no longer names the current attempt/input"
.into(),
});
}
let saved = &current;
if saved.version_stopped {
return Err(SubjektivHostError::Conflict {
code: "journal_version_stopped".into(),
message:
"Version-stopped Journal cannot resume or convert checkpoints into completion"
.into(),
});
}
self.journal_grant(saved)?; // verifies the immutable body; never fetch live text
let grant = self.journal_grant_scope(saved)?;
let context = authority(self, &grant.session_id)?;
if let Some(active) = self
let binding = JobAttemptBinding {
job_id: saved.request.job_id.clone(),
attempt_id: saved.attempt.attempt_id.clone(),
};
let generation = self
.catalog
.store
.active_journal_generation(&context, &grant.journal_id)
.map_err(journal_error)?
{
if active.job.job_id != old.request.job_id {
return Err(SubjektivHostError::InvalidArgument(
"another generation owns legacy migration".into(),
));
.retained_journal_generation(&context, &grant.journal_id, &binding)
.map_err(journal_error)?;
if let Some(generation) = generation.as_ref() {
let recovery = JournalJobContext::validated_recovery(
&self.catalog.store,
&context,
&generation.id,
binding.clone(),
)
.map_err(journal_error)?;
match subjektiv::journal::service::finish(&self.catalog.store, &recovery) {
Ok(completion) => {
let settled = jobs
.accept_reconciled(&job::JobResultSubmission {
job_id: binding.job_id.clone(),
attempt_id: binding.attempt_id.clone(),
input_digest: saved.input_digest.clone(),
result: serde_json::to_value(completion)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
})
.map_err(service_error)?;
self.complete_journal_job(jobs, &settled)?;
return Ok(None);
}
Err(JournalError::Conflict(_)) => {} // incomplete, retained checkpoint
Err(error) => return Err(journal_error(error)),
}
}
if !saved.executions_released() {
return Err(SubjektivHostError::Conflict {code:"journal_execution_unreleased".into(),message:"Saved Journal input/operations retained; preceding execution/cleanup is not proven stopped".into()});
}
let receipt = if let Some(generation) = generation {
self.catalog
.store
.close_journal_generation(&context, &active.id, &active.job)
.close_journal_generation(&context, &generation.id, &binding)
.map_err(journal_error)?;
}
jobs.cancel(&old.request.job_id).map_err(service_error)?;
let current = self
.catalog
.store
.read_journal(&context, &grant.journal_id)
.map_err(journal_error)?;
let end = grant
.ranges
.last()
.ok_or_else(|| SubjektivHostError::InvalidArgument("empty legacy selection".into()))?;
let end_position = if end.bytes.end == end.source.total_bytes {
(end.source.order + 1, 0)
format!("journal-generation-closed:{}", generation.id)
} else {
(end.source.order, end.bytes.end)
// The authoritative domain lookup found no generation for this exact attempt.
"journal-no-generation".to_string()
};
let replacement = if (current.process_order, current.process_byte) >= end_position {
None
} else {
let mut capture = self.recording_capture(&grant.session_id)?;
capture
.entries
.retain(|entry| grant.ranges.iter().any(|r| r.source.order == entry.order));
for entry in &mut capture.entries {
let range = grant
.ranges
.iter()
.find(|r| r.source.order == entry.order)
.unwrap();
let session_store::capture::CaptureBody::CanonicalPublic {
descriptor,
byte_offset,
text,
..
} = &mut entry.body
else {
unreachable!()
};
if source_descriptor(&grant.session_id, descriptor) != range.source
|| !grant.origins.iter().any(|origin| {
origin.pointer == range.source.pointer && origin.origin == descriptor.origin
})
|| !text.is_char_boundary(range.bytes.start as usize)
|| !text.is_char_boundary(range.bytes.end as usize)
{
return Err(SubjektivHostError::InvalidArgument(
"legacy original source identity/range changed".into(),
));
}
*text = text[range.bytes.start as usize..range.bytes.end as usize].to_owned();
*byte_offset = range.bytes.start;
}
let input = self.build_journal_input(&context, &grant.journal_id, Some(&capture))?;
let request = self.journal_request(&grant.journal_id, &input)?;
let next_grant = LocalJournalGrant {
ranges: input.selected_ranges.clone(),
origins: vec![],
..grant
};
Some(
jobs.request_granted(
request,
serde_json::to_value(next_grant)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
)
.map_err(service_error)?
.request
.job_id,
)
};
jobs.reclaim_input(
&old.request.job_id,
&old.input_digest,
JobInputRetirementReason::LegacyInvalidation,
)
.map_err(service_error)?;
Ok(replacement)
jobs.reconcile_granted_attempt(&binding.job_id, &binding.attempt_id, &receipt)
.map_err(service_error)?;
let resumed = jobs
.resume_granted(&binding.job_id, &binding.attempt_id)
.map_err(service_error)?;
Ok(Some(resumed))
}
fn journal_request(
@@ -1455,6 +1353,9 @@ impl LocalSubjectHost {
}
jobs.acknowledge(&snapshot.request.job_id)
.map_err(service_error)?;
if !snapshot.executions_released() {
return Err(SubjektivHostError::Conflict {code:"journal_cleanup_unconfirmed".into(),message:"Checkpoint/result completed; execution cleanup remains independent and unconfirmed".into()});
}
jobs.reclaim_input(
&snapshot.request.job_id,
&snapshot.input_digest,
@@ -1560,6 +1461,10 @@ impl JournalJobHost for LocalJournalHost {
&self,
request: JournalJobRequest,
) -> Result<JournalJobResponse, SubjektivHostError> {
let _effects = self
.fence
.enter()
.map_err(SubjektivHostError::Unavailable)?;
let context = self.live()?;
// The generic runner has supplied this exact immutable input before it can
// invoke this bound tool. Preparation alone is not evidence of delivery.
@@ -1579,6 +1484,10 @@ impl JournalJobHost for LocalJournalHost {
.map_err(journal_error)
}
fn finish(&self) -> Result<JournalJobCompletion, SubjektivHostError> {
let _effects = self
.fence
.enter()
.map_err(SubjektivHostError::Unavailable)?;
let context = self.live()?;
subjektiv::journal::service::finish(&self.host.catalog.store, &context)
.map_err(journal_error)
crates/standalone/src/subjektiv/journal/tests.rs
+109 -77
View File
@@ -11,6 +11,7 @@ mod compaction;
mod connected_compaction;
mod handoff;
mod memory_links;
mod recovery;
struct Fixture {
_temp: tempfile::TempDir,
@@ -1092,7 +1093,7 @@ async fn persisted_pre_binding_job_recovers_only_under_original_session_lifetime
}
#[tokio::test]
async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_create() {
async fn shutdown_reopen_resumes_same_job_and_frozen_input_without_duplicate_create() {
let mut fixture = Fixture::new("proposed, corrected, partially performed, then interrupted");
let state = fixture.host().catalog.state_root.clone();
let worker = fixture._allocation.as_ref().unwrap().worker_id();
@@ -1129,12 +1130,27 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
.items;
assert_eq!(entries.len(), 1);
let saved_entry = entries[0].id.clone();
let original_input = jobs.get(&id).unwrap().request.input;
let original = jobs.get(&id).unwrap();
let original_input = original.request.input.clone();
fixture.append(
"newer-than-snapshot",
"NEW INPUT MUST NOT MIX INTO RESUMED SNAPSHOT",
);
jobs.shutdown().await.unwrap();
let stopped_store = crate::jobs::StandaloneJobs::open(
&fixture.connection.jobs_path,
fixture._temp.path().into(),
)
.unwrap();
let stopped = stopped_store.get(&id).unwrap();
assert_eq!(stopped.state, JobState::Failed);
assert_eq!(stopped.attempt.state, JobAttemptState::Failed);
assert_eq!(stopped.attempt.attempt_id, original.attempt.attempt_id);
assert_eq!(stopped.request, original.request);
assert!(!stopped.version_stopped);
assert!(!stopped.executions.is_empty());
assert!(stopped.executions_released());
stopped_store.shutdown().await.unwrap();
fixture.connection.close().unwrap();
let mut reopened = StandaloneSubjects::open_existing(&state)
.unwrap()
@@ -1143,8 +1159,8 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
let jobs = crate::jobs::StandaloneJobs::open(&reopened.jobs_path, fixture._temp.path().into())
.unwrap();
reopened.host.bind_jobs(jobs.clone()).unwrap();
assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown);
// Deny every original-source read while preparing and running the replacement.
assert_eq!(jobs.get(&id).unwrap(), stopped);
// Deny every original-source read while preparing and running the new attempt.
let source_path = fixture
.sessions
.root_dir()
@@ -1154,28 +1170,35 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
.path()
.join("temporarily-unavailable-original");
fs::rename(&source_path, &denied).unwrap();
let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap();
let replacement_input = jobs.get(&replacement).unwrap().request.input;
assert_eq!(replacement_input["capture"], original_input["capture"]);
let (_, resumed_id) = reopened.host.next_journal_job().unwrap().unwrap();
assert_eq!(resumed_id, id);
let resumed = jobs.get(&resumed_id).unwrap();
assert_eq!(resumed.request, original.request);
assert_eq!(resumed.input_digest, original.input_digest);
assert_eq!(resumed.intent_fingerprint, original.intent_fingerprint);
assert_eq!(resumed.domain_grant, original.domain_grant);
assert_eq!(resumed.executions, stopped.executions);
assert_eq!(resumed.state, JobState::Pending);
assert_eq!(resumed.attempt.state, JobAttemptState::Reserved);
assert_eq!(resumed.attempt.number, original.attempt.number + 1);
assert_ne!(resumed.attempt.attempt_id, original.attempt.attempt_id);
assert_eq!(resumed.input_disposition, JobInputDisposition::Available);
assert!(
!replacement_input
!resumed
.request
.input
.to_string()
.contains("NEW INPUT MUST NOT MIX")
);
assert_eq!(jobs.get(&id).unwrap().request.input, original_input);
assert_eq!(
jobs.get(&id).unwrap().input_disposition,
JobInputDisposition::Available
);
assert_ne!(
replacement, id,
"unknown external execution is not blindly replayed"
);
jobs.start_with_model_client(&replacement, JournalScript::default())
let history = jobs
.list_session_snapshots(&fixture.context.session_id, None, 100)
.unwrap();
assert_eq!(history.len(), 1, "recovery must not reserve a second Job");
jobs.start_with_model_client(&resumed_id, JournalScript::default())
.await
.unwrap();
let completed =
tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&replacement))
tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&resumed_id))
.await
.unwrap()
.unwrap();
@@ -1894,7 +1917,7 @@ async fn saved_body_jobs_process_over_a_hundred_entries_and_utf8_without_live_so
}
#[tokio::test]
async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful_records() {
async fn descriptor_job_version_cutover_retains_input_and_records_without_live_recapture() {
let fixture = Fixture::new("original descriptor-only input");
let (adapter, _, modern) = fixture.adapter();
let jobs = fixture.host().jobs().unwrap();
@@ -1929,13 +1952,8 @@ async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful
request.job_id = format!("legacy-descriptor:{}", uuid::Uuid::now_v7());
request.input =
serde_json::json!({"selected_ranges":old_grant.ranges,"origins":old_grant.origins});
let old = jobs
.request_granted(request, serde_json::to_value(&old_grant).unwrap())
.unwrap();
assert!(
fixture.host().journal_grant(&old).is_err(),
"descriptor jobs must not execute as snapshots"
);
let path = fixture._temp.path().join("descriptor-schema-five.sqlite3");
recovery::seed_schema_five_journal(&path, &request, &old_grant);
let generation = fixture
.host()
.catalog
@@ -1944,8 +1962,8 @@ async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful
&adapter.context,
&old_grant.journal_id,
JobAttemptBinding {
job_id: old.request.job_id.clone(),
attempt_id: old.attempt.attempt_id.clone(),
job_id: request.job_id.clone(),
attempt_id: job::attempt_id(&request.job_id, 1),
},
old_grant.ranges.clone(),
)
@@ -1978,56 +1996,52 @@ async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful
},
)
.unwrap();
fixture.append(
"new-not-selected",
"LATEST SOURCE MUST WAIT UNTIL OLD INPUT FINISHES",
// Normal recovery is not the migration runner. Seed the old running attempt,
// then use the production initializer before consulting any live Session.
let prior = rusqlite::Connection::open(&path).unwrap();
prior
.execute("UPDATE job_attempts SET state='dispatched'", [])
.unwrap();
drop(prior);
let migrated = crate::jobs::StandaloneJobs::open(&path, fixture._temp.path().into()).unwrap();
let old = migrated.get(&request.job_id).unwrap();
assert!(old.version_stopped);
assert_eq!(old.state, JobState::Failed);
assert_eq!(old.attempt.state, JobAttemptState::Failed);
assert_eq!(
old.attempt.failure.as_deref(),
Some("バージョン更新による実行打切り")
);
fixture.host().admit_journal(&fixture.context).unwrap();
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
assert_ne!(next, old.request.job_id);
let replacement = jobs.get(&next).unwrap();
let replacement_input = JournalInput::decode(&replacement.request.input).unwrap();
assert_eq!(old.request, request);
assert_eq!(old.input_disposition, JobInputDisposition::Available);
assert!(!old.executions_released());
assert!(fixture.host().journal_grant(&old).is_err());
fixture.append("new-not-selected", "LATEST SOURCE MUST NOT BE RECAPTURED");
let error = fixture
.host()
.recover_journal_job(&migrated, &old)
.unwrap_err();
assert!(
!replacement
.request
.input
.to_string()
.contains("LATEST SOURCE MUST WAIT")
matches!(error, SubjektivHostError::Conflict { ref code, .. } if code == "journal_version_stopped")
);
assert_eq!(replacement_input.selected_ranges, old_grant.ranges);
let retired = jobs.get(&old.request.job_id).unwrap();
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
assert_eq!(retired.input_digest, old.input_digest);
assert_eq!(retired.intent_fingerprint, old.intent_fingerprint);
assert!(retired.request.input.is_null());
assert!(migrated.retry(&request.job_id).is_err());
assert!(
migrated
.resume_granted(&request.job_id, &old.attempt.attempt_id)
.is_err()
);
assert!(migrated.pending().unwrap().is_empty());
assert_eq!(migrated.get(&request.job_id).unwrap(), old);
assert_eq!(
fixture
.host()
.catalog
.store
.execute_journal(
&context,
JournalOperation {
request_id: "legacy-delayed".into(),
expected_state: saved.state_token,
action: JournalAction::ConfirmUnchanged,
covered_ranges: old_grant.ranges,
}
)
.is_err()
.active_journal_generation(&adapter.context, &adapter.grant.journal_id,)
.unwrap()
.unwrap(),
generation
);
jobs.start_with_model_client(&next, JournalScript::default())
.await
.unwrap();
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&next))
.await
.unwrap()
.unwrap();
assert_eq!(completed.state, JobState::Completed);
fixture
.host()
.complete_journal_job(&jobs, &completed)
.unwrap();
let entries = fixture
.host()
.catalog
@@ -2036,6 +2050,16 @@ async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful
.unwrap();
assert_eq!(entries.items.len(), 1);
assert_eq!(entries.items[0].id, saved.entry_id.unwrap());
drop(migrated);
let reopened = crate::jobs::StandaloneJobs::open(&path, fixture._temp.path().into()).unwrap();
assert_eq!(reopened.get(&request.job_id).unwrap(), old);
assert_eq!(
reopened
.list_session_snapshots(&fixture.context.session_id, None, 100)
.unwrap()
.len(),
1
);
jobs.shutdown().await.unwrap();
}
@@ -2373,23 +2397,31 @@ async fn close_cancels_retained_local_job_before_erase_can_acquire_exclusive_lea
.is_ok();
// Always tear down before asserting the observed violation, so a failed regression
// cannot leak the blocked model future into another test.
let _ = retained.shutdown().await;
tokio::time::timeout(std::time::Duration::from_secs(20), retained.shutdown())
.await
.unwrap()
.unwrap();
assert!(model.request_dropped.load(Ordering::Acquire));
fixture.connection.close().unwrap();
assert!(fixture.connection.lease.is_none());
assert!(fixture.host().jobs.lock().unwrap().is_none());
// Cancellation is not completion/erasure. Reopening retains the same unknown
// attempt's immutable body until the explicit erasure transaction retires it.
// Cancellation is not completion/erasure. The owned shutdown has confirmed cleanup,
// but reopening must preserve the failed attempt and its immutable body.
let reopened = crate::jobs::StandaloneJobs::open(
&fixture.connection.jobs_path,
fixture._temp.path().into(),
)
.unwrap();
let unknown = reopened.get(&id).unwrap();
assert_eq!(unknown.attempt.state, crate::jobs::JobAttemptState::Unknown);
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
let failed = reopened.get(&id).unwrap();
assert_eq!(failed.state, JobState::Failed);
assert_eq!(failed.attempt.state, JobAttemptState::Failed);
assert!(!failed.version_stopped);
assert_eq!(failed.attempt.number, 1);
assert!(!failed.executions.is_empty());
assert!(failed.executions_released());
assert_eq!(failed.input_disposition, JobInputDisposition::Available);
assert!(
unknown
failed
.request
.input
.to_string()
crates/standalone/src/subjektiv/journal/tests/connected_compaction.rs
+7 -3
View File
@@ -111,7 +111,7 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab
assert_eq!(journal.process_byte, before.current_journal.process_byte);
assert_eq!(journal.process_order, 0);
assert_eq!(journal.process_byte, 0, "saved open records do not advance processing checkpoint");
let expected_json = serde_json::to_string(&expected).unwrap();
let expected_json = serde_json::to_value(&expected).unwrap();
let adapter = Arc::new(adapter);
let model = MockClient { adapter: adapter.clone(), requests: Arc::default(), updated: Arc::new(AtomicBool::new(false)) };
let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap();
@@ -142,7 +142,11 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab
assert_eq!(journal_items.len(), 1);
let journal_item = Item::from(journal_items[0].item.clone());
let body = journal_item.as_text().unwrap();
assert!(body.contains(&expected_json), "durable seed contains the full selected JSON and processing checkpoint");
// Display is concise YAML; the Feature payload retains the full frozen
// snapshot, including processing checkpoints omitted from display.
let payload: serde_json::Value = journal_items[0].metadata.input_metadata.as_ref()
.unwrap().feature_payload().unwrap().decode("journal:compaction", 1).unwrap();
assert_eq!(payload["snapshot"], expected_json, "durable payload retains the complete frozen snapshot and processing checkpoint");
assert!(body.contains(BEFORE));
assert!(!body.contains(AFTER));
assert!(seed.iter().any(|entry| entry.metadata.input_metadata.as_ref().is_some_and(|input| input.usage().compaction == session_store::InputCompactionUse::OpeningSummary) && Item::from(entry.item.clone()).as_text().is_some_and(|text| text.contains(COMPLEMENT))));
@@ -152,7 +156,7 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab
_ => None,
}).expect("actual compactor user request");
assert!(input.ends_with(body), "exact same frozen reference goes to actual compactor and seed");
assert_eq!(input.matches(&expected_json).count(), 1);
assert_eq!(input.matches(body).count(), 1);
assert!(!input.contains(AFTER));
drop(requests);
let latest = fixture.host().journal_compaction_context(&fixture.context).unwrap();
crates/standalone/src/subjektiv/journal/tests/recovery.rs
+321
View File
@@ -0,0 +1,321 @@
//! Journal recovery connects the real local Host, Journal store and temporary Job SQLite
//! stores. Persisted prior-owner rows are injected without spawning a Worker; they cannot
//! supply cleanup proof. Scripted owned-Worker shutdown is covered in the parent suite.
use super::*;
#[test]
fn restart_reconciles_complete_checkpoint_but_retains_partial_input_and_all_unproven_cleanup() {
for checkpoint_complete in [false, true] {
let fixture = Fixture::new("restart uses only this frozen body");
let (adapter, _, original) = fixture.adapter();
let initial = adapter.state().unwrap();
let saved = save(
&adapter,
"restart-record",
&initial.current_journal.state_token,
JournalAction::Create {
title: "saved once".into(),
body: "retained operation".into(),
ongoing: "ongoing".into(),
state: JournalEntryState::Open,
source_end: None,
correction_refs: vec![],
},
initial.selected_ranges.clone(),
);
if checkpoint_complete {
let position = save(
&adapter,
"restart-position",
&saved.state_token,
JournalAction::UpdatePosition {
current_position: "selected input reflected".into(),
},
vec![],
);
save(
&adapter,
"restart-checkpoint",
&position.state_token,
JournalAction::Checkpoint {
record_operation_ids: vec!["restart-record".into()],
},
initial.selected_ranges,
);
}
let path = fixture._temp.path().join("restart-job.sqlite3");
let mut prior = crate::job_store::JobStore::open(&path).unwrap();
prior.bind_host_lease("prior-host").unwrap();
let reserved = prior
.reserve_granted(original.request.clone(), original.domain_grant.clone())
.unwrap();
let dispatched = prior
.dispatch(&original.request.job_id, &reserved.attempt.attempt_id, 8)
.unwrap();
drop(prior);
let mut next = crate::job_store::JobStore::open(&path).unwrap();
next.bind_host_lease("replacement-host").unwrap();
let failed = next.reconcile_host_restart().unwrap().pop().unwrap();
assert_eq!(failed.state, JobState::Failed);
assert_eq!(failed.executions, dispatched.executions);
assert!(!failed.version_stopped);
drop(next);
let jobs = crate::jobs::StandaloneJobs::open(&path, fixture._temp.path().into()).unwrap();
// A missing live Session makes any source re-fetch an observable failure.
let source = fixture
.sessions
.root_dir()
.join(&fixture.context.session_id);
let unavailable = fixture._temp.path().join("unavailable-live-session");
fs::rename(&source, &unavailable).unwrap();
let error = fixture
.host()
.recover_journal_job(&jobs, &failed)
.unwrap_err();
let settled = jobs.get(&original.request.job_id).unwrap();
if checkpoint_complete {
assert!(
matches!(error, SubjektivHostError::Conflict {ref code, ..} if code == "journal_cleanup_unconfirmed")
);
assert_eq!(settled.state, JobState::Completed);
let completion: JournalJobCompletion =
serde_json::from_value(settled.result.clone().unwrap()).unwrap();
assert_eq!(completion.generation_id, adapter.generation.id);
assert!(settled.acknowledged);
} else {
assert!(
matches!(error, SubjektivHostError::Conflict {ref code, ..} if code == "journal_execution_unreleased")
);
assert_eq!(settled, failed);
assert!(settled.result.is_none());
}
assert_eq!(settled.attempt.attempt_id, reserved.attempt.attempt_id);
assert_eq!(settled.attempt.number, 1);
assert_eq!(settled.request, original.request);
assert_eq!(settled.input_disposition, JobInputDisposition::Available);
assert_eq!(settled.executions, dispatched.executions);
assert!(!settled.executions_released());
assert!(jobs.pending().unwrap().is_empty());
assert!(
jobs.resume_granted(&settled.request.job_id, &settled.attempt.attempt_id)
.is_err()
);
let entries = fixture
.host()
.catalog
.store
.list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32)
.unwrap();
assert_eq!(entries.items.len(), 1);
assert_eq!(entries.items[0].id, saved.entry_id.unwrap());
fs::rename(&unavailable, &source).unwrap();
}
}
#[test]
fn current_schema_reopen_preserves_unconfirmed_execution_and_blocks_journal_resume() {
let fixture = Fixture::new("saved input must not be recaptured");
let (adapter, _, original) = fixture.adapter();
let state = adapter.state().unwrap();
let saved = save(
&adapter,
"saved-before-owner-loss",
&state.current_journal.state_token,
JournalAction::Create {
title: "partial operation".into(),
body: "durable record without checkpoint".into(),
ongoing: "unconfirmed execution".into(),
state: JournalEntryState::Open,
source_end: None,
correction_refs: vec![],
},
state.selected_ranges,
);
let source = fixture
.sessions
.root_dir()
.join(&fixture.context.session_id);
let denied = fixture._temp.path().join("unavailable-recovery-source");
fs::rename(&source, &denied).unwrap();
for fail_attempt in [false, true] {
let path = fixture._temp.path().join(if fail_attempt {
"prior-failed.sqlite3"
} else {
"prior-dispatched.sqlite3"
});
let mut prior = crate::job_store::JobStore::open(&path).unwrap();
let reserved = prior
.reserve_granted(original.request.clone(), original.domain_grant.clone())
.unwrap();
let mut retained = prior
.dispatch(&original.request.job_id, &reserved.attempt.attempt_id, 8)
.unwrap();
if fail_attempt {
retained = prior
.finish(
&original.request.job_id,
&reserved.attempt.attempt_id,
JobAttemptState::Failed,
Some("prior execution failed without cleanup evidence"),
)
.unwrap();
}
drop(prior);
let reopened =
crate::jobs::StandaloneJobs::open(&path, fixture._temp.path().into()).unwrap();
assert_eq!(reopened.get(&original.request.job_id).unwrap(), retained);
assert!(!retained.version_stopped);
assert!(!retained.executions_released());
assert_eq!(retained.executions.len(), 1);
assert!(!retained.executions[0].writes_revoked);
assert!(retained.executions[0].cleanup_receipt.is_none());
let error = match fixture.host().recover_journal_job(&reopened, &retained) {
Err(error) => error,
Ok(_) => panic!("unconfirmed prior execution must block Journal recovery"),
};
assert!(matches!(
error,
SubjektivHostError::Conflict { ref code, .. }
if code == "journal_execution_unreleased"
));
assert!(reopened.retry(&original.request.job_id).is_err());
assert!(
reopened
.reconcile_granted_attempt(
&original.request.job_id,
&retained.attempt.attempt_id,
"domain-receipts-alone-are-not-cleanup",
)
.is_err()
);
assert!(
reopened
.resume_granted(&original.request.job_id, &retained.attempt.attempt_id)
.is_err()
);
let mut replacement = original.request.clone();
replacement.job_id = format!("replacement-{fail_attempt}");
assert!(
reopened
.request_granted(replacement, original.domain_grant.clone().unwrap())
.is_err(),
"unconfirmed execution retains the serialization fence"
);
assert_eq!(reopened.get(&original.request.job_id).unwrap(), retained);
assert_eq!(
reopened
.list_session_snapshots(&fixture.context.session_id, None, 100)
.unwrap()
.len(),
1
);
assert!(matches!(
reopened.try_shutdown(),
Err(crate::jobs::JobServiceError::ShutdownIncomplete)
));
// Only prior-owner DB rows exist here: dropping this observer is not cleanup proof.
drop(reopened);
let reread = crate::job_store::JobStore::open(&path).unwrap();
assert_eq!(reread.get(&original.request.job_id).unwrap(), retained);
}
fs::rename(&denied, &source).unwrap();
let entries = fixture
.host()
.catalog
.store
.list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32)
.unwrap();
assert_eq!(entries.items.len(), 1);
assert_eq!(entries.items[0].id, saved.entry_id.unwrap());
assert_eq!(
fixture
.host()
.catalog
.store
.active_journal_generation(&adapter.context, &adapter.grant.journal_id)
.unwrap()
.unwrap(),
adapter.generation
);
}
/// Minimal schema-five storage graph for the Journal cutover boundary. Legacy state
/// spellings belong to this persisted input fixture, never to the current Rust enums.
pub(super) fn seed_schema_five_journal(
path: &std::path::Path,
request: &job::JobRequest,
grant: &LocalJournalGrant,
) {
let mut conn = rusqlite::Connection::open(path).unwrap();
feature_storage::configure_connection(&conn).unwrap();
let tx = conn.transaction().unwrap();
tx.execute_batch(
"CREATE TABLE job_intents (
job_id TEXT PRIMARY KEY NOT NULL,
request_json TEXT NOT NULL,
fingerprint TEXT NOT NULL,
serialization_key TEXT,
max_concurrent INTEGER NOT NULL,
state TEXT NOT NULL CHECK(state IN ('pending','completed','failed','unknown','cancelled')),
current_attempt INTEGER NOT NULL,
acknowledged INTEGER NOT NULL DEFAULT 0,
input_disposition TEXT NOT NULL DEFAULT 'available',
FOREIGN KEY(job_id,current_attempt) REFERENCES job_attempts(job_id,number)
DEFERRABLE INITIALLY DEFERRED
);
CREATE TABLE job_attempts (
job_id TEXT NOT NULL REFERENCES job_intents(job_id),
number INTEGER NOT NULL,
attempt_id TEXT NOT NULL UNIQUE,
input_digest TEXT NOT NULL,
state TEXT NOT NULL CHECK(state IN ('reserved','dispatched','completed','failed','unknown','cancelled')),
result_json TEXT,
result_digest TEXT,
failure TEXT,
PRIMARY KEY(job_id,number)
);
CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key)
WHERE serialization_key IS NOT NULL AND state='pending';
CREATE TABLE job_domain_grants (
job_id TEXT PRIMARY KEY NOT NULL REFERENCES job_intents(job_id),
grant_json TEXT NOT NULL
);
CREATE TABLE job_domain_reconciliations (
job_id TEXT NOT NULL REFERENCES job_intents(job_id),
attempt_id TEXT NOT NULL,
domain_receipt TEXT NOT NULL,
PRIMARY KEY(job_id,attempt_id)
);",
)
.unwrap();
tx.execute(
"INSERT INTO job_intents(job_id,request_json,fingerprint,serialization_key,max_concurrent,state,current_attempt)
VALUES(?1,?2,?3,?4,?5,'pending',1)",
rusqlite::params![
request.job_id,
serde_json::to_string(request).unwrap(),
request.fingerprint().unwrap(),
request.serialization_key,
request.limits.max_concurrent_jobs,
],
)
.unwrap();
tx.execute(
"INSERT INTO job_attempts(job_id,number,attempt_id,input_digest,state)
VALUES(?1,1,?2,?3,'reserved')",
rusqlite::params![
request.job_id,
job::attempt_id(&request.job_id, 1),
request.input_digest().unwrap(),
],
)
.unwrap();
tx.execute(
"INSERT INTO job_domain_grants(job_id,grant_json) VALUES(?1,?2)",
rusqlite::params![request.job_id, serde_json::to_string(grant).unwrap()],
)
.unwrap();
tx.pragma_update(None, "user_version", 5).unwrap();
tx.commit().unwrap();
}
crates/standalone/src/subjektiv/tests.rs
+452 -2
View File
@@ -1,5 +1,425 @@
//! Consolidation recovery connects the production Subject receipt store, SQLite Job
//! store and local recovery dispatcher. Fixtures persist an interrupted attempt;
//! the named joined-execution contrast substitutes that execution boundary's
//! cleanup receipt, not the domain lookup. No test here proves an OS crash/drain.
use super::*;
struct ConsolidationRecoveryFixture {
body: SubjectConnection,
jobs: crate::jobs::StandaloneJobs,
saved: crate::jobs::JobSnapshot,
execution_owner: crate::job_store::JobStore,
_temp: tempfile::TempDir,
}
impl Drop for ConsolidationRecoveryFixture {
fn drop(&mut self) {
// Revoke this fixture's Host before deleting its isolated storage. This
// does not settle the retained prior execution or manufacture drain proof.
self.body.host.live.store(false, Ordering::Release);
}
}
impl ConsolidationRecoveryFixture {
fn new() -> Self {
let temp = tempfile::tempdir().unwrap();
let catalog = StandaloneSubjects::open(temp.path().join("state")).unwrap();
let subject = catalog.create("recovery-owner", None).unwrap();
let body = catalog
.connect(&subject.id, protocol::WorkerId::now_v7())
.unwrap();
body.host
.catalog
.store
.stage_candidate(subjektiv::SubjectStagingRecord::attach(
&subject.id,
memory::extract::StagingRecord::from_candidate(
"candidate",
"dedup",
memory::schema::SourceRef {
segment_id: "segment".into(),
range: [1, 1],
},
memory::extract::ExtractedCandidate {
kind: memory::extract::CandidateKind::Lesson,
claim: "claim".into(),
why_useful: "reason".into(),
staleness: None,
evidence_ids: vec![],
},
vec![],
vec![],
),
))
.unwrap();
let grant = LocalConsolidationGrant {
binding: body.binding.clone(),
candidate_ids: vec!["candidate".into()],
};
let request = job::JobRequest {
job_id: "consolidation-recovery".into(),
purpose: "subjektiv_consolidation".into(),
input_ref: format!("subjektiv://{}/consolidation", subject.id),
input: serde_json::json!({"subject_id":subject.id,"candidate_ids":grant.candidate_ids}),
instruction: "Resolve the immutable batch".into(),
profile: "builtin:standalone-subjektiv-consolidation".into(),
serialization_key: Some(format!("subjektiv:{}", subject.id)),
limits: Default::default(),
};
let mut store = crate::job_store::JobStore::open(&body.jobs_path).unwrap();
let reserved = store
.reserve_granted(request, Some(serde_json::to_value(grant).unwrap()))
.unwrap();
store
.dispatch(&reserved.request.job_id, &reserved.attempt.attempt_id, 1)
.unwrap();
let saved = store
.finish(
&reserved.request.job_id,
&reserved.attempt.attempt_id,
crate::jobs::JobAttemptState::Failed,
Some("host_restart_execution_unconfirmed: retained prior owner"),
)
.unwrap();
let jobs =
crate::jobs::StandaloneJobs::open(&body.jobs_path, temp.path().to_path_buf()).unwrap();
body.host.bind_jobs(jobs.clone()).unwrap();
Self {
_temp: temp,
body,
jobs,
saved,
execution_owner: store,
}
}
fn confirm_execution_joined(&mut self) {
// The execution boundary is substituted in this domain-consumer fixture;
// use its exact owner/run to commit the receipt through the real store.
self.execution_owner
.confirm_execution_cleanup(
&self.saved.request.job_id,
&self.saved.executions[0],
"fixture-owned-execution-joined",
)
.unwrap();
self.saved = self.jobs.get(&self.saved.request.job_id).unwrap();
}
fn resolve_candidate(&self) {
self.body
.host
.catalog
.store
.decide_candidate(
&self.body.binding.subject_id,
subjektiv::CandidateDecisionRequest {
request_id: "decision".into(),
candidate_id: "candidate".into(),
reason: "invalid".into(),
decision: subjektiv::CandidateDecision::Close {
action: subjektiv::StagingResolutionAction::Invalid,
affected_memory: vec![],
},
},
)
.unwrap();
}
fn generation(&self, attempt_id: &str) -> subjektiv::SurfaceGeneration {
self.body
.host
.catalog
.store
.prepare_job_surface_generation(
&self.body.binding.subject_id,
Some((&self.saved.request.job_id, attempt_id)),
)
.unwrap()
}
fn publish(&self, generation: &subjektiv::SurfaceGeneration) {
self.body
.host
.catalog
.store
.publish_surface_generation(&self.body.binding.subject_id, &generation.id, vec![])
.unwrap();
}
}
#[tokio::test]
async fn consolidation_recovery_accepts_same_attempt_receipts_without_claiming_owner_drain() {
for availability in ["ready", "failed"] {
let fixture = ConsolidationRecoveryFixture::new();
fixture.resolve_candidate();
let generation = fixture.generation(&fixture.saved.attempt.attempt_id);
if availability == "ready" {
fixture.publish(&generation);
} else {
fixture
.body
.host
.catalog
.store
.fail_surface_generation(
&fixture.body.binding.subject_id,
&generation.id,
"model_failure",
)
.unwrap();
}
let resolution = serde_json::to_value(
fixture
.body
.host
.catalog
.store
.staging_resolution(&fixture.body.binding.subject_id, "candidate")
.unwrap(),
)
.unwrap();
assert!(fixture.jobs.pending().unwrap().is_empty());
fixture.body.host.recover_jobs();
let completed = fixture.jobs.get(&fixture.saved.request.job_id).unwrap();
assert_eq!(completed.state, crate::jobs::JobState::Completed);
assert_eq!(
completed.attempt.attempt_id,
fixture.saved.attempt.attempt_id
);
assert_eq!(completed.attempt.number, fixture.saved.attempt.number);
assert_eq!(completed.input_digest, fixture.saved.input_digest);
assert_eq!(completed.executions, fixture.saved.executions);
assert!(!completed.executions_released());
assert!(completed.acknowledged);
let result = completed.result.as_ref().unwrap();
assert_eq!(result["surface"]["generation_id"], generation.id);
assert_eq!(result["surface"]["availability"], availability);
assert_eq!(
result["candidate_dispositions"],
serde_json::json!([{"candidate_id":"candidate","action":"invalid"}])
);
if availability == "failed" {
assert_eq!(result["surface"]["reason_code"], "model_failure");
}
fixture.body.host.recover_jobs();
assert_eq!(
fixture.jobs.get(&completed.request.job_id).unwrap(),
completed
);
assert_eq!(
serde_json::to_value(
fixture
.body
.host
.catalog
.store
.staging_resolution(&fixture.body.binding.subject_id, "candidate")
.unwrap(),
)
.unwrap(),
resolution
);
// A separate production store read proves settlement was persisted, not
// merely projected by the dispatcher; no execution cleanup is inferred.
let reopened = crate::job_store::JobStore::open(&fixture.body.jobs_path).unwrap();
assert_eq!(reopened.get(&completed.request.job_id).unwrap(), completed);
}
}
#[tokio::test]
async fn consolidation_recovery_reads_saved_result_after_execution_join_and_writer_fence() {
let mut fixture = ConsolidationRecoveryFixture::new();
fixture.resolve_candidate();
let generation = fixture.generation(&fixture.saved.attempt.attempt_id);
fixture.publish(&generation);
fixture.confirm_execution_joined();
assert!(fixture.saved.executions_released());
assert!(fixture.saved.executions[0].writes_revoked);
assert!(
fixture
.body
.host
.catalog
.store
.journal_writer_active(&fixture.body.host.writer)
.unwrap()
);
fixture.body.host.recover_jobs();
let completed = fixture.jobs.get(&fixture.saved.request.job_id).unwrap();
assert_eq!(completed.state, crate::jobs::JobState::Completed);
assert_eq!(
completed.attempt.attempt_id,
fixture.saved.attempt.attempt_id
);
assert_eq!(completed.input_digest, fixture.saved.input_digest);
assert_eq!(completed.executions, fixture.saved.executions);
assert!(completed.executions_released());
assert_eq!(
completed.result.as_ref().unwrap()["surface"]["generation_id"],
generation.id
);
}
#[tokio::test]
async fn consolidation_recovery_never_replays_incomplete_foreign_or_stale_receipts() {
for case in ["missing_surface", "unresolved", "foreign_attempt", "stale"] {
let fixture = ConsolidationRecoveryFixture::new();
if case != "unresolved" {
fixture.resolve_candidate();
}
if case != "missing_surface" {
let attempt = if case == "foreign_attempt" {
"other-attempt"
} else {
&fixture.saved.attempt.attempt_id
};
let generation = fixture.generation(attempt);
fixture.publish(&generation);
}
if case == "stale" {
fixture
.body
.host
.catalog
.store
.create_memory(
&fixture.body.binding.subject_id,
subjektiv::MemoryDraft::active(
memory::extract::CandidateKind::Lesson,
"new lesson",
"new body",
"reason",
"change",
),
)
.unwrap();
}
let before_surface = fixture
.body
.host
.catalog
.store
.resident_surface(&fixture.body.binding.subject_id)
.unwrap();
fixture.body.host.recover_jobs();
tokio::task::yield_now().await;
assert_eq!(
fixture.jobs.get(&fixture.saved.request.job_id).unwrap(),
fixture.saved,
"{case}"
);
assert_eq!(
fixture
.body
.host
.catalog
.store
.resident_surface(&fixture.body.binding.subject_id)
.unwrap(),
before_surface,
"{case}"
);
}
}
#[tokio::test]
async fn consolidation_recovery_rejects_retired_input_version_stop_and_released_writer() {
for case in [
"erased",
"invalidated",
"version_stopped",
"released_writer",
] {
let fixture = ConsolidationRecoveryFixture::new();
fixture.resolve_candidate();
let generation = fixture.generation(&fixture.saved.attempt.attempt_id);
fixture.publish(&generation);
match case {
"erased" | "invalidated" => {
fixture
.jobs
.reclaim_input(
&fixture.saved.request.job_id,
&fixture.saved.input_digest,
if case == "erased" {
crate::jobs::JobInputRetirementReason::ExplicitErasure
} else {
crate::jobs::JobInputRetirementReason::LegacyInvalidation
},
)
.unwrap();
}
"version_stopped" => {
// Seed the persisted cutover marker, not a new migration or an
// execution termination proof.
rusqlite::Connection::open(&fixture.body.jobs_path)
.unwrap()
.execute(
"INSERT INTO job_version_stops(job_id) VALUES (?1)",
[&fixture.saved.request.job_id],
)
.unwrap();
}
"released_writer" => {
fixture
.body
.host
.catalog
.store
.release_journal_writer(&fixture.body.host.writer)
.unwrap();
}
_ => unreachable!(),
}
let before = fixture.jobs.get(&fixture.saved.request.job_id).unwrap();
assert!(
fixture
.body
.host
.recover_consolidation_job(&fixture.jobs, &fixture.saved)
.is_err(),
"{case}"
);
fixture.body.host.recover_jobs();
assert_eq!(
fixture.jobs.get(&fixture.saved.request.job_id).unwrap(),
before,
"{case}"
);
assert!(before.result.is_none());
assert!(!before.executions_released());
}
}
#[test]
fn consolidation_recovery_rejects_stale_attempt_and_digest_observations() {
let fixture = ConsolidationRecoveryFixture::new();
fixture.resolve_candidate();
let generation = fixture.generation(&fixture.saved.attempt.attempt_id);
fixture.publish(&generation);
for change_attempt in [false, true] {
let mut stale = fixture.saved.clone();
if change_attempt {
stale.attempt.attempt_id = "other-attempt".into();
} else {
stale.input_digest = "other-digest".into();
}
assert!(
fixture
.body
.host
.recover_consolidation_job(&fixture.jobs, &stale)
.is_err()
);
assert_eq!(
fixture.jobs.get(&fixture.saved.request.job_id).unwrap(),
fixture.saved
);
}
}
#[test]
fn catalog_reopen_preserves_random_scope_subject_and_behavior() {
let temp = tempfile::tempdir().unwrap();
@@ -182,8 +602,16 @@ fn different_local_roots_do_not_authorize_shared_subject_ids() {
#[test]
fn consolidation_operation_policy_is_attempt_bound_and_does_not_inherit_body_rights() {
struct Fence(AtomicBool);
struct Fence(AtomicBool, AtomicBool);
impl crate::jobs::JobAttemptFence for Fence {
fn enter(&self) -> Result<Box<dyn crate::jobs::JobEffectGuard + '_>, String> {
if !self.1.load(Ordering::Acquire) {
return Err("attempt admission closed".into());
}
self.ensure_live()?;
Ok(Box::new(()))
}
fn ensure_live(&self) -> Result<(), String> {
self.0
.load(Ordering::Acquire)
@@ -197,7 +625,7 @@ fn consolidation_operation_policy_is_attempt_bound_and_does_not_inherit_body_rig
let mut body = catalog
.connect(&subject.id, protocol::WorkerId::now_v7())
.unwrap();
let fence = Arc::new(Fence(AtomicBool::new(true)));
let fence = Arc::new(Fence(AtomicBool::new(true), AtomicBool::new(true)));
let job = LocalConsolidationHost {
host: body.host.clone(),
grant: LocalConsolidationGrant {
@@ -255,6 +683,21 @@ fn consolidation_operation_policy_is_attempt_bound_and_does_not_inherit_body_rig
)
.is_err()
);
fence.1.store(false, Ordering::Release);
assert!(
job.authorize_operation(&context, "memory.staging.read")
.is_ok()
);
assert!(matches!(
job.memory(
&context,
SubjektivMemoryBackendOperation::ReadCandidate(SubjektivMemoryCandidateReadRequest {
candidate_id: "outside-batch".into()
})
),
Err(SubjektivHostError::Unavailable(reason)) if reason == "attempt admission closed"
));
fence.1.store(true, Ordering::Release);
fence.0.store(false, Ordering::Release);
assert!(
job.authorize_operation(&context, "memory.staging.list")
@@ -297,6 +740,13 @@ fn session_registration_uses_real_capability_and_attribution_decisions() {
.session_registration_cause(),
SessionRegistrationCause::binding_mismatch(),
);
// Normal factory startup creates the Worker Session store before registration.
// Keep that real filesystem prerequisite without pre-creating attribution.
let _sessions = session_store::FsStore::new_with_retention(
root.join(id.to_string()).join("sessions"),
crate::store::session_retention(&root),
)
.unwrap();
let success = port.record_session(&context, true).unwrap();
assert_eq!(success.subject_id, subject.id);
assert_eq!(success.session_id, context.session_id);
crates/standalone/tests/jobs.rs
+181 -41
View File
@@ -137,6 +137,18 @@ async fn host_job_runs_without_dialog_and_result_survives_restore_and_ack() {
assert_eq!(completed.state, JobState::Completed);
assert_eq!(completed.result, Some(serde_json::json!({"valid":true})));
assert!(!completed.acknowledged);
assert!(completed.executions_released());
assert_eq!(completed.executions.len(), 1);
assert_eq!(
completed.executions[0].run_id,
worker::job::execution_run_id(&intent, &completed.attempt.attempt_id)
.unwrap()
.to_string()
);
assert_eq!(
completed.executions[0].cleanup_receipt.as_deref(),
Some(format!("owned-run:{}", completed.executions[0].run_id).as_str())
);
let neutral = completed.outcome();
assert_eq!(neutral.state, job::JobState::Completed);
assert_eq!(neutral.result, completed.result);
@@ -196,6 +208,46 @@ async fn host_job_runs_without_dialog_and_result_survives_restore_and_ack() {
restored.shutdown().await.unwrap();
}
#[tokio::test]
async fn host_restore_resumes_reserved_result_job_on_same_attempt_without_client_action() {
let temp = tempfile::tempdir().unwrap();
let (host, dialog) = host(&temp).await;
let worker_id = host.worker_id();
let saved = host
.jobs()
.request(request("reserved-before-dispatch"))
.unwrap();
host.shutdown().await.unwrap();
assert!(dialog.requests().is_empty());
let model = success();
let restored = StandaloneHost::restore_with_model_client(
temp.path().join("state"),
worker_id,
model.clone(),
)
.await
.unwrap();
// The production from_started dispatcher, not a test start/retry/probe call,
// must supply the already-accepted immutable input to the result-only Job.
wait_started(&model).await;
let completed = tokio::time::timeout(
Duration::from_secs(10),
restored.jobs().wait("reserved-before-dispatch"),
)
.await
.unwrap()
.unwrap();
assert_eq!(completed.state, JobState::Completed);
assert_eq!(completed.attempt.attempt_id, saved.attempt.attempt_id);
assert_eq!(completed.input_digest, saved.input_digest);
assert_eq!(completed.request, saved.request);
assert_eq!(completed.executions.len(), 1);
assert!(completed.executions_released());
assert_eq!(model.requests().len(), 2);
restored.shutdown().await.unwrap();
}
#[tokio::test]
async fn live_resource_limits_cancellation_and_wait_are_host_owned() {
let temp = tempfile::tempdir().unwrap();
@@ -261,13 +313,13 @@ async fn prose_failure_requires_explicit_bounded_new_attempt_and_exact_replay_do
}
#[tokio::test]
async fn timeout_is_unknown_not_prose_success_or_automatic_retry() {
async fn timeout_fails_without_automatic_retry_and_cleanup_allows_explicit_retry() {
let temp = tempfile::tempdir().unwrap();
let (host, _) = host(&temp).await;
let jobs = host.jobs();
let mut intent = request("timeout");
intent.limits.timeout_seconds = 1;
jobs.request(intent).unwrap();
let reserved = jobs.request(intent.clone()).unwrap();
let stalled = ScriptedClient::stall();
jobs.start_with_model_client("timeout", stalled.clone())
.await
@@ -277,22 +329,45 @@ async fn timeout_is_unknown_not_prose_success_or_automatic_retry() {
.await
.unwrap()
.unwrap();
assert_eq!(outcome.state, JobState::Unknown);
assert_eq!(outcome.state, JobState::Failed);
assert_eq!(
outcome.attempt.state,
standalone::jobs::JobAttemptState::Failed
);
assert_eq!(outcome.attempt.failure.as_deref(), Some("timeout"));
assert_eq!(outcome.attempt.attempt_id, reserved.attempt.attempt_id);
assert_eq!(outcome.request, intent);
assert!(outcome.result.is_none());
assert!(jobs.retry("timeout").is_err());
assert!(outcome.executions_released());
assert_eq!(jobs.request(intent).unwrap(), outcome);
assert_eq!(
stalled.requests().len(),
1,
"timeout never reexecutes the model"
);
let retried = jobs.retry("timeout").unwrap();
assert_eq!(retried.attempt.number, 2);
assert_ne!(retried.attempt.attempt_id, outcome.attempt.attempt_id);
jobs.start_with_model_client("timeout", success())
.await
.unwrap();
assert_eq!(
jobs.wait("timeout").await.unwrap().state,
JobState::Completed
);
host.shutdown().await.unwrap();
}
#[tokio::test]
async fn shutdown_keeps_reserved_results_and_fences_running_attempts_on_reopen() {
async fn shutdown_preserves_reserved_intent_and_reopen_preserves_confirmed_failure() {
let temp = tempfile::tempdir().unwrap();
let (host, _) = host(&temp).await;
let id = host.worker_id();
let jobs = host.jobs();
jobs.request(request("reserved")).unwrap();
let reserved = jobs.request(request("reserved")).unwrap();
jobs.request(request("cancelled")).unwrap();
jobs.cancel("cancelled").unwrap();
jobs.request(request("running")).unwrap();
let cancelled = jobs.cancel("cancelled").unwrap();
let running = jobs.request(request("running")).unwrap();
let stalled = ScriptedClient::stall();
jobs.start_with_model_client("running", stalled.clone())
.await
@@ -300,32 +375,51 @@ async fn shutdown_keeps_reserved_results_and_fences_running_attempts_on_reopen()
wait_started(&stalled).await;
host.shutdown().await.unwrap();
assert!(matches!(jobs.get("running"), Err(JobServiceError::Closed)));
let resume_model = success();
let restored = StandaloneHost::restore_with_model_client(
temp.path().join("state"),
id,
ScriptedClient::new(Vec::new()),
resume_model.clone(),
)
.await
.unwrap();
wait_started(&resume_model).await;
let jobs = restored.jobs();
let resumed = jobs.wait("reserved").await.unwrap();
assert_eq!(resumed.state, JobState::Completed);
assert_eq!(resumed.attempt.attempt_id, reserved.attempt.attempt_id);
assert_eq!(resumed.input_digest, reserved.input_digest);
assert_eq!(resumed.request, reserved.request);
assert!(jobs.pending().unwrap().is_empty());
assert_eq!(jobs.get("cancelled").unwrap(), cancelled);
let stopped = jobs.get("running").unwrap();
assert_eq!(stopped.state, JobState::Failed);
assert_eq!(
jobs.pending()
.unwrap()
.iter()
.map(|snapshot| snapshot.request.job_id.as_str())
.collect::<Vec<_>>(),
["reserved"]
stopped.attempt.state,
standalone::jobs::JobAttemptState::Failed
);
assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled);
assert_eq!(jobs.get("running").unwrap().state, JobState::Unknown);
assert!(jobs.retry("running").is_err());
jobs.start_with_model_client("reserved", success())
assert_eq!(
stopped.attempt.failure.as_deref(),
Some("Host cancellation/shutdown")
);
assert_eq!(stopped.request, running.request);
assert_eq!(stopped.attempt.attempt_id, running.attempt.attempt_id);
assert!(stopped.executions_released());
assert!(
!stopped.version_stopped,
"ordinary shutdown is not a version cutover"
);
let retry = jobs.retry("running").unwrap();
assert_eq!(retry.attempt.number, 2);
jobs.start_with_model_client("running", success())
.await
.unwrap();
assert_eq!(
jobs.wait("reserved").await.unwrap().state,
jobs.wait("running").await.unwrap().state,
JobState::Completed
);
assert_eq!(jobs.get("reserved").unwrap(), resumed);
assert_eq!(resume_model.requests().len(), 2);
restored.shutdown().await.unwrap();
}
@@ -357,7 +451,7 @@ async fn profile_requirements_are_rejected_without_model_execution_or_implicit_f
// The child exits deliberately without destructors to prove actual process-loss
// recovery, rather than treating a hand-edited SQLite fixture as that evidence.
#[tokio::test]
#[ignore = "entry point for process_loss_recovers_durable_jobs_without_replaying_unknown"]
#[ignore = "entry point for process_loss_fails_same_attempt_without_claiming_execution_cleanup"]
async fn process_job_fixture() {
let root = std::path::PathBuf::from(
std::env::var_os("YOI_JOB_PROCESS_FIXTURE_DIR").expect("fixture root"),
@@ -392,12 +486,17 @@ async fn process_job_fixture() {
.await
.unwrap();
wait_started(&stalled).await;
std::fs::write(
root.join("interrupted-snapshot.json"),
serde_json::to_vec(&jobs.get("interrupted").unwrap()).unwrap(),
)
.unwrap();
// Test-owned process only, never the operating Yoi Worker or dogfood daemon.
std::process::exit(23);
}
#[tokio::test]
async fn process_loss_recovers_durable_jobs_without_replaying_unknown() {
async fn process_loss_fails_same_attempt_without_claiming_execution_cleanup() {
let temp = tempfile::tempdir().unwrap();
let output = std::process::Command::new(std::env::current_exe().unwrap())
.args(["--exact", "process_job_fixture", "--ignored", "--nocapture"])
@@ -415,34 +514,73 @@ async fn process_loss_recovers_durable_jobs_without_replaying_unknown() {
.unwrap()
.parse()
.unwrap();
let recovery_model = success();
let host = StandaloneHost::restore_with_model_client(
temp.path().join("state"),
worker_id,
ScriptedClient::new(Vec::new()),
recovery_model.clone(),
)
.await
.unwrap();
let jobs = host.jobs();
assert_eq!(jobs.get("interrupted").unwrap().state, JobState::Unknown);
let persisted: standalone::jobs::JobSnapshot = serde_json::from_slice(
&std::fs::read(temp.path().join("interrupted-snapshot.json")).unwrap(),
)
.unwrap();
let interrupted = jobs.get("interrupted").unwrap();
assert_eq!(interrupted.request, persisted.request);
assert_eq!(interrupted.input_digest, persisted.input_digest);
assert_eq!(interrupted.attempt.attempt_id, persisted.attempt.attempt_id);
assert_eq!(interrupted.attempt.number, persisted.attempt.number);
assert_eq!(interrupted.executions, persisted.executions);
assert_eq!(interrupted.result, persisted.result);
assert_eq!(interrupted.state, JobState::Failed);
assert_eq!(
interrupted.attempt.state,
standalone::jobs::JobAttemptState::Failed
);
assert!(
interrupted
.attempt
.failure
.as_deref()
.unwrap()
.starts_with("host_restart_execution_unconfirmed:")
);
assert_eq!(interrupted.executions.len(), 1);
assert!(!interrupted.executions_released());
assert!(interrupted.executions[0].cleanup_receipt.is_none());
assert!(!interrupted.executions[0].writes_revoked);
assert!(!interrupted.version_stopped);
assert!(jobs.retry("interrupted").is_err());
let replay = success();
assert!(
jobs.start_with_model_client("interrupted", replay.clone())
.await
.is_err()
);
assert!(replay.requests().is_empty());
assert_eq!(jobs.wait("interrupted").await.unwrap(), interrupted);
let completed = jobs.get("accepted").unwrap();
assert_eq!(completed.state, JobState::Completed);
assert_eq!(completed.result, Some(serde_json::json!({"valid":true})));
assert!(!completed.acknowledged);
assert_eq!(jobs.get("cancelled").unwrap().state, JobState::Cancelled);
assert_eq!(jobs.pending().unwrap().len(), 1);
jobs.start_with_model_client("reserved", success())
.await
.unwrap();
wait_started(&recovery_model).await;
let reserved = jobs.wait("reserved").await.unwrap();
assert_eq!(reserved.state, JobState::Completed);
assert_eq!(reserved.attempt.number, 1);
assert_eq!(recovery_model.requests().len(), 2);
assert!(jobs.pending().unwrap().is_empty());
assert_eq!(jobs.get("interrupted").unwrap(), interrupted);
assert_eq!(
jobs.wait("reserved").await.unwrap().state,
JobState::Completed
host.shutdown().await,
Err(standalone::StandaloneShutdownError::JobCleanup)
);
host.shutdown().await.unwrap();
}
#[tokio::test]
async fn model_failure_is_durable_known_failure_with_explicit_successful_retry() {
async fn model_failure_retains_unconfirmed_execution_and_forbids_retry() {
let temp = tempfile::tempdir().unwrap();
let (host, _) = host(&temp).await;
let jobs = host.jobs();
@@ -454,24 +592,26 @@ async fn model_failure_is_durable_known_failure_with_explicit_successful_retry()
jobs.start_with_model_client("model-failure", client)
.await
.unwrap();
let failure = jobs.wait("model-failure").await.unwrap();
assert!(matches!(
jobs.wait("model-failure").await,
Err(JobServiceError::Cleanup(_))
));
let failure = jobs.get("model-failure").unwrap();
assert_eq!(failure.state, JobState::Failed);
assert!(failure.result.is_none());
assert!(
failure
.attempt
.failure
.as_deref()
.unwrap()
.contains("scripted model failure")
);
let next = jobs.retry("model-failure").unwrap();
assert_eq!(next.attempt.number, 2);
jobs.start_with_model_client("model-failure", success())
.await
.unwrap();
assert!(!failure.executions_released());
assert!(jobs.retry("model-failure").is_err());
assert_eq!(jobs.get("model-failure").unwrap(), failure);
assert_eq!(
jobs.wait("model-failure").await.unwrap().state,
JobState::Completed
host.shutdown().await,
Err(standalone::StandaloneShutdownError::JobCleanup)
);
host.shutdown().await.unwrap();
}
crates/subjektiv/src/job.rs
+87
View File
@@ -118,6 +118,93 @@ pub fn validate_result(
})
}
/// Reconstructs only a stored terminal outcome for this exact Job/attempt.
/// Missing work returns `None`; stale, erased, retired or inconsistent evidence is
/// an error. No candidate decision, generation, model call or finalizer is rerun.
/// Hosts still fence current grants and independently retain execution cleanup.
pub fn lookup_result(
store: &SubjektivStore,
subject_id: &str,
batch: &[String],
job_id: &str,
attempt_id: &str,
) -> OperationResult<Option<serde_json::Value>> {
use crate::SubjektivError;
use rusqlite::{OptionalExtension, params};
if batch.len() > CANDIDATE_BATCH_LIMIT {
return Err(OperationError::InvalidInput(
"consolidation batch exceeds limit".into(),
));
}
let result = store.database.try_transaction(|tx| {
crate::require_active_subject(tx, subject_id)?;
crate::validate_label("Job id", job_id)?;
crate::validate_label("attempt id", attempt_id)?;
let mut complete = true;
for id in batch {
let candidate: Option<String> = tx.query_row(
"SELECT record_json FROM staging_records WHERE subject_id=?1 AND candidate_id=?2",
params![subject_id, id], |row| row.get(0)
).optional()?;
let candidate = candidate.ok_or_else(|| SubjektivError::CandidateNotFound(id.clone()))?;
crate::parse_staging(store.workspace_id(), &candidate)?;
let resolution: Option<String> = tx.query_row(
"SELECT resolution_json FROM staging_resolutions WHERE subject_id=?1 AND candidate_id=?2",
params![subject_id, id], |row| row.get(0)
).optional()?;
match resolution {
Some(raw) => {
let saved = crate::parse_resolution(&raw)?;
if saved.subject_id != subject_id || saved.candidate_id != *id {
return Err(SubjektivError::InvalidRecord("consolidation disposition identity mismatch".into()));
}
}
None => complete = false,
}
}
// Use the existing immutable generation binding and terminal fingerprint
// contract, not arrival time or a newly prepared generation. Several
// preparations for the same input may share a stored outcome.
let fingerprint = crate::memory_fingerprint_in_connection(tx, subject_id)?;
let generation: Option<String> = tx.query_row(
"SELECT generation_id FROM surface_generation_runs
WHERE subject_id=?1 AND job_id=?2 AND attempt_id=?3 AND memory_fingerprint=?4
ORDER BY generation_id LIMIT 1",
params![subject_id, job_id, attempt_id, fingerprint], |row| row.get(0)
).optional()?;
let Some(generation_id) = generation else {
let prior = tx.prepare(
"SELECT 1 FROM surface_generation_runs WHERE subject_id=?1 AND job_id=?2 AND attempt_id=?3"
)?.exists(params![subject_id, job_id, attempt_id])?;
if prior {
return Err(SubjektivError::InvalidRecord("consolidation generation input is stale".into()));
}
return Ok(None);
};
let state: Option<(String, String, Option<String>, Option<String>)> = tx.query_row(
"SELECT memory_fingerprint,status,snapshot_id,reason_code FROM surface_generation_state WHERE subject_id=?1",
[subject_id], |row| Ok((row.get(0)?,row.get(1)?,row.get(2)?,row.get(3)?))
).optional()?;
let Some((saved_fingerprint, status, snapshot_id, reason_code)) = state else { return Ok(None) };
if saved_fingerprint != fingerprint || !matches!(status.as_str(), "ready" | "failed") {
return Err(SubjektivError::InvalidRecord("consolidation terminal outcome is stale or inconsistent".into()));
}
if !complete { return Ok(None) }
let mut surface = serde_json::json!({
"availability": status, "generation_id": generation_id, "memory_fingerprint": fingerprint
});
if let Some(id) = snapshot_id { surface["snapshot_id"] = id.into(); }
if let Some(reason) = reason_code { surface["reason_code"] = reason.into(); }
Ok(Some(serde_json::json!({"subject_id":subject_id, "candidate_ids":batch,"surface":surface})))
}).map_err(|error: SubjektivError| OperationError::Store(error.to_string()))?;
let Some(result) = result else {
return Ok(None);
};
validate_result(store, subject_id, batch, job_id, attempt_id, &result)?;
with_dispositions(store, subject_id, batch, &result).map(Some)
}
/// Adds a compact projection of stored dispositions to a validated result.
/// Reasons, change references, and provenance stay in the receipt store, not duplicated in bounded Job results.
/// Neither reapplies candidates nor creates a new ledger.
crates/subjektiv/src/job/tests.rs
+15
View File
@@ -127,6 +127,11 @@ fn result_checks_durable_dispositions_and_exact_job_surface_outcome() {
.contains("unresolved")
);
assert!(with_dispositions(&store, &subject.id, &batch, &ready).is_err());
assert!(
lookup_result(&store, &subject.id, &batch, "job", "attempt")
.unwrap()
.is_none()
);
store
.decide_candidate(
&subject.id,
@@ -143,6 +148,10 @@ fn result_checks_durable_dispositions_and_exact_job_surface_outcome() {
.unwrap();
validate_result(&store, &subject.id, &batch, "job", "attempt", &ready).unwrap();
let enriched = with_dispositions(&store, &subject.id, &batch, &ready).unwrap();
assert_eq!(
lookup_result(&store, &subject.id, &batch, "job", "attempt").unwrap(),
Some(enriched.clone())
);
assert_eq!(
enriched["candidate_dispositions"],
serde_json::json!([{"candidate_id": "candidate", "action": "invalid"}])
@@ -176,6 +185,12 @@ fn result_checks_durable_dispositions_and_exact_job_surface_outcome() {
let failed = serde_json::json!({"subject_id": subject.id, "candidate_ids": batch,
"surface": {"availability": "failed", "generation_id": failure.id, "memory_fingerprint": memory_fingerprint, "reason_code": "model_failure"}});
validate_result(&store, &subject.id, &batch, "job", "attempt", &failed).unwrap();
// The same attempt can prepare again after committing Memory. Recovery must
// select its compatible stored outcome, not its first/last-arriving generation.
assert_eq!(
lookup_result(&store, &subject.id, &batch, "job", "attempt").unwrap(),
Some(with_dispositions(&store, &subject.id, &batch, &failed).unwrap())
);
assert!(validate_result(&store, &subject.id, &batch, "job", "attempt", &ready).is_err());
let mut false_reason = failed;
false_reason["surface"]["reason_code"] = serde_json::json!("invented");
crates/worker-runtime/src/execution.rs
+138
View File
@@ -80,20 +80,86 @@ pub struct WorkerNotificationAck {
pub notification_request_id: String,
}
/// Actual execution-owner termination, not Backend Host drain, writer revocation,
/// disposable-resource cleanup, or completion of the owning domain operation.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerExecutionTermination {
pub scope: WorkerExecutionTerminationScope,
pub worker_ref: WorkerRef,
pub stop_operation_id: WorkerLifecycleOperationId,
pub execution_id: uuid::Uuid,
pub shutdown_command_id: Option<u64>,
/// Matching durable acknowledgements observed by this execution owner;
/// absence does not establish that another submission was unaccepted.
pub submissions: Vec<WorkerSubmissionAck>,
/// Nonempty failures leave Feature scope termination/cleanup unconfirmed,
/// even though the Controller and adapter-owned tasks have joined.
pub feature_failures: Vec<WorkerFeatureShutdownFailure>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkerExecutionTerminationScope {
Controller,
UnpublishedStartupCandidate,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerFeatureShutdownFailure {
pub category: worker::hook::HookErrorCategory,
pub diagnostic: String,
}
impl WorkerExecutionTermination {
pub(crate) fn joined_startup_candidate(
worker_ref: WorkerRef,
stop_operation_id: WorkerLifecycleOperationId,
execution_id: uuid::Uuid,
) -> Self {
Self {
scope: WorkerExecutionTerminationScope::UnpublishedStartupCandidate,
worker_ref,
stop_operation_id,
execution_id,
shutdown_command_id: None,
submissions: Vec::new(),
feature_failures: Vec::new(),
}
}
pub fn confirms_scope_shutdown(
&self,
worker_ref: &WorkerRef,
operation_id: WorkerLifecycleOperationId,
) -> bool {
&self.worker_ref == worker_ref
&& self.stop_operation_id == operation_id
&& self.feature_failures.is_empty()
}
}
/// Typed execution result class. Results are transient operation outcomes and
/// are not persisted as Worker lifecycle authority.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerExecutionResult {
pub operation: WorkerExecutionOperation,
pub outcome: WorkerExecutionOutcome,
/// Actual Controller targeted or observed by this operation. Receipt lookup
/// does not identify the originating run of a receipt retained across restore.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub execution_id: Option<uuid::Uuid>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
/// Durable acceptance, independent of a later operation failure. Receipt
/// reconciliation can retain this on an Errored result; it is not run completion.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub submission: Option<WorkerSubmissionAck>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub notification: Option<WorkerNotificationAck>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub termination: Option<WorkerExecutionTermination>,
/// Projected primary facts. Never stop, cleanup, or retry evidence.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub startup_failure: Option<protocol::startup_failure::StartupFailureAttachment>,
@@ -123,10 +189,12 @@ impl WorkerExecutionResult {
Self {
operation,
outcome: WorkerExecutionOutcome::Accepted,
execution_id: None,
worker_state: None,
message: None,
submission: None,
notification: None,
termination: None,
startup_failure: None,
secondary_failures: Vec::new(),
startup_resources: None,
@@ -149,6 +217,7 @@ impl WorkerExecutionResult {
Self {
operation,
outcome: WorkerExecutionOutcome::Accepted,
execution_id: None,
worker_state: None,
message: None,
submission: Some(WorkerSubmissionAck {
@@ -157,6 +226,7 @@ impl WorkerExecutionResult {
disposition,
}),
notification: None,
termination: None,
startup_failure: None,
secondary_failures: Vec::new(),
startup_resources: None,
@@ -172,12 +242,14 @@ impl WorkerExecutionResult {
Self {
operation,
outcome: WorkerExecutionOutcome::Accepted,
execution_id: None,
worker_state: None,
message: None,
submission: None,
notification: Some(WorkerNotificationAck {
notification_request_id: notification_request_id.into(),
}),
termination: None,
startup_failure: None,
secondary_failures: Vec::new(),
startup_resources: None,
@@ -190,10 +262,12 @@ impl WorkerExecutionResult {
Self {
operation,
outcome: WorkerExecutionOutcome::Busy,
execution_id: None,
worker_state: None,
message: Some(message.into()),
submission: None,
notification: None,
termination: None,
startup_failure: None,
secondary_failures: Vec::new(),
startup_resources: None,
@@ -206,10 +280,12 @@ impl WorkerExecutionResult {
Self {
operation,
outcome: WorkerExecutionOutcome::Rejected,
execution_id: None,
worker_state: None,
message: Some(message.into()),
submission: None,
notification: None,
termination: None,
startup_failure: None,
secondary_failures: Vec::new(),
startup_resources: None,
@@ -222,10 +298,12 @@ impl WorkerExecutionResult {
Self {
operation,
outcome: WorkerExecutionOutcome::Errored,
execution_id: None,
worker_state: None,
message: Some(message.into()),
submission: None,
notification: None,
termination: None,
startup_failure: None,
secondary_failures: Vec::new(),
startup_resources: None,
@@ -238,10 +316,12 @@ impl WorkerExecutionResult {
Self {
operation,
outcome: WorkerExecutionOutcome::Unsupported,
execution_id: None,
worker_state: None,
message: Some(message.into()),
submission: None,
notification: None,
termination: None,
startup_failure: None,
secondary_failures: Vec::new(),
startup_resources: None,
@@ -730,6 +810,7 @@ pub struct WorkerProtocolTransport {
dispatch:
Arc<dyn Fn(Method) -> Result<Vec<protocol::Event>, WorkerExecutionResult> + Send + Sync>,
validate: Arc<dyn Fn() -> Result<(), WorkerExecutionResult> + Send + Sync>,
stop: Option<Arc<dyn Fn(WorkerExecutionStopRequest) -> WorkerExecutionResult + Send + Sync>>,
}
#[cfg(feature = "ws-server")]
@@ -749,6 +830,25 @@ impl WorkerProtocolTransport {
events,
dispatch,
validate,
stop: None,
}
}
pub fn with_stop_dispatch(
mut self,
stop: Arc<dyn Fn(WorkerExecutionStopRequest) -> WorkerExecutionResult + Send + Sync>,
) -> Self {
self.stop = Some(stop);
self
}
pub fn stop(&self, request: WorkerExecutionStopRequest) -> WorkerExecutionResult {
match &self.stop {
Some(stop) => stop(request),
None => WorkerExecutionResult::unsupported(
WorkerExecutionOperation::Stop,
"bound protocol transport cannot confirm scope shutdown",
),
}
}
@@ -1034,6 +1134,29 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
Ok(())
}
/// Look up only the receipt for this exact input (request ID, source namespace
/// and payload digest). Missing or inaccessible receipts remain unknown; this
/// operation must not submit new work.
fn reconcile_submission(
&self,
_worker_ref: &WorkerRef,
_input: WorkerInput,
) -> WorkerExecutionResult {
WorkerExecutionResult::errored(
WorkerExecutionOperation::Input,
"submission acceptance is unknown; exact receipt lookup is unavailable",
)
}
/// Read an actual retained scope-termination receipt. A backend must reject
/// receipts belonging to a replaced execution; absence is not shutdown proof.
fn termination_receipt(
&self,
_worker_ref: &WorkerRef,
) -> Result<Option<WorkerExecutionTermination>, String> {
Err("execution backend has no scope-termination receipt reader".into())
}
fn dispatch_input(&self, worker_ref: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult;
fn upload_file(
@@ -1327,6 +1450,21 @@ impl WorkerExecutionBackendRef {
self.backend.observe_workspace_prompt_projection(projection)
}
pub(crate) fn reconcile_submission(
&self,
worker_ref: &WorkerRef,
input: WorkerInput,
) -> WorkerExecutionResult {
self.backend.reconcile_submission(worker_ref, input)
}
pub(crate) fn termination_receipt(
&self,
worker_ref: &WorkerRef,
) -> Result<Option<WorkerExecutionTermination>, String> {
self.backend.termination_receipt(worker_ref)
}
pub(crate) fn dispatch_input(
&self,
worker_ref: &WorkerRef,
crates/worker-runtime/src/http_server.rs
+23
View File
@@ -3556,6 +3556,29 @@ mod tests {
fn stop_worker(&self, _worker_ref: &WorkerRef) -> WorkerExecutionResult {
WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
}
fn stop_worker_operation(
&self,
request: crate::execution::WorkerExecutionStopRequest,
) -> WorkerExecutionResult {
// Only execution is substituted in these HTTP/retention fixtures.
// Model its bound stop report explicitly; do not relax Runtime's
// production requirement or claim real Controller/Host coverage.
let execution_id =
uuid::Uuid::parse_str(&request.worker_ref.worker_id.to_string()).unwrap();
let mut result = self.stop_worker(&request.worker_ref);
result.execution_id = Some(execution_id);
result.termination = Some(crate::execution::WorkerExecutionTermination {
scope: crate::execution::WorkerExecutionTerminationScope::Controller,
worker_ref: request.worker_ref,
stop_operation_id: request.operation_id,
execution_id,
shutdown_command_id: None,
submissions: Vec::new(),
feature_failures: Vec::new(),
});
result
}
}
#[cfg(feature = "fs-store")]
crates/worker-runtime/src/runtime.rs
+528 -28
View File
@@ -1842,6 +1842,44 @@ impl Runtime {
Ok(worker.detail())
}
/// Look up the originally reserved Job Worker by its saved creation identity.
/// This confirms identity only: it never creates/activates a Controller or
/// proves Submit acceptance, execution termination, or Host drain.
pub fn lookup_job_worker_creation(
&self,
workspace_id: &str,
worker_ref: &WorkerRef,
create_fingerprint: &str,
binding: &crate::catalog::BackendJobExecutionBinding,
) -> Result<Option<WorkerDetail>, RuntimeError> {
if workspace_id.is_empty() || create_fingerprint.is_empty() {
return Err(RuntimeError::InvalidRequest(
"Job creation lookup requires Workspace and fingerprint".into(),
));
}
let state = self.lock()?;
let Some(worker) = state.workers.get(&worker_ref.worker_id) else {
return Ok(None);
};
let request = worker
.request
.as_ref()
.filter(|_| worker.execution_metadata_available)
.ok_or_else(|| RuntimeError::WorkerExecutionUnavailable {
worker_id: worker_ref.worker_id,
message: "saved Job Worker creation metadata unavailable".into(),
})?;
if worker.workspace_id.as_deref() != Some(workspace_id)
|| request.create_fingerprint != create_fingerprint
|| request.backend_job.as_ref() != Some(binding)
{
return Err(RuntimeError::InvalidRequest(
"saved Job Worker creation binding mismatch".into(),
));
}
Ok(Some(worker.detail()))
}
fn ensure_worker_in_workspace(
&self,
scope: &RuntimeWorkspaceScope,
@@ -3217,6 +3255,102 @@ impl Runtime {
self.send_input_with_scope(worker_ref, input, None)
}
/// Read the execution owner's retained termination receipt without stopping,
/// restoring or inferring shutdown from persisted status or process restart.
pub fn execution_termination(
&self,
worker_ref: &WorkerRef,
) -> Result<Option<crate::execution::WorkerExecutionTermination>, RuntimeError> {
let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?;
let _guard = operation_lock
.lock()
.map_err(|_| RuntimeError::StatePoisoned)?;
let backend = {
let state = self.lock()?;
state.ensure_worker_ref(worker_ref)?;
state.worker(worker_ref)?;
state.execution_backend.clone().ok_or_else(|| {
RuntimeError::WorkerExecutionUnavailable {
worker_id: worker_ref.worker_id,
message: "runtime has no execution backend".into(),
}
})?
};
let receipt = backend.termination_receipt(worker_ref).map_err(|message| {
RuntimeError::WorkerExecutionUnavailable {
worker_id: worker_ref.worker_id,
message,
}
})?;
if receipt
.as_ref()
.is_some_and(|receipt| &receipt.worker_ref != worker_ref)
{
return Err(RuntimeError::WorkerExecutionUnavailable {
worker_id: worker_ref.worker_id,
message: "scope-termination receipt Worker mismatch".into(),
});
}
Ok(receipt)
}
/// Read the existing exact Submit receipt without dispatching input, restoring a
/// Controller, changing lifecycle state, or replacing retained execution failures.
/// A missing receipt does not establish nonacceptance.
pub fn reconcile_input(
&self,
worker_ref: &WorkerRef,
input: WorkerInput,
) -> Result<WorkerExecutionResult, RuntimeError> {
validate_worker_input(&input)?;
let expected = input
.submission_request_id
.as_deref()
.filter(|id| !id.trim().is_empty())
.ok_or_else(|| {
RuntimeError::InvalidRequest(
"receipt lookup requires a stable submission request id".into(),
)
})?;
if !matches!(
input.kind,
WorkerInputKind::User | WorkerInputKind::UserIfIdle
) {
return Err(RuntimeError::InvalidRequest(
"receipt lookup requires Submit input".into(),
));
}
let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?;
let _guard = operation_lock
.lock()
.map_err(|_| RuntimeError::StatePoisoned)?;
let backend = {
let state = self.lock()?;
state.ensure_running()?;
state.ensure_worker_ref(worker_ref)?;
state.worker(worker_ref)?;
state.execution_backend.clone().ok_or_else(|| {
RuntimeError::WorkerExecutionUnavailable {
worker_id: worker_ref.worker_id,
message: "runtime has no execution backend".into(),
}
})?
};
let mut receipt = backend.reconcile_submission(worker_ref, input.clone());
if receipt.is_accepted()
&& (receipt.operation != WorkerExecutionOperation::Input
|| receipt.submission.as_ref().is_none_or(|ack| {
ack.submission_request_id != expected || ack.submission_id.trim().is_empty()
}))
{
receipt.outcome = crate::execution::WorkerExecutionOutcome::Errored;
receipt.submission = None;
receipt.message =
Some("exact submission receipt mismatch; acceptance is not established".into());
}
Ok(receipt)
}
fn send_input_with_scope(
&self,
worker_ref: &WorkerRef,
@@ -3283,7 +3417,47 @@ impl Runtime {
})?
};
let dispatch_result = backend.dispatch_input(worker_ref, input.clone());
let mut dispatch_result = backend.dispatch_input(worker_ref, input.clone());
if let Some(expected) = expected_submission_id.as_ref()
&& matches!(
dispatch_result.outcome,
crate::execution::WorkerExecutionOutcome::Accepted
| crate::execution::WorkerExecutionOutcome::Errored
)
&& dispatch_result.submission.as_ref().is_none_or(|ack| {
&ack.submission_request_id != expected || ack.submission_id.trim().is_empty()
})
{
let receipt = backend.reconcile_submission(worker_ref, input.clone());
if receipt.is_accepted()
&& receipt.operation == WorkerExecutionOperation::Input
&& dispatch_result
.execution_id
.is_none_or(|execution_id| receipt.execution_id == Some(execution_id))
&& receipt.submission.as_ref().is_some_and(|ack| {
&ack.submission_request_id == expected && !ack.submission_id.trim().is_empty()
})
{
// Receipt acceptance is independent of a later dispatch failure.
// Keep the original outcome and startup failure attachments intact.
dispatch_result.submission = receipt.submission;
if dispatch_result.is_accepted() && receipt.worker_state.is_some() {
dispatch_result.worker_state = receipt.worker_state;
}
} else {
// A missing receipt, changed Controller or mismatched acknowledgement
// cannot establish rejection. Do not retain a foreign request's ack.
dispatch_result.submission = None;
dispatch_result.outcome = crate::execution::WorkerExecutionOutcome::Errored;
let uncertainty = format!(
"submission acceptance is unknown for request {expected}; exact receipt reconciliation did not confirm acceptance on the same execution"
);
dispatch_result.message = Some(match dispatch_result.message.take() {
Some(message) => format!("{message}; {uncertainty}"),
None => uncertainty,
});
}
}
if !dispatch_result.is_accepted() {
self.record_execution_result(worker_ref, dispatch_result.clone())?;
return Err(RuntimeError::WorkerExecutionRejected {
@@ -3294,25 +3468,6 @@ impl Runtime {
result: dispatch_result,
});
}
if let Some(expected_submission_id) = expected_submission_id
&& dispatch_result
.submission
.as_ref()
.is_none_or(|ack| ack.submission_request_id != expected_submission_id)
{
let result = WorkerExecutionResult::rejected(
WorkerExecutionOperation::Input,
"execution backend did not acknowledge the committed Runtime submission request id",
);
self.record_execution_result(worker_ref, result.clone())?;
return Err(RuntimeError::WorkerExecutionRejected {
worker_id: worker_ref.worker_id.clone(),
operation: result.operation,
outcome: result.outcome,
message: result.message_or_default(),
result,
});
}
if let Some(expected_notification_id) = expected_notification_id
&& dispatch_result
@@ -3805,9 +3960,12 @@ impl Runtime {
if matches!(&method, Method::Shutdown { .. }) {
let operation_id = WorkerLifecycleOperationId::new();
self.begin_stop_operation(worker_ref, operation_id)?;
let events = transport.dispatch(method).map_err(rejection)?;
self.commit_stopped_worker(worker_ref, operation_id)?;
return Ok(events);
let result = transport.stop(WorkerExecutionStopRequest {
operation_id,
worker_ref: worker_ref.clone(),
});
self.commit_stopped_worker(worker_ref, operation_id, result)?;
return Ok(Vec::new());
}
transport.dispatch(method).map_err(rejection)
}
@@ -3852,8 +4010,16 @@ impl Runtime {
backend: &WorkerExecutionBackendRef,
worker_ref: &WorkerRef,
) -> Result<(), RuntimeError> {
let stop_result = backend.stop_worker(worker_ref);
if stop_result.is_accepted() {
let operation_id = WorkerLifecycleOperationId::new();
let stop_result = backend.stop_worker_operation(WorkerExecutionStopRequest {
operation_id,
worker_ref: worker_ref.clone(),
});
if stop_result.is_accepted()
&& stop_result.termination.as_ref().is_some_and(|termination| {
termination.confirms_scope_shutdown(worker_ref, operation_id)
})
{
return self.rollback_failed_create(worker_ref);
}
let mut state = self.lock()?;
@@ -4197,7 +4363,7 @@ impl Runtime {
result,
});
}
self.commit_stopped_worker(worker_ref, operation_id)
self.commit_stopped_worker(worker_ref, operation_id, result)
}
fn admit_worker_stop_under_lock(
@@ -4945,7 +5111,7 @@ impl Runtime {
result,
});
}
self.commit_stopped_worker(worker_ref, operation_id)?;
self.commit_stopped_worker(worker_ref, operation_id, result)?;
Ok(())
}
@@ -5011,7 +5177,27 @@ impl Runtime {
&self,
worker_ref: &WorkerRef,
operation_id: WorkerLifecycleOperationId,
mut result: WorkerExecutionResult,
) -> Result<WorkerLifecycleAck, RuntimeError> {
if !result.is_accepted()
|| result.operation != WorkerExecutionOperation::Stop
|| result.termination.as_ref().is_none_or(|termination| {
!termination.confirms_scope_shutdown(worker_ref, operation_id)
})
{
if result.is_accepted() {
result.outcome = crate::execution::WorkerExecutionOutcome::Errored;
result.message = Some("stop has no matching actual scope termination evidence; reconciliation remains pending".into());
}
self.record_execution_result(worker_ref, result.clone())?;
return Err(RuntimeError::WorkerExecutionRejected {
worker_id: worker_ref.worker_id,
operation: result.operation,
outcome: result.outcome,
message: result.message_or_default(),
result,
});
}
let mut state = self.lock()?;
let mut candidate = state.worker(worker_ref)?.clone();
if candidate
@@ -9030,6 +9216,63 @@ mod tests {
}
}
#[test]
fn job_creation_lookup_checks_saved_identity_without_activating_or_resubmitting() {
let (runtime, backend) = runtime_and_backend();
let mut request = scoped_task_request("saved allocation", "workspace-a");
let binding = crate::catalog::BackendJobExecutionBinding {
job_id: "job-1".into(),
attempt_id: "attempt-1".into(),
input_digest: Some(format!("sha256:{}", "a".repeat(64))),
subjektiv_consolidation: false,
subjektiv_journal: false,
};
request.backend_job = Some(binding.clone());
let worker = runtime
.create_worker_scoped(&scope("workspace-a", "server-a"), request.clone())
.unwrap();
let bindings_before = backend.job_bindings.lock().unwrap().clone();
let activations_before = backend.activation_operation_ids.lock().unwrap().clone();
let query = |workspace: &str,
fingerprint: &str,
binding: &crate::catalog::BackendJobExecutionBinding| {
runtime.lookup_job_worker_creation(workspace, &worker.worker_ref, fingerprint, binding)
};
assert!(
query("workspace-a", &request.create_fingerprint, &binding)
.unwrap()
.is_some()
);
assert!(query("workspace-b", &request.create_fingerprint, &binding).is_err());
assert!(query("workspace-a", "foreign-fingerprint", &binding).is_err());
let mut foreign = binding.clone();
foreign.attempt_id = "attempt-other".into();
assert!(query("workspace-a", &request.create_fingerprint, &foreign).is_err());
foreign = binding.clone();
foreign.input_digest = Some("foreign-input".into());
assert!(query("workspace-a", &request.create_fingerprint, &foreign).is_err());
let mut absent = worker.worker_ref.clone();
absent.worker_id = WorkerId::now_v7();
assert!(
runtime
.lookup_job_worker_creation(
"workspace-a",
&absent,
&request.create_fingerprint,
&binding
)
.unwrap()
.is_none()
);
assert_eq!(*backend.job_bindings.lock().unwrap(), bindings_before);
assert_eq!(
*backend.activation_operation_ids.lock().unwrap(),
activations_before
);
assert!(backend.dispatched_inputs.lock().unwrap().is_empty());
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
}
#[cfg(feature = "fs-store")]
#[test]
fn backend_job_binding_survives_restart_and_restore_without_profile_inference() {
@@ -9565,10 +9808,18 @@ mod tests {
before_spawn_result: Mutex<Option<Box<dyn FnOnce() + Send>>>,
job_bindings: Mutex<Vec<Option<crate::catalog::BackendJobExecutionBinding>>>,
dispatch_result: Mutex<Option<WorkerExecutionResult>>,
submission_receipt_result: Mutex<Option<WorkerExecutionResult>>,
submission_receipt_queries: Mutex<Vec<WorkerInput>>,
stop_result: Mutex<Option<WorkerExecutionResult>>,
stop_gate: Mutex<Option<Arc<RestoreGate>>>,
stop_count: Mutex<u64>,
stop_operation_ids: Mutex<Vec<WorkerLifecycleOperationId>>,
stop_terminations: Mutex<
std::collections::HashMap<
WorkerLifecycleOperationId,
crate::execution::WorkerExecutionTermination,
>,
>,
restore_result: Mutex<Option<WorkerExecutionSpawnResult>>,
preflight_restore_result: Mutex<Option<WorkerExecutionResult>>,
restore_gate: Mutex<Option<Arc<RestoreGate>>>,
@@ -10009,6 +10260,24 @@ mod tests {
}]
}
fn reconcile_submission(
&self,
_worker_ref: &WorkerRef,
input: WorkerInput,
) -> WorkerExecutionResult {
self.submission_receipt_queries.lock().unwrap().push(input);
self.submission_receipt_result
.lock()
.unwrap()
.clone()
.unwrap_or_else(|| {
WorkerExecutionResult::errored(
WorkerExecutionOperation::Input,
"receipt missing; acceptance unknown",
)
})
}
fn dispatch_input(
&self,
worker_ref: &WorkerRef,
@@ -10068,7 +10337,30 @@ mod tests {
.lock()
.unwrap()
.push(request.operation_id);
self.stop_worker(&request.worker_ref)
let configured = self.stop_result.lock().unwrap().is_some();
let mut result = self.stop_worker(&request.worker_ref);
if !configured && result.is_accepted() {
// This catalog fixture substitutes only the backend response;
// real Controller completion/join is tested in worker_backend.
let termination = self
.stop_terminations
.lock()
.unwrap()
.entry(request.operation_id)
.or_insert_with(|| crate::execution::WorkerExecutionTermination {
scope: crate::execution::WorkerExecutionTerminationScope::Controller,
worker_ref: request.worker_ref.clone(),
stop_operation_id: request.operation_id,
execution_id: uuid::Uuid::now_v7(),
shutdown_command_id: None,
submissions: Vec::new(),
feature_failures: Vec::new(),
})
.clone();
result.execution_id = Some(termination.execution_id);
result.termination = Some(termination);
}
result
}
fn stop_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult {
@@ -14796,6 +15088,214 @@ mod tests {
assert_eq!(after.worker_state, Some(fresh));
}
#[test]
fn input_ack_mismatch_queries_exact_receipt_without_resubmitting_or_claiming_rejection() {
for found in [false, true] {
let (runtime, backend) = runtime_and_backend();
let detail = runtime
.create_worker(task_request("receipt reconciliation"))
.unwrap();
backend.preserve_submission_acknowledgement_id();
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
"another-request",
"another-submission",
protocol::SubmissionDisposition::Started,
));
let mut input = WorkerInput::user("same payload");
input.submission_request_id = Some("exact-request".into());
if found {
*backend.submission_receipt_result.lock().unwrap() =
Some(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
"exact-request",
"retained-submission",
protocol::SubmissionDisposition::Started,
));
}
let response = runtime.send_input(&detail.worker_ref, input);
if found {
assert_eq!(
response.unwrap().submission.unwrap().submission_id,
"retained-submission"
);
} else {
assert!(matches!(
response,
Err(RuntimeError::WorkerExecutionRejected {
outcome: crate::execution::WorkerExecutionOutcome::Errored,
..
})
));
}
let queries = backend.submission_receipt_queries.lock().unwrap();
assert_eq!(queries.len(), 1);
assert_eq!(
queries[0].submission_request_id.as_deref(),
Some("exact-request")
);
assert_eq!(queries[0].content, "same payload");
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
assert_eq!(*backend.stop_count.lock().unwrap(), 0);
}
}
#[test]
fn receipt_from_replacement_controller_does_not_repair_original_dispatch_ack() {
let (runtime, backend) = runtime_and_backend();
let detail = runtime
.create_worker(task_request("receipt execution binding"))
.unwrap();
let original_execution_id = uuid::Uuid::now_v7();
let mut dispatched = WorkerExecutionResult::accepted(WorkerExecutionOperation::Input);
dispatched.execution_id = Some(original_execution_id);
backend.set_dispatch_result(dispatched);
let mut receipt = WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
"bound-request",
"retained-submission",
protocol::SubmissionDisposition::Started,
);
receipt.execution_id = Some(uuid::Uuid::now_v7());
*backend.submission_receipt_result.lock().unwrap() = Some(receipt);
let mut input = WorkerInput::user("same payload");
input.submission_request_id = Some("bound-request".into());
let RuntimeError::WorkerExecutionRejected { result, .. } =
runtime.send_input(&detail.worker_ref, input).unwrap_err()
else {
panic!("replacement Controller cannot repair original execution acknowledgement");
};
assert_eq!(
result.outcome,
crate::execution::WorkerExecutionOutcome::Errored
);
assert_eq!(result.execution_id, Some(original_execution_id));
assert!(result.submission.is_none());
assert!(result.termination.is_none());
assert_eq!(backend.submission_receipt_queries.lock().unwrap().len(), 1);
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
}
#[test]
fn reconciled_receipt_preserves_independent_dispatch_failure_facts() {
use protocol::startup_failure::*;
let (runtime, backend) = runtime_and_backend();
let detail = runtime
.create_worker(task_request("receipt and Worker failure"))
.unwrap();
let mut failure = WorkerExecutionResult::errored(
WorkerExecutionOperation::Input,
"post-admission Worker failure",
)
.with_startup_failure(
worker::startup_failure::runtime_startup_attachment(RuntimeStartupCause {
stage: StartupStage::Feature,
reason: RuntimeStartupReason::FeatureFailed,
})
.unwrap(),
);
failure.execution_id = Some(uuid::Uuid::now_v7());
failure
.secondary_failures
.push(StartupSecondaryFailure::Cleanup {
failure: StartupCleanupFailure::Feature,
});
failure.startup_resources = Some(StartupResourceFacts {
resources: StartupResources::WorkerStartup,
cleanup: StartupCleanup::Failed(StartupCleanupFailure::Feature),
durable_effects_possible: true,
});
failure.startup_failure_persistence = Some(StartupPersistenceOutcome::NotSaved);
backend.set_dispatch_result(failure.clone());
let mut receipt = WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
"accepted-before-failure",
"retained-submission",
protocol::SubmissionDisposition::Started,
);
receipt.execution_id = failure.execution_id;
failure.submission = receipt.submission.clone();
*backend.submission_receipt_result.lock().unwrap() = Some(receipt);
let mut input = WorkerInput::user("same payload");
input.submission_request_id = Some("accepted-before-failure".into());
let RuntimeError::WorkerExecutionRejected { result, .. } = runtime
.send_input(&detail.worker_ref, input.clone())
.unwrap_err()
else {
panic!("receipt must not erase a later Worker failure");
};
assert_eq!(result, failure);
assert_eq!(
backend
.submission_receipt_queries
.lock()
.unwrap()
.as_slice(),
&[input]
);
assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1);
assert_eq!(*backend.stop_count.lock().unwrap(), 0);
}
#[test]
fn explicit_receipt_lookup_is_read_only_and_rejects_foreign_acknowledgements() {
let (runtime, backend) = runtime_and_backend();
let detail = runtime
.create_worker(task_request("read-only receipt lookup"))
.unwrap();
let mut input = WorkerInput::user("frozen input");
input.submission_request_id = Some("exact-request".into());
let before = runtime.worker_detail(&detail.worker_ref).unwrap();
let receipts = [None, Some("exact-request"), Some("foreign-request")];
for request_id in receipts {
*backend.submission_receipt_result.lock().unwrap() = request_id.map(|id| {
WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
id,
"retained-submission",
protocol::SubmissionDisposition::Started,
)
});
let result = runtime
.reconcile_input(&detail.worker_ref, input.clone())
.unwrap();
assert_eq!(result.is_accepted(), request_id == Some("exact-request"));
assert_eq!(runtime.worker_detail(&detail.worker_ref).unwrap(), before);
assert!(backend.dispatched_inputs.lock().unwrap().is_empty());
assert_eq!(*backend.stop_count.lock().unwrap(), 0);
}
let queries = backend.submission_receipt_queries.lock().unwrap().len();
input.submission_request_id = None;
assert!(runtime.reconcile_input(&detail.worker_ref, input).is_err());
assert_eq!(
backend.submission_receipt_queries.lock().unwrap().len(),
queries
);
}
#[test]
fn accepted_stop_without_bound_termination_keeps_operation_pending() {
let (runtime, backend) = runtime_and_backend();
let detail = runtime
.create_worker(task_request("stop evidence"))
.unwrap();
backend.set_stop_result(WorkerExecutionResult::accepted(
WorkerExecutionOperation::Stop,
));
let error = runtime.stop_worker(&detail.worker_ref, None).unwrap_err();
assert!(matches!(
error,
RuntimeError::WorkerExecutionRejected {
outcome: crate::execution::WorkerExecutionOutcome::Errored,
..
}
));
let state = runtime.lock().unwrap();
let worker = state.worker(&detail.worker_ref).unwrap();
assert!(worker.pending_stop.is_some());
assert_ne!(worker.status, WorkerStatus::Stopped);
}
#[test]
fn create_worker_rejects_mismatched_submission_acknowledgement() {
let (runtime, backend) = runtime_and_backend();
crates/worker-runtime/src/worker_backend.rs
+1228 -107
View File
File diff suppressed because it is too large Load Diff
crates/worker/src/controller.rs
+46 -7
View File
@@ -42,6 +42,14 @@ use workdir::{
// WorkerHandle — client-facing, Clone-able
// ---------------------------------------------------------------------------
/// Saved acceptance only; neither processing nor execution cleanup is implied.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SubmissionReceipt {
pub submission_request_id: String,
pub submission_id: String,
pub disposition: protocol::SubmissionDisposition,
}
#[derive(Clone)]
pub struct WorkerHandle {
method_tx: mpsc::Sender<Method>,
@@ -61,6 +69,25 @@ pub struct WorkerHandle {
}
impl WorkerHandle {
/// Read the existing receipt without resending input or activating a run.
/// Missing evidence is not proof of rejection and must not authorize replay.
pub fn reconcile_submission(
&self,
submission_request_id: &str,
input: &[protocol::Segment],
source: &protocol::AuthenticatedInputSource,
) -> Result<Option<SubmissionReceipt>, String> {
let namespace = if matches!(source, protocol::AuthenticatedInputSource::UntrustedWire) {
format!("direct:{}", self.session_id)
} else {
source.namespace()
};
self.pending_activations
.lock()
.map_err(|_| "pending activation receipt state is poisoned".to_string())?
.reconcile_submission(submission_request_id, input, &namespace)
}
/// Transport closure belongs to this Controller endpoint, not a WorkerRef.
pub fn protocol_is_closed(&self) -> bool {
self.method_tx.is_closed()
@@ -651,7 +678,15 @@ fn prepare_pending_run<St: Store + Clone>(
// WorkerController — actor that owns a Worker
// ---------------------------------------------------------------------------
pub type ShutdownReceiver = oneshot::Receiver<()>;
/// Sent only after Controller admission, child tasks, Workdir sessions, and observer tasks
/// have closed. Feature cleanup failures are retained separately; this is not Backend Host
/// operation drain, domain writer revocation, or a resource-removal receipt.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ControllerShutdownReport {
pub feature_failures: Vec<crate::hook::HookError>,
}
pub type ShutdownReceiver = oneshot::Receiver<ControllerShutdownReport>;
pub struct WorkerController;
@@ -664,7 +699,7 @@ pub(crate) struct InitializedWorkerController<C: LlmClient, St: Store> {
pub(crate) worker: Worker<C, St>,
pub(crate) handle: WorkerHandle,
method_rx: mpsc::Receiver<Method>,
shutdown_tx: oneshot::Sender<()>,
shutdown_tx: oneshot::Sender<ControllerShutdownReport>,
shutdown_rx: ShutdownReceiver,
registry_reclaim: PreparedSpawnedWorkerRegistryLoad<St>,
shutdown_after_idle: ShutdownAfterIdleRequest,
@@ -858,7 +893,7 @@ impl WorkerController {
// === 1. Initialization (channels / RuntimeDir / worker-immutable
// snapshots / SpawnedWorkerRegistry / alerter attach) ===
let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
let (shutdown_tx, shutdown_rx) = oneshot::channel::<ControllerShutdownReport>();
let (method_tx, method_rx) = mpsc::channel::<Method>(32);
let (working_event_tx, _) = broadcast::channel::<Event>(256);
let alerter = Alerter::new(working_event_tx.clone());
@@ -2380,7 +2415,7 @@ async fn controller_loop<C, St>(
pause_tx: mpsc::Sender<()>,
notify_buffer: NotifyBuffer,
spawned_registry: Arc<SpawnedWorkerRegistry>,
shutdown_tx: oneshot::Sender<()>,
shutdown_tx: oneshot::Sender<ControllerShutdownReport>,
shutdown_after_idle: ShutdownAfterIdleRequest,
mut command_observer: Option<tokio::task::JoinHandle<()>>,
) where
@@ -3245,7 +3280,11 @@ async fn controller_loop<C, St>(
drop(method_rx);
// Feature callbacks and tasks share the Worker scope. Stop them before
// Memory/Workdir teardown so they cannot observe a partially closed Worker.
worker.stop_feature_runtime("controller shutdown").await;
let feature_failures = worker
.stop_feature_runtime_report("controller shutdown")
.await
.err()
.unwrap_or_default();
loop {
match spawned_registry.shutdown_internal().await {
@@ -3275,7 +3314,7 @@ async fn controller_loop<C, St>(
let _ = observer.await;
}
let _ = shutdown_tx.send(());
let _ = shutdown_tx.send(ControllerShutdownReport { feature_failures });
}
/// Stage an agent-visible Internal SubWorker lifecycle event.
@@ -5524,7 +5563,7 @@ mod tests {
fn controller_shutdown_orders_child_cleanup_before_workdir_close() {
let source = include_str!("controller.rs");
let shutdown_start = source
.rfind("worker.stop_feature_runtime(\"controller shutdown\")")
.rfind(".stop_feature_runtime_report(\"controller shutdown\")")
.expect("controller shutdown block");
let shutdown = &source[shutdown_start..];
let children = shutdown
crates/worker/src/hook.rs
+155 -7
View File
@@ -947,19 +947,36 @@ impl HookRegistry {
&self,
context: &WorkerStoppingContext,
) -> Result<(), HookExecutionError> {
let mut failures = Vec::new();
self.on_worker_stopping_report(context, &mut failures).await;
failures
.into_iter()
.find(|error| error.policy == HookFailurePolicy::FailClosed)
.map_or(Ok(()), Err)
}
/// Run the same policy-governed chain once, retaining even attention/fail-open errors.
/// Continuing a lifecycle chain is not evidence that its cleanup succeeded. This report
/// does not infer failures from the bounded, shared diagnostics buffer. The caller owns
/// collected failures even if its chain deadline cancels the current callback.
pub(crate) async fn on_worker_stopping_report(
&self,
context: &WorkerStoppingContext,
failures: &mut Vec<HookExecutionError>,
) {
for registration in &self.worker_stopping {
if let Err(error) = registration.call(context).await {
self.record_diagnostic(error.clone());
match error.policy {
HookFailurePolicy::FailOpenWithDiagnostic
| HookFailurePolicy::AttentionRequired => {
tracing::warn!(owner = %error.owner, error = %error.source, "worker-stopping hook requires attention");
}
HookFailurePolicy::FailClosed => return Err(error),
let fail_closed = error.policy == HookFailurePolicy::FailClosed;
if !fail_closed {
tracing::warn!(owner = %error.owner, error = %error.source, "worker-stopping hook requires attention");
}
failures.push(error);
if fail_closed {
break;
}
}
}
Ok(())
}
}
@@ -1208,6 +1225,137 @@ mod tests {
assert_eq!(error.category, HookErrorCategory::InvalidInput);
}
#[derive(Clone)]
struct FailingStoppingHook {
calls: Arc<Mutex<Vec<&'static str>>>,
name: &'static str,
entered: Option<Arc<tokio::sync::Notify>>,
}
#[async_trait]
impl Hook<WorkerStopping> for FailingStoppingHook {
async fn call(&self, _: &WorkerStoppingContext) -> Result<(), HookError> {
self.calls.lock().unwrap().push(self.name);
if let Some(entered) = &self.entered {
entered.notify_one();
return std::future::pending().await;
}
Err(HookError::new(HookErrorCategory::Dependency, self.name))
}
}
fn stopping_context() -> WorkerStoppingContext {
WorkerStoppingContext {
invocation: rewrite_context().invocation,
reason: "owned shutdown".into(),
}
}
#[tokio::test]
async fn stopping_report_retains_all_policies_once_and_preserves_fail_closed_order() {
let calls = Arc::new(Mutex::new(Vec::new()));
let mut builder = HookRegistryBuilder::new();
for (name, failure) in [
("attention", HookFailurePolicy::AttentionRequired),
("open", HookFailurePolicy::FailOpenWithDiagnostic),
("closed", HookFailurePolicy::FailClosed),
("unreached", HookFailurePolicy::AttentionRequired),
] {
builder
.add_named_worker_stopping(
name,
HookExecutionPolicy::new(failure, 1_000),
FailingStoppingHook {
calls: calls.clone(),
name,
entered: None,
},
)
.unwrap();
}
let registry = builder.build();
let mut failures = Vec::new();
registry
.on_worker_stopping_report(&stopping_context(), &mut failures)
.await;
assert_eq!(*calls.lock().unwrap(), ["attention", "open", "closed"]);
assert_eq!(
failures
.iter()
.map(|error| error.source.diagnostic.as_str())
.collect::<Vec<_>>(),
["attention", "open", "closed"]
);
assert!(
failures
.iter()
.all(|error| error.source.category == HookErrorCategory::Dependency)
);
assert_eq!(registry.diagnostics().len(), 3);
}
#[tokio::test]
async fn stopping_compatibility_result_keeps_fail_open_policy_without_repeating_callback() {
for failure in [
HookFailurePolicy::AttentionRequired,
HookFailurePolicy::FailOpenWithDiagnostic,
HookFailurePolicy::FailClosed,
] {
let calls = Arc::new(Mutex::new(Vec::new()));
let mut builder = HookRegistryBuilder::new();
builder
.add_named_worker_stopping(
"stop",
HookExecutionPolicy::new(failure, 1_000),
FailingStoppingHook {
calls: calls.clone(),
name: "stop",
entered: None,
},
)
.unwrap();
let registry = builder.build();
let result = registry.on_worker_stopping(&stopping_context()).await;
assert_eq!(result.is_err(), failure == HookFailurePolicy::FailClosed);
assert_eq!(*calls.lock().unwrap(), ["stop"]);
assert_eq!(registry.diagnostics().len(), 1);
}
}
#[tokio::test]
async fn cancelling_stopping_chain_preserves_failures_already_collected_by_owner() {
let calls = Arc::new(Mutex::new(Vec::new()));
let entered = Arc::new(tokio::sync::Notify::new());
let mut builder = HookRegistryBuilder::new();
for (name, pending) in [("completed-failure", false), ("in-flight", true)] {
builder
.add_named_worker_stopping(
name,
HookExecutionPolicy::new(HookFailurePolicy::AttentionRequired, 1_000),
FailingStoppingHook {
calls: calls.clone(),
name,
entered: pending.then(|| entered.clone()),
},
)
.unwrap();
}
let registry = builder.build();
let context = stopping_context();
let mut failures = Vec::new();
{
let chain = registry.on_worker_stopping_report(&context, &mut failures);
tokio::pin!(chain);
tokio::select! {
_ = &mut chain => panic!("second callback must remain in flight"),
_ = entered.notified() => {},
}
}
assert_eq!(*calls.lock().unwrap(), ["completed-failure", "in-flight"]);
assert_eq!(failures.len(), 1);
assert_eq!(failures[0].source.diagnostic, "completed-failure");
}
#[test]
fn hook_diagnostics_are_utf8_bounded() {
let error = HookError::new(HookErrorCategory::Internal, "界".repeat(1_000));
crates/worker/src/internal_worker.rs
+44 -6
View File
@@ -107,6 +107,40 @@ pub(crate) async fn run_internal_worker_with_cancel_sender<F>(
spec: InternalWorkerSpec,
on_cancel_sender: F,
) -> Result<InternalWorkerResult, InternalWorkerError>
where
F: FnOnce(tokio::sync::mpsc::Sender<()>),
{
run_internal_worker_with_cancel_sender_impl(spec, on_cancel_sender, &mut None).await
}
pub(crate) struct InternalWorkerExecution {
pub result: Result<InternalWorkerResult, InternalWorkerError>,
// None means construction did not return a Worker whose Feature runtime could be joined.
pub feature_shutdown: Option<Result<(), Vec<crate::hook::HookError>>>,
}
pub(crate) async fn run_internal_worker_owned<F>(
spec: InternalWorkerSpec,
on_cancel_sender: F,
) -> InternalWorkerExecution
where
F: FnOnce(tokio::sync::mpsc::Sender<()>),
{
let mut feature_shutdown = None;
let result =
run_internal_worker_with_cancel_sender_impl(spec, on_cancel_sender, &mut feature_shutdown)
.await;
InternalWorkerExecution {
result,
feature_shutdown,
}
}
async fn run_internal_worker_with_cancel_sender_impl<F>(
spec: InternalWorkerSpec,
on_cancel_sender: F,
feature_shutdown: &mut Option<Result<(), Vec<crate::hook::HookError>>>,
) -> Result<InternalWorkerResult, InternalWorkerError>
where
F: FnOnce(tokio::sync::mpsc::Sender<()>),
{
@@ -208,9 +242,11 @@ where
.copied()
.collect::<Vec<_>>()
.join(", ");
worker
.stop_feature_runtime("internal Worker feature installation failed")
.await;
*feature_shutdown = Some(
worker
.stop_feature_runtime_report("internal Worker feature installation failed")
.await,
);
return Err(InternalWorkerError {
source: WorkerError::FeatureInstall(format!(
"internal Worker feature installation failed: {diagnostics}; missing tools: {missing}"
@@ -255,9 +291,11 @@ where
history_entries: store.entries_count(session_id, segment_id),
}),
};
worker
.stop_feature_runtime("internal Worker terminal outcome")
.await;
*feature_shutdown = Some(
worker
.stop_feature_runtime_report("internal Worker terminal outcome")
.await,
);
outcome
}
crates/worker/src/job.rs
+153 -30
View File
@@ -6,9 +6,10 @@
//! not a final answer or completion of execution.
//!
//! The Host manages concurrency, timeouts, and retries, and awaits `run` and resource release even
//! after cancellation. Emergency drop or abort cannot confirm asynchronous cleanup, so leases and
//! unknown-outcome state remain. The result Feature has no background work, but dropping a tool
//! future cannot interrupt a running synchronous sink. The Host must also bound the sink's work.
//! after cancellation. Emergency drop or abort cannot confirm asynchronous cleanup, so the Host
//! retains the exact execution obligations and their diagnosis. The result Feature has no background
//! work, but dropping a tool future cannot interrupt a running synchronous sink. The Host must also
//! bound the sink's work.
mod result;
pub use result::{JobResultFeature, JobResultFinalizer, JobResultSink, SUBMIT_JOB_RESULT_TOOL};
@@ -18,11 +19,11 @@ use std::sync::Arc;
use agen::llm_client::LlmClient;
use manifest::{ProfileDiscovery, ProfileResolveOptions, ProfileResolver, Scope, WorkerManifest};
use sha2::{Digest, Sha256};
use crate::feature::FeatureRegistryBuilder;
use crate::internal_worker::{
InternalWorkerAuthority, InternalWorkerIdentity, InternalWorkerSpec,
run_internal_worker_with_cancel_sender,
InternalWorkerAuthority, InternalWorkerIdentity, InternalWorkerSpec, run_internal_worker_owned,
};
use crate::{
PromptCatalog, PromptCatalogSource, SystemPromptContext, SystemPromptTemplate,
@@ -48,7 +49,7 @@ pub enum JobExecutionError {
#[error("Job model configuration failed: {0}")]
Model(#[from] crate::ProviderError),
#[error("Job execution failed: {0}")]
Worker(String),
Worker(#[source] Box<crate::WorkerError>),
/// A sink error can follow durable acceptance. Hosts must reconcile their store and must
/// not classify this as a known retryable failure unless they can prove non-acceptance.
#[error(
@@ -68,6 +69,76 @@ pub struct InternalJobResult {
pub usage: Option<agen::timeline::event::UsageEvent>,
}
/// Result acceptance and owned Feature shutdown from one exact internal execution.
///
/// Feature shutdown covers the Worker-managed stopping hooks and background registry only.
/// Hosts must separately join/revoke their domain callbacks and persistence operations before
/// releasing an attempt. Neither a dropped future nor this value's absence is shutdown evidence.
#[derive(Debug)]
pub struct OwnedInternalJobExecution {
pub job_id: String,
pub attempt_id: String,
pub input_digest: String,
pub run_id: uuid::Uuid,
pub result: Result<InternalJobResult, JobExecutionError>,
unconfirmed_submission: Option<::job::JobResultSubmission>,
post_result_failure: Option<Box<crate::WorkerError>>,
feature_shutdown: Option<Result<(), Vec<crate::hook::HookError>>>,
}
impl OwnedInternalJobExecution {
/// The exact sealed submission whose sink acknowledgement failed, after finalization and
/// result validation. The Host can reconcile/retry only this payload against its saved
/// attempt; this is not accepted output or authority to rerun the finalizer/model.
pub fn unconfirmed_submission(&self) -> Option<&::job::JobResultSubmission> {
self.unconfirmed_submission.as_ref()
}
pub fn feature_shutdown_complete(&self) -> bool {
matches!(self.feature_shutdown, Some(Ok(())))
}
pub fn feature_shutdown_failures(&self) -> &[crate::hook::HookError] {
match &self.feature_shutdown {
Some(Err(errors)) => errors,
_ => &[],
}
}
/// Existing typed cleanup ownership is distinct from a failed model/Run diagnosis. False
/// is not termination proof: callers still require the independently joined Feature scope
/// and their exact Host effects. Construction and compaction cleanup must retain their owner.
pub fn requires_worker_cleanup_reconciliation(&self) -> bool {
fn retained(error: &crate::WorkerError) -> bool {
use crate::WorkerError;
match error {
WorkerError::ConstructionCleanup { .. }
| WorkerError::CompactionCleanupPending { .. } => true,
WorkerError::StartupFailure { source, .. }
| WorkerError::RunFailureAlreadyDelivered { source }
| WorkerError::RestoreActivationEffects { source } => retained(source),
WorkerError::RunTerminalPersistence {
run_failure,
persist_error,
} => retained(run_failure) || retained(persist_error),
_ => false,
}
}
self.worker_failure().is_some_and(retained)
}
/// A later Worker failure does not invalidate durable result acceptance. Keep its typed
/// cause (including startup attachments and retained cleanup handles) for the Host owner.
pub fn worker_failure(&self) -> Option<&crate::WorkerError> {
self.post_result_failure
.as_deref()
.or_else(|| match &self.result {
Err(JobExecutionError::Worker(error)) => Some(error.as_ref()),
_ => None,
})
}
}
/// Resolved policy/model/prompt, with no running resources and no inherited authority.
pub struct PreparedInternalJob {
request: ::job::JobRequest,
@@ -196,11 +267,32 @@ fn prepare_job_from_registry_with_grant(
})
}
/// Stable internal execution identity for the immutable attempt. Hosts can persist this binding
/// before dispatch; replay of a receipt must never start a second execution with the same identity.
pub fn execution_run_id(
request: &::job::JobRequest,
attempt_id: &str,
) -> Result<uuid::Uuid, ::job::JobError> {
request.validate()?;
::job::validate_identifier("attempt_id", attempt_id, 512)?;
let identity = ::job::fingerprint(&(
"internal-job-execution",
&request.job_id,
attempt_id,
request.input_digest()?,
))?;
let digest = Sha256::digest(identity.as_bytes());
let mut bytes = [0; 16];
bytes.copy_from_slice(&digest[..16]);
// UUIDv8 reserves application-defined identity bits; this is not a random run token.
bytes[6] = (bytes[6] & 0x0f) | 0x80;
bytes[8] = (bytes[8] & 0x3f) | 0x80;
Ok(uuid::Uuid::from_bytes(bytes))
}
impl PreparedInternalJob {
/// Run one immutable attempt. The sink must perform fenced, durable acceptance before
/// returning Ok. Timeout is Host-owned: cancel and then await this future for cleanup.
/// A result already accepted by the sink remains success even if the model subsequently
/// fails, reaches its turn limit or is cancelled; callers never rerun an accepted result.
/// Return result acceptance only. This convenience API does not prove Feature or Host
/// shutdown; resource-owning callers must use [Self::run_owned] and their domain fence.
pub async fn run<F>(
self,
attempt_id: String,
@@ -210,6 +302,26 @@ impl PreparedInternalJob {
where
F: FnOnce(tokio::sync::mpsc::Sender<()>),
{
self.run_owned(attempt_id, submit, on_cancel_sender)
.await?
.result
}
/// Await the actual run and Feature shutdown, independently of accepted model output.
/// Timeout is Host-owned: cancel and keep owning this future and domain operations until
/// their separate termination/write-revocation evidence has been observed.
pub async fn run_owned<F>(
self,
attempt_id: String,
submit: Arc<dyn JobResultSink>,
on_cancel_sender: F,
) -> Result<OwnedInternalJobExecution, JobExecutionError>
where
F: FnOnce(tokio::sync::mpsc::Sender<()>),
{
let run_id = execution_run_id(&self.request, &attempt_id)?;
let job_id = self.request.job_id.clone();
let input_digest = self.request.input_digest()?;
let mut feature = JobResultFeature::new(&self.request, attempt_id.clone(), submit)?;
if let Some(finalizer) = self.finalizer {
feature =
@@ -220,7 +332,7 @@ impl PreparedInternalJob {
let spec = InternalWorkerSpec {
identity: InternalWorkerIdentity {
kind: "job",
run_id: uuid::Uuid::now_v7(),
run_id,
},
max_turns: engine_policy.max_turns.map(|n| n.get()),
compaction_budget: None,
@@ -241,29 +353,40 @@ impl PreparedInternalJob {
workdir_session: None,
},
};
let outcome = run_internal_worker_with_cancel_sender(spec, on_cancel_sender).await;
let execution = run_internal_worker_owned(spec, on_cancel_sender).await;
feature.revoke_and_join().await;
let outcome = execution.result;
let usage = match &outcome {
Ok(run) => run.usage.clone(),
Err(error) => error.usage.clone(),
};
if let Some(submission) = accepted.lock().expect("Job result state poisoned").clone() {
return Ok(InternalJobResult { submission, usage });
}
if let Some(error) = feature.unconfirmed() {
return Err(JobExecutionError::ResultUnconfirmed(error));
}
match outcome {
Err(error) => Err(JobExecutionError::Worker(error.source.to_string())),
Ok(run)
if matches!(
run.lifecycle,
WorkerRunResult::Cancelled | WorkerRunResult::RolledBack
) =>
{
Err(JobExecutionError::Cancelled)
}
Ok(_) => Err(JobExecutionError::MissingResult),
}
let lifecycle = outcome.as_ref().ok().map(|run| run.lifecycle.clone());
let mut worker_failure = outcome.err().map(|error| Box::new(error.source));
let submission = accepted.lock().expect("Job result state poisoned").clone();
let result = if let Some(submission) = submission {
Ok(InternalJobResult { submission, usage })
} else if let Some(error) = feature.unconfirmed() {
Err(JobExecutionError::ResultUnconfirmed(error))
} else if let Some(error) = worker_failure.take() {
Err(JobExecutionError::Worker(error))
} else if matches!(
lifecycle,
Some(WorkerRunResult::Cancelled | WorkerRunResult::RolledBack)
) {
Err(JobExecutionError::Cancelled)
} else {
Err(JobExecutionError::MissingResult)
};
Ok(OwnedInternalJobExecution {
job_id,
attempt_id,
input_digest,
run_id,
result,
unconfirmed_submission: feature.unconfirmed_submission(),
post_result_failure: worker_failure,
feature_shutdown: execution.feature_shutdown,
})
}
}
crates/worker/src/job/result.rs
+24 -5
View File
@@ -124,6 +124,24 @@ impl JobResultFeature {
.clone()
}
pub(super) fn unconfirmed_submission(&self) -> Option<::job::JobResultSubmission> {
let state = self.state.lock().expect("Job submission state poisoned");
state.unconfirmed.as_ref()?;
Some(::job::JobResultSubmission {
job_id: self.job_id.clone(),
attempt_id: self.attempt_id.clone(),
input_digest: self.input_digest.clone(),
result: state.prepared.clone()?,
})
}
/// Revoke every clone and join the exact serialized finalizer/sink operation. Cancellation
/// alone does not prove a synchronous Host sink or an asynchronous finalizer has returned.
pub(super) async fn revoke_and_join(&self) {
self.cancellation.cancel();
let _operation = self.serial.lock().await;
}
fn definition(&self) -> ToolDefinition {
let feature = self.clone();
Arc::new(move || {
@@ -190,11 +208,12 @@ impl Tool for JobResultFeature {
::job::result_digest(&prepared, self.max_result_bytes)
.map_err(|e| ToolError::InvalidArgument(e.to_string()))?;
let mut state = self.state.lock().expect("Job submission state poisoned");
if self
.cancelled
.lock()
.expect("Job cancellation state poisoned")
.contains(&context.execution_id())
if self.cancellation.is_cancelled()
|| self
.cancelled
.lock()
.expect("Job cancellation state poisoned")
.contains(&context.execution_id())
{
return Err(ToolError::Cancelled(ToolOutput {
summary: "Job result submission cancelled".into(),
crates/worker/src/job/tests.rs
+271 -2
View File
@@ -565,7 +565,7 @@ async fn accepted_result_survives_later_provider_error_and_cancellation() {
let (tx, rx) = tokio::sync::oneshot::channel();
let recording = sink.clone();
let run = tokio::spawn(async move {
job.run("attempt".into(), recording, move |sender| {
job.run_owned("attempt".into(), recording, move |sender| {
let _ = tx.send(sender);
})
.await
@@ -585,7 +585,22 @@ async fn accepted_result_survives_later_provider_error_and_cancellation() {
.unwrap()
.unwrap()
.unwrap();
assert_eq!(result.submission.result, json!(true));
assert_eq!(result.job_id, "job-a");
assert_eq!(result.attempt_id, "attempt");
assert_eq!(
result.input_digest,
request("user:a").input_digest().unwrap()
);
assert!(result.feature_shutdown_complete());
assert!(result.feature_shutdown_failures().is_empty());
assert!(!result.requires_worker_cleanup_reconciliation());
if !cancel {
assert!(matches!(
result.worker_failure(),
Some(crate::WorkerError::Engine(_))
));
}
assert_eq!(result.result.unwrap().submission.result, json!(true));
assert_eq!(sink.0.lock().unwrap().len(), 1);
assert!(sender.is_closed());
}
@@ -1237,6 +1252,39 @@ async fn unconfirmed_sink_acceptance_is_not_classified_as_missing_result_or_know
assert_eq!(sink.0.lock().unwrap().len(), 1);
}
#[tokio::test]
async fn unconfirmed_sink_retains_exact_finalized_payload_without_claiming_acceptance() {
let root = tempfile::tempdir().unwrap();
let finalizer = Arc::new(ControlledFinalizer::default());
let sink = Arc::new(UnknownSink(Mutex::new(vec![])));
let mut job = prepared(
root.path(),
&request("user:a"),
ScriptedClient::new(vec![
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{"claim":true}})),
prose(),
]),
);
job.finalizer = Some(finalizer.clone());
let execution = job
.run_owned("attempt".into(), sink.clone(), |_| {})
.await
.unwrap();
assert!(matches!(
&execution.result,
Err(JobExecutionError::ResultUnconfirmed(_))
));
let observed = sink.0.lock().unwrap();
assert_eq!(observed.len(), 1);
assert_eq!(execution.unconfirmed_submission(), observed.first());
assert_eq!(
execution.unconfirmed_submission().unwrap().result,
json!({"prepared":{"claim":true},"completion":1})
);
assert_eq!(finalizer.calls.load(Ordering::SeqCst), 1);
assert!(execution.feature_shutdown_complete());
}
#[tokio::test]
async fn feature_registration_rejects_forged_fences_seals_unknown_outcomes_and_honors_cancel() {
let sink = Arc::new(UnknownSink(Mutex::new(vec![])));
@@ -1282,3 +1330,224 @@ async fn feature_registration_rejects_forged_fences_seals_unknown_outcomes_and_h
));
assert_eq!(sink.0.lock().unwrap().len(), 2);
}
#[tokio::test]
async fn revocation_joins_finalizer_cleanup_and_disables_every_result_clone() {
let root = tempfile::tempdir().unwrap();
let prepared = prepared(root.path(), &request("user:a"), ScriptedClient::new(vec![]));
let finalizer = Arc::new(ControlledFinalizer {
gated: true,
..Default::default()
});
let sink = Arc::new(RecordingSink::default());
let feature = JobResultFeature::new(&request("user:a"), "attempt".into(), sink.clone())
.unwrap()
.with_finalizer(finalizer.clone(), prepared.manifest, prepared.client);
let executing = feature.clone();
let operation = tokio::spawn(async move {
executing
.execute(r#"{"result":true}"#, ToolExecutionContext::direct())
.await
});
signalled(&finalizer.entered).await;
let stopping = feature.clone();
let shutdown = tokio::spawn(async move { stopping.revoke_and_join().await });
signalled(&finalizer.cancellation_seen).await;
assert!(
!shutdown.is_finished(),
"cancellation is not cleanup completion"
);
assert_eq!(finalizer.active.load(Ordering::SeqCst), 1);
assert!(sink.0.lock().unwrap().is_empty());
finalizer.cleanup.notify_one();
assert!(matches!(
operation.await.unwrap(),
Err(ToolError::Cancelled(_))
));
tokio::time::timeout(std::time::Duration::from_secs(2), shutdown)
.await
.unwrap()
.unwrap();
assert_eq!(finalizer.active.load(Ordering::SeqCst), 0);
assert!(matches!(
feature
.execute(r#"{"result":true}"#, ToolExecutionContext::direct())
.await,
Err(ToolError::Cancelled(_))
));
assert_eq!(finalizer.calls.load(Ordering::SeqCst), 1);
assert!(sink.0.lock().unwrap().is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn revocation_does_not_claim_a_running_synchronous_sink_has_ended() {
struct BlockingSink {
entered: tokio::sync::Notify,
release: Mutex<std::sync::mpsc::Receiver<()>>,
accepted: AtomicUsize,
}
impl JobResultSink for BlockingSink {
fn submit(&self, _: ::job::JobResultSubmission) -> Result<(), String> {
self.entered.notify_one();
self.release
.lock()
.unwrap()
.recv_timeout(std::time::Duration::from_secs(5))
.expect("test must release the synchronous sink within its finite budget");
self.accepted.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
let (release, receiver) = std::sync::mpsc::channel();
let sink = Arc::new(BlockingSink {
entered: Default::default(),
release: Mutex::new(receiver),
accepted: AtomicUsize::new(0),
});
let feature =
JobResultFeature::new(&request("user:a"), "attempt".into(), sink.clone()).unwrap();
let executing = feature.clone();
let operation = tokio::spawn(async move {
executing
.execute(r#"{"result":true}"#, ToolExecutionContext::direct())
.await
});
signalled(&sink.entered).await;
let stopping = feature.clone();
let started = Arc::new(tokio::sync::Notify::new());
let observed = started.clone();
let shutdown = tokio::spawn(async move {
observed.notify_one();
stopping.revoke_and_join().await;
});
signalled(&started).await;
assert!(
!shutdown.is_finished(),
"a live sink still owns its Host effect"
);
assert_eq!(sink.accepted.load(Ordering::SeqCst), 0);
release.send(()).unwrap();
operation.await.unwrap().unwrap();
tokio::time::timeout(std::time::Duration::from_secs(2), shutdown)
.await
.unwrap()
.unwrap();
assert_eq!(sink.accepted.load(Ordering::SeqCst), 1);
assert!(matches!(
feature
.execute(r#"{"result":true}"#, ToolExecutionContext::direct())
.await,
Err(ToolError::Cancelled(_))
));
assert_eq!(sink.accepted.load(Ordering::SeqCst), 1);
}
#[derive(Clone)]
struct StoppingFeature {
entered: Arc<tokio::sync::Notify>,
release: Arc<tokio::sync::Notify>,
policy: crate::hook::HookFailurePolicy,
failure: Option<crate::hook::HookError>,
}
#[async_trait]
impl crate::hook::Hook<crate::hook::WorkerStopping> for StoppingFeature {
async fn call(
&self,
_: &crate::hook::WorkerStoppingContext,
) -> Result<(), crate::hook::HookError> {
self.entered.notify_one();
self.release.notified().await;
self.failure.clone().map_or(Ok(()), Err)
}
}
impl FeatureModule for StoppingFeature {
fn descriptor(&self) -> FeatureDescriptor {
FeatureDescriptor::builtin("job-stop-observation", "Owned stopping observation").with_hook(
crate::feature::HookDeclaration::new(
"stop",
crate::feature::FeatureHookPoint::WorkerStopping,
),
)
}
fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> {
context.hooks().add_worker_stopping(
"stop",
crate::hook::HookExecutionPolicy::new(self.policy, 1_000),
self.clone(),
)
}
}
// Production Internal Worker/Feature lifecycle and result capability, with only the model
// transport and one stopping callback replaced. This is not a process-termination test.
#[tokio::test]
async fn accepted_result_waits_for_stopping_hook_and_retains_every_failure_policy() {
use crate::hook::{HookError, HookErrorCategory, HookFailurePolicy};
for (policy, failure) in [
(HookFailurePolicy::FailClosed, None),
(
HookFailurePolicy::FailClosed,
Some(HookError::new(
HookErrorCategory::Dependency,
"stop rejected",
)),
),
(
HookFailurePolicy::AttentionRequired,
Some(HookError::new(
HookErrorCategory::Dependency,
"stop rejected",
)),
),
(
HookFailurePolicy::FailOpenWithDiagnostic,
Some(HookError::new(
HookErrorCategory::Dependency,
"stop rejected",
)),
),
] {
let root = tempfile::tempdir().unwrap();
let callback = StoppingFeature {
entered: Arc::default(),
release: Arc::default(),
policy,
failure: failure.clone(),
};
let entered = callback.entered.clone();
let release = callback.release.clone();
let client = ScriptedClient::new(vec![
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":true})),
prose(),
]);
let mut job = prepared(root.path(), &request("user:a"), client);
job.features = job.features.with_module(callback);
let sink = Arc::new(RecordingSink::default());
let recording = sink.clone();
let run =
tokio::spawn(async move { job.run_owned("attempt".into(), recording, |_| {}).await });
signalled(&entered).await;
assert_eq!(sink.0.lock().unwrap().len(), 1);
assert!(
!run.is_finished(),
"result acceptance cannot bypass a live stopping hook"
);
release.notify_one();
let execution = tokio::time::timeout(std::time::Duration::from_secs(2), run)
.await
.unwrap()
.unwrap()
.unwrap();
assert_eq!(execution.feature_shutdown_complete(), failure.is_none());
match failure {
Some(error) => assert!(execution.feature_shutdown_failures().contains(&error)),
None => assert!(execution.feature_shutdown_failures().is_empty()),
}
assert_eq!(execution.result.unwrap().submission.result, json!(true));
assert_eq!(sink.0.lock().unwrap().len(), 1);
}
}
crates/worker/src/worker.rs
+209 -23
View File
@@ -273,6 +273,36 @@ where
}
impl PendingActivationState {
pub(crate) fn reconcile_submission(
&self,
request_id: &str,
input: &[Segment],
namespace: &str,
) -> Result<Option<crate::controller::SubmissionReceipt>, String> {
let Some(receipt) = self.receipts.iter().find(|receipt| {
receipt.submission_request_id == request_id && receipt.source_namespace == namespace
}) else {
return Ok(None);
};
if receipt.payload_digest != submission_payload_digest(input) {
return Err(PendingSubmissionError::IdempotencyConflict.to_string());
}
// Immediate admission is only durable after UserInput and its clearing
// checkpoint commit. Queued admission already persisted its receipt.
if receipt.disposition == protocol::SubmissionDisposition::Started
&& self.activating.as_ref().is_some_and(|pending| {
pending.submission_id == receipt.submission_id && !pending.was_queued
})
{
return Ok(None);
}
Ok(Some(crate::controller::SubmissionReceipt {
submission_request_id: receipt.submission_request_id.clone(),
submission_id: receipt.submission_id.clone(),
disposition: receipt.disposition,
}))
}
pub(crate) fn has_unfinished_admission(&self) -> bool {
self.activating.is_some()
|| !self.pending.is_empty()
@@ -4696,29 +4726,43 @@ impl<C: LlmClient + 'static, St: Store + 'static> Worker<C, St> {
}
pub async fn stop_feature_runtime(&mut self, reason: impl Into<String>) {
if let Err(failures) = self.stop_feature_runtime_report(reason).await {
for error in failures {
tracing::warn!(%error, "Feature runtime shutdown requires attention");
}
}
}
/// Await Worker-owned stopping hooks/background tasks and report actual failures separately
/// from a Run/result. Success covers only this Feature scope, not external Host callbacks,
/// Controller children, domain writer fencing or resource removal.
pub async fn stop_feature_runtime_report(
&mut self,
reason: impl Into<String>,
) -> Result<(), Vec<crate::hook::HookError>> {
let mut failures = Vec::new();
if let Some(hooks) = self.hook_registry.clone() {
let context = WorkerStoppingContext {
invocation: self.hook_invocation_context(None),
reason: reason.into(),
};
match tokio::time::timeout(
let mut hook_failures = Vec::new();
let deadline = tokio::time::timeout(
FEATURE_HOOK_CHAIN_TIMEOUT,
hooks.on_worker_stopping(&context),
hooks.on_worker_stopping_report(&context, &mut hook_failures),
)
.await
{
Ok(Ok(())) => {}
Ok(Err(error)) => {
tracing::warn!(error = %error, "worker-stopping hook requires attention");
}
Err(_) => {
hooks.record_chain_timeout("worker-stopping");
tracing::warn!("worker-stopping hook chain exceeded its host deadline");
}
.await;
failures.extend(hook_failures.into_iter().map(|error| error.source));
if deadline.is_err() {
hooks.record_chain_timeout("worker-stopping");
failures.push(crate::hook::HookError::new(
crate::hook::HookErrorCategory::Timeout,
"worker-stopping hook chain exceeded its host deadline",
));
}
}
if let Err(error) = self.feature_background_tasks.shutdown().await {
tracing::warn!(error = %error, "feature background task shutdown failed");
failures.push(error);
}
if let Some(plugins) = &self.plugin_tools {
// The registry has joined the drivers, including bounded on-stop.
@@ -4729,6 +4773,11 @@ impl<C: LlmClient + 'static, St: Store + 'static> Worker<C, St> {
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
}
}
if failures.is_empty() {
Ok(())
} else {
Err(failures)
}
}
}
@@ -13152,8 +13201,24 @@ mod build_summary_prompt_tests {
.unwrap(),
}),
AttributionOutcome::Rejected => Ok(WorkspaceResponse {
status: 503,
body: "temporarily unavailable".to_string(),
status: 403,
body: serde_json::to_string(
&server_api::RepositoryApiError::new(
403,
"PermissionDenied",
"Session registration permission denied.",
Vec::new(),
)
.with_failure(
protocol::session_failure::session_registration_attachment(
protocol::session_failure::SessionRegistrationCause::denied(
protocol::session_failure::SessionPrincipal::Subject,
),
)
.unwrap(),
),
)
.unwrap(),
}),
AttributionOutcome::Success => {
let input: server_api::SubjektivRecordSessionRequest =
@@ -13484,10 +13549,23 @@ mod build_summary_prompt_tests {
#[tokio::test]
async fn unknown_and_failed_attribution_remain_incomplete_until_lifecycle_retry() {
for (first, expected_state) in [
(AttributionOutcome::TransportUnknown, "outcome_unknown"),
(AttributionOutcome::Rejected, "failed"),
(AttributionOutcome::Unauthorized, "failed"),
use protocol::session_failure::{SessionPrincipal, SessionRegistrationCause};
for (first, expected_state, expected_cause) in [
(
AttributionOutcome::TransportUnknown,
"outcome_unknown",
SessionRegistrationCause::outcome_unknown(),
),
(
AttributionOutcome::Rejected,
"failed",
SessionRegistrationCause::denied(SessionPrincipal::Subject),
),
(
AttributionOutcome::Unauthorized,
"outcome_unknown",
SessionRegistrationCause::outcome_unknown(),
),
] {
let client = Arc::new(AttributionLifecycleClient::new([
first,
@@ -13500,10 +13578,10 @@ mod build_summary_prompt_tests {
SubjektivSessionAttributionLifecycle::NewSession,
)
.unwrap_err();
assert!(error.to_string().contains(expected_state));
assert_eq!(error.session_registration_cause(), Some(expected_cause));
if matches!(first, AttributionOutcome::Unauthorized) {
assert!(error.to_string().contains("rejected with HTTP 401"));
assert!(!error.to_string().contains("outcome_unknown"));
// A legacy HTTP status is not a domain authorization attestation.
// Attribution remains incomplete and no implicit retry is issued.
assert_eq!(
client.requests.lock().unwrap().len(),
1,
@@ -13672,7 +13750,12 @@ mod build_summary_prompt_tests {
)
.unwrap_err();
assert!(error.to_string().contains("legacy_unknown"));
assert_eq!(
error.session_registration_cause(),
Some(protocol::session_failure::SessionRegistrationCause::denied(
protocol::session_failure::SessionPrincipal::Subject,
)),
);
let requests = client.requests.lock().unwrap();
assert_eq!(requests.len(), 1);
let request: server_api::SubjektivRecordSessionRequest =
@@ -17787,6 +17870,109 @@ permission = "write"
assert_eq!(state.pending[1].provenance, account_b);
}
#[test]
fn submission_receipt_reconciliation_reads_persisted_identity_without_replay() {
let temp = tempfile::tempdir().unwrap();
let handle = PendingSubmissionHandle::for_test(temp.path());
let input = vec![Segment::text("accepted Job input")];
let accepted = handle
.accept_from_source(
"job-input".into(),
input.clone(),
"backend:job-attempt".into(),
WorkerHistoryProvenance::LegacyUnknown,
false,
)
.unwrap();
let location = handle.writer.state.location();
let before = handle
.writer
.store
.read_all(location.session_id, location.segment_id)
.unwrap();
let payload = before
.iter()
.rev()
.find_map(|entry| match entry {
LogEntry::Extension {
domain, payload, ..
} if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN => {
Some(payload.clone())
}
_ => None,
})
.unwrap();
let restored: PendingActivationState = serde_json::from_value(payload).unwrap();
let receipt = restored
.reconcile_submission("job-input", &input, "backend:job-attempt")
.unwrap()
.unwrap();
assert_eq!(receipt.submission_id, accepted.submission_id);
assert_eq!(receipt.disposition, protocol::SubmissionDisposition::Queued);
assert!(
restored
.reconcile_submission("job-input", &input, "backend:other-attempt")
.unwrap()
.is_none()
);
assert!(
restored
.reconcile_submission("missing", &input, "backend:job-attempt")
.unwrap()
.is_none()
);
assert!(
restored
.reconcile_submission(
"job-input",
&[Segment::text("changed")],
"backend:job-attempt"
)
.is_err()
);
assert_eq!(restored.pending.len(), 1);
assert_eq!(
serde_json::to_value(
handle
.writer
.store
.read_all(location.session_id, location.segment_id)
.unwrap()
)
.unwrap(),
serde_json::to_value(before).unwrap()
);
}
#[test]
fn submission_receipt_reconciliation_never_confirms_uncommitted_start() {
let temp = tempfile::tempdir().unwrap();
let handle = PendingSubmissionHandle::for_test(temp.path());
let input = vec![Segment::text("not committed yet")];
let accepted = handle
.accept("job-start".into(), input.clone(), true)
.unwrap();
assert!(
handle
.state
.lock()
.unwrap()
.reconcile_submission("job-start", &input, &handle.direct_client_namespace())
.unwrap()
.is_none()
);
handle.abort_activation(accepted.activation.unwrap());
assert!(
handle
.state
.lock()
.unwrap()
.reconcile_submission("job-start", &input, &handle.direct_client_namespace())
.unwrap()
.is_none()
);
}
#[test]
fn restore_reconciliation_clears_interrupted_acceptance_pin_for_retry() {
let temp = tempfile::tempdir().unwrap();
crates/workspace-server/src/backend_job.rs
+17 -20
View File
@@ -12,10 +12,10 @@
//! input is rejected. Attempts without execution slots remain durably reserved; dispatch advances
//! on completion and restart. Only explicit reevaluation adds attempts. The store checks confirmed
//! failure, remaining attempt budget, completed cleanup of prior attempt Workers, and no occupancy
//! by other Jobs. Unknown execution outcomes are not rerun as new attempts.
//! by other Jobs. Missing execution evidence never authorizes a new attempt.
//!
//! Serialization-key resource conflicts return the original immutable request as resource_reused.
//! This does not mean new input was processed. Pending/unknown Jobs and unfinished cleanup
//! This does not mean new input was processed. Pending Jobs and unfinished cleanup
//! retain resource occupancy, distinct from replay of an identical request.
//!
//! # Structured results and notifications
@@ -28,9 +28,9 @@
//!
//! Notifications have delivery records separate from results. After restart, unknown deliveries
//! are retried with the same tracked notification ID. Results survive stopped/missing destinations
//! without rerunning the model. Execution recovery matches observable Workers and tracked
//! submissions; lost evidence leaves the outcome unknown. Exactly-once execution of all external
//! side effects is not guaranteed.
//! without rerunning the model. Execution reconciliation and resource release are separate from
//! the accepted result; absence of a Worker or a response is not proof that its effects ended.
//! Exactly-once execution of arbitrary external side effects is not guaranteed.
use serde::{Deserialize, Serialize};
use serde_json::Value;
@@ -63,7 +63,6 @@ pub enum BackendJobState {
Pending,
Completed,
Failed,
Unknown,
}
impl From<BackendJobState> for job::JobState {
@@ -72,7 +71,6 @@ impl From<BackendJobState> for job::JobState {
BackendJobState::Pending => Self::Pending,
BackendJobState::Completed => Self::Completed,
BackendJobState::Failed => Self::Failed,
BackendJobState::Unknown => Self::Unknown,
}
}
}
@@ -83,7 +81,6 @@ impl BackendJobState {
Self::Pending => "pending",
Self::Completed => "completed",
Self::Failed => "failed",
Self::Unknown => "unknown",
}
}
@@ -92,7 +89,6 @@ impl BackendJobState {
"pending" => Ok(Self::Pending),
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"unknown" => Ok(Self::Unknown),
other => Err(Error::Store(format!("unknown Backend Job state `{other}`"))),
}
}
@@ -106,7 +102,6 @@ pub enum BackendJobAttemptState {
Dispatched,
Completed,
Failed,
Unknown,
}
impl From<BackendJobAttemptState> for job::JobAttemptState {
@@ -117,7 +112,6 @@ impl From<BackendJobAttemptState> for job::JobAttemptState {
BackendJobAttemptState::Dispatched => Self::Dispatched,
BackendJobAttemptState::Completed => Self::Completed,
BackendJobAttemptState::Failed => Self::Failed,
BackendJobAttemptState::Unknown => Self::Unknown,
}
}
}
@@ -130,7 +124,6 @@ impl BackendJobAttemptState {
Self::Dispatched => "dispatched",
Self::Completed => "completed",
Self::Failed => "failed",
Self::Unknown => "unknown",
}
}
@@ -141,7 +134,6 @@ impl BackendJobAttemptState {
"dispatched" => Ok(Self::Dispatched),
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"unknown" => Ok(Self::Unknown),
other => Err(Error::Store(format!(
"unknown Backend Job attempt state `{other}`"
))),
@@ -149,7 +141,7 @@ impl BackendJobAttemptState {
}
pub fn terminal(self) -> bool {
matches!(self, Self::Completed | Self::Failed | Self::Unknown)
matches!(self, Self::Completed | Self::Failed)
}
}
@@ -282,8 +274,8 @@ pub struct SubjektivConsolidationGrant {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct SubjektivJournalGrant {
/// Frozen writer generation. Legacy absence preserves its original canonical wire/digest;
/// recovery must use only the immutable original Session and still-current body.
/// Frozen writer generation. Legacy absence preserves its original canonical wire/digest
/// at the read/migration boundary only; it never authorizes a fresh Session read or selection.
#[serde(default, skip_serializing_if = "String::is_empty")]
pub binding_id: String,
pub subject_id: String,
@@ -586,6 +578,9 @@ pub struct BackendJobAttemptRecord {
pub worker: Option<RuntimeWorkerRef>,
pub runtime_run_id: Option<String>,
pub dispatched_at: Option<String>,
/// The Reserved -> Failed transaction closed dispatch admission before any
/// allocation was allowed. Absence of a Worker or a terminal state is not this proof.
pub allocation_precluded_at: Option<String>,
pub deadline_at: String,
pub result: Option<Value>,
pub result_digest: Option<String>,
@@ -756,12 +751,11 @@ mod tests {
}
#[test]
fn backend_state_projections_preserve_existing_wire_vocabulary() {
fn backend_state_projections_use_current_execution_vocabulary() {
for state in [
BackendJobState::Pending,
BackendJobState::Completed,
BackendJobState::Failed,
BackendJobState::Unknown,
] {
let shared = job::JobState::from(state);
assert_eq!(
@@ -775,7 +769,6 @@ mod tests {
BackendJobAttemptState::Dispatched,
BackendJobAttemptState::Completed,
BackendJobAttemptState::Failed,
BackendJobAttemptState::Unknown,
] {
let shared = job::JobAttemptState::from(state);
assert_eq!(
@@ -783,6 +776,11 @@ mod tests {
serde_json::to_value(shared).unwrap()
);
}
// Legacy uncertain outcomes are accepted only by the migration reader.
assert!(BackendJobState::parse("unknown").is_err());
assert!(BackendJobAttemptState::parse("unknown").is_err());
assert!(serde_json::from_str::<BackendJobState>("\"unknown\"").is_err());
assert!(serde_json::from_str::<BackendJobAttemptState>("\"unknown\"").is_err());
// The shared read vocabulary does not add Backend cancellation support.
assert!(BackendJobState::parse("cancelled").is_err());
assert!(BackendJobAttemptState::parse("cancelled").is_err());
@@ -851,7 +849,6 @@ mod tests {
),
(BackendJobState::Pending, BackendJobAttemptState::Dispatched),
(BackendJobState::Failed, BackendJobAttemptState::Failed),
(BackendJobState::Unknown, BackendJobAttemptState::Unknown),
] {
reservation.job.state = state;
reservation.attempt.state = attempt_state;
crates/workspace-server/src/backend_job_host_effects.rs
+248
View File
@@ -0,0 +1,248 @@
//! Live ownership of Host effects admitted for one exact Job execution.
//!
//! This supplements durable allocation/submission/domain receipts; it is not a Job state machine.
//! Create it only when initially admitting an execution, before its first Host callback. Recovery
//! must not manufacture a drain receipt by reconstructing an empty owner after a restart. A lost
//! owner requires reconciliation of the retained domain operations and writer fences instead.
//! Closing admission does not cancel accepted operations. Their existing DbExecutor lifetimes
//! retain the request lease through actual work and discarded-result cleanup, even if the HTTP
//! waiter disappears. Detached work must explicitly retain this same lease or is not covered.
use std::sync::{Arc, Mutex};
use tokio::sync::Notify;
use worker_runtime::identity::RuntimeWorkerRef;
use crate::backend_job::BackendJobAttemptRecord;
use crate::db_executor::DbOperationLifetime;
use crate::{Error, Result};
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub(crate) struct JobHostOwner {
pub workspace_id: String,
pub job_id: String,
pub attempt_id: String,
pub input_digest: String,
pub worker: RuntimeWorkerRef,
pub runtime_run_id: String,
}
impl TryFrom<&BackendJobAttemptRecord> for JobHostOwner {
type Error = Error;
fn try_from(attempt: &BackendJobAttemptRecord) -> Result<Self> {
Ok(Self {
workspace_id: attempt.workspace_id.clone(),
job_id: attempt.job_id.clone(),
attempt_id: attempt.attempt_id.clone(),
input_digest: attempt.input_digest.clone(),
worker: attempt.worker.clone().ok_or_else(|| {
Error::InvalidInput("Job Host ownership requires an exact Worker binding".into())
})?,
runtime_run_id: attempt
.runtime_run_id
.clone()
.filter(|id| !id.is_empty())
.ok_or_else(|| {
Error::InvalidInput(
"Job Host ownership requires a tracked Runtime submission".into(),
)
})?,
})
}
}
struct Admission {
open: bool,
requests: usize,
}
pub(crate) struct JobHostEffects {
owner: JobHostOwner,
admission: Mutex<Admission>,
finished: Notify,
}
impl JobHostEffects {
pub(crate) fn initially_admit(owner: JobHostOwner) -> Arc<Self> {
Arc::new(Self {
owner,
admission: Mutex::new(Admission {
open: true,
requests: 0,
}),
finished: Notify::new(),
})
}
pub(crate) fn admit(self: &Arc<Self>, expected: &JobHostOwner) -> Result<JobHostRequestLease> {
if &self.owner != expected {
return Err(Error::InvalidInput(
"Job Host execution owner mismatch".into(),
));
}
let mut admission = self
.admission
.lock()
.map_err(|_| Error::Store("Job Host admission poisoned".into()))?;
if !admission.open {
return Err(Error::InvalidInput(
"Job Host execution admission is closed".into(),
));
}
admission.requests = admission
.requests
.checked_add(1)
.ok_or_else(|| Error::Store("Job Host request count overflow".into()))?;
Ok(JobHostRequestLease {
request: Arc::new(AdmittedRequest {
effects: self.clone(),
}),
})
}
/// The receipt proves only this live Host owner's accepted operations ended. Its constructor
/// is private so a missing map entry, Job terminal state, or timeout cannot create evidence.
pub(crate) async fn close_and_drain(
self: &Arc<Self>,
expected: &JobHostOwner,
) -> Result<JobHostDrain> {
if &self.owner != expected {
return Err(Error::InvalidInput("Job Host close owner mismatch".into()));
}
self.admission
.lock()
.map_err(|_| Error::Store("Job Host admission poisoned".into()))?
.open = false;
loop {
let notified = self.finished.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self
.admission
.lock()
.map_err(|_| Error::Store("Job Host admission poisoned".into()))?
.requests
== 0
{
return Ok(JobHostDrain {
owner: self.owner.clone(),
});
}
notified.await;
}
}
}
/// Every clone refers to the same admitted request, including its synchronous child operations.
/// Only the last real owner ends it. Returning/dropping the async waiter alone is insufficient.
#[derive(Clone)]
pub(crate) struct JobHostRequestLease {
request: Arc<AdmittedRequest>,
}
impl DbOperationLifetime for JobHostRequestLease {
fn retain(&self) -> Box<dyn Send> {
Box::new(self.clone())
}
}
struct AdmittedRequest {
effects: Arc<JobHostEffects>,
}
impl Drop for AdmittedRequest {
fn drop(&mut self) {
let mut admission = self
.effects
.admission
.lock()
.unwrap_or_else(|poison| poison.into_inner());
admission.requests -= 1;
drop(admission);
self.effects.finished.notify_waiters();
}
}
pub(crate) struct JobHostDrain {
owner: JobHostOwner,
}
impl JobHostDrain {
pub(crate) fn owner(&self) -> &JobHostOwner {
&self.owner
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::db_executor::{DbExecutor, DbExecutorLimits, with_operation_lifetime};
use std::{num::NonZeroUsize, time::Duration};
fn owner(attempt: &str) -> JobHostOwner {
JobHostOwner {
workspace_id: "workspace".into(),
job_id: "job".into(),
attempt_id: attempt.into(),
input_digest: "input".into(),
worker: RuntimeWorkerRef::new("runtime", "worker"),
runtime_run_id: format!("run:{attempt}"),
}
}
#[tokio::test]
async fn cancelled_waiter_cannot_release_accepted_host_effect_or_admit_after_close() {
let expected = owner("attempt-1");
let effects = JobHostEffects::initially_admit(expected.clone());
let lease = effects.admit(&expected).unwrap();
let executor = DbExecutor::new(DbExecutorLimits {
workers: NonZeroUsize::new(1).unwrap(),
queued: NonZeroUsize::new(1).unwrap(),
})
.unwrap();
let (entered, started) = tokio::sync::oneshot::channel();
let (release, wait) = std::sync::mpsc::channel();
let work = executor.clone();
let waiter: tokio::task::JoinHandle<Result<()>> =
tokio::spawn(with_operation_lifetime(Arc::new(lease), async move {
work.run(move || {
entered.send(()).unwrap();
wait.recv_timeout(Duration::from_secs(5)).unwrap();
Ok(())
})
.await
}));
started.await.unwrap();
waiter.abort();
assert!(waiter.await.unwrap_err().is_cancelled());
let draining = effects.clone();
let binding = expected.clone();
let drain = tokio::spawn(async move { draining.close_and_drain(&binding).await });
// An active accepted lease remains an owner even though its waiter is gone.
assert_eq!(effects.admission.lock().unwrap().requests, 1);
release.send(()).unwrap();
let receipt = tokio::time::timeout(Duration::from_secs(2), drain)
.await
.unwrap()
.unwrap()
.unwrap();
assert_eq!(receipt.owner(), &expected);
assert!(effects.admit(&expected).is_err());
assert!(effects.admit(&owner("attempt-2")).is_err());
}
#[tokio::test]
async fn stale_close_cannot_revoke_a_successor_owner_or_create_its_drain_receipt() {
let current = owner("attempt-2");
let effects = JobHostEffects::initially_admit(current.clone());
assert!(effects.close_and_drain(&owner("attempt-1")).await.is_err());
let request = effects.admit(&current).unwrap();
let child = request.clone();
drop(request);
assert_eq!(effects.admission.lock().unwrap().requests, 1);
drop(child);
let receipt = effects.close_and_drain(&current).await.unwrap();
assert_eq!(receipt.owner(), &current);
assert!(effects.admit(&current).is_err());
}
}
crates/workspace-server/src/backend_job_recovery_store.rs
+1360
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/backend_job_upgrade.rs
+580
View File
@@ -0,0 +1,580 @@
//! One-shot retirement of Jobs that predate the current execution contract.
//!
//! The Workspace ordered migration owner must validate the retained storage
//! family first, disable foreign keys BEFORE its exclusive transaction, call
//! `install`, and commit the Workspace migration receipt in that same transaction.
//! It must restore foreign keys on both success and failure. This leaf neither
//! owns a version ledger nor runs from ordinary recovery/open after that receipt.
//!
//! A failed Job here means only "do not continue this old request". In particular,
//! no execution/cleanup termination, writer revocation, Journal completion, input
//! disposal or resource release is inferred. Pre-content captures preserve the
//! original cleanup values rather than the legacy cutover's inferred cleanup;
//! external Journal operations/checkpoints are untouched. Consumers must exclude
//! `backend_job_upgrade_history` Jobs from recovery/retry and consult the actual
//! execution/Journal owners before deletion. The guards also reject stale writes.
//! Notification delivery is a separate lifecycle: its saved states, notification
//! IDs and same-ID acceptance reconciliation are deliberately outside this cutover.
use super::{Connection, Error, Result};
/// Final history/guards. Fresh composition calls `install_fresh`, which also
/// installs EMPTY raw evidence; this SQL alone is not the full fresh schema.
pub(super) const MIGRATION_NAME: &str = "Backend Job execution retirement and owned reconciliation";
pub(super) const SCHEMA: &str = include_str!("backend_job_upgrade.sql");
pub(super) const FAILURE_CATEGORY: &str = "version_upgrade_execution_aborted";
pub(super) const FAILURE_DETAIL: &str = "バージョン更新による実行打切り。旧Jobの再開・再試行は行わない。このFailedは実行停止・書込み失効・Journal完了・rollback・資源解放の証明ではない。";
/// Evidence, not a second migration ledger. The existing schema-86 receipt owns
/// populated captures; fresh creation and later baselines install it EMPTY.
/// Raw request bytes remain historical and cannot authorize execution. Their only
/// allowed mutation is body retirement together with the live Job's owner.
pub(super) const RAW_SCHEMA: &str = r#"
CREATE TABLE backend_job_pre_content_history (
workspace_id TEXT NOT NULL,
job_id TEXT NOT NULL,
purpose TEXT NOT NULL,
input_revision TEXT NOT NULL,
input_ref TEXT NOT NULL,
resource_key TEXT,
request_json TEXT NOT NULL CHECK (json_valid(request_json)),
input_retired INTEGER NOT NULL CHECK (input_retired IN (0,1)),
intent_fingerprint TEXT NOT NULL,
current_attempt INTEGER NOT NULL,
state TEXT NOT NULL,
result_digest TEXT,
failure_category TEXT,
failure_detail TEXT,
updated_at TEXT NOT NULL,
completed_at TEXT,
captured_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
PRIMARY KEY (workspace_id,job_id),
FOREIGN KEY (workspace_id,job_id) REFERENCES backend_jobs(workspace_id,job_id) ON DELETE CASCADE
);
CREATE TABLE backend_job_attempt_pre_content_history (
workspace_id TEXT NOT NULL,
job_id TEXT NOT NULL,
attempt_id TEXT NOT NULL,
attempt INTEGER NOT NULL,
input_revision TEXT NOT NULL,
state TEXT NOT NULL,
runtime_id TEXT,
worker_id TEXT,
runtime_run_id TEXT,
dispatched_at TEXT,
deadline_at TEXT NOT NULL,
result_digest TEXT,
failure_category TEXT,
failure_detail TEXT,
worker_cleanup_state TEXT,
worker_cleanup_failure_category TEXT,
worker_cleanup_failure_detail TEXT,
worker_cleanup_updated_at TEXT,
worker_cleanup_completed_at TEXT,
updated_at TEXT NOT NULL,
completed_at TEXT,
captured_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
PRIMARY KEY (workspace_id,job_id,attempt_id),
FOREIGN KEY (workspace_id,job_id,attempt_id) REFERENCES backend_job_attempts(workspace_id,job_id,attempt_id) ON DELETE CASCADE,
FOREIGN KEY (workspace_id,job_id) REFERENCES backend_job_pre_content_history(workspace_id,job_id) ON DELETE CASCADE
);
-- Delivery is captured because the retained content migration modifies it, not
-- because notification reconciliation participates in the Job execution cutover.
CREATE TABLE backend_job_delivery_pre_content_history (
workspace_id TEXT NOT NULL,
delivery_id TEXT NOT NULL,
job_id TEXT NOT NULL,
attempt_id TEXT NOT NULL,
target_runtime_id TEXT NOT NULL,
target_worker_id TEXT NOT NULL,
state TEXT NOT NULL,
failure_category TEXT,
failure_detail TEXT,
updated_at TEXT NOT NULL,
delivered_at TEXT,
captured_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
PRIMARY KEY (workspace_id,delivery_id),
FOREIGN KEY (workspace_id,delivery_id) REFERENCES backend_job_deliveries(workspace_id,delivery_id) ON DELETE CASCADE
);
CREATE TRIGGER backend_job_pre_content_capture_boundary
BEFORE INSERT ON backend_job_pre_content_history
WHEN (SELECT max(version) FROM __yoi_schema_migrations) IS NOT 85
BEGIN SELECT RAISE(ABORT,'pre-content evidence requires the schema-85 transaction'); END;
CREATE TRIGGER backend_job_attempt_pre_content_capture_boundary
BEFORE INSERT ON backend_job_attempt_pre_content_history
WHEN (SELECT max(version) FROM __yoi_schema_migrations) IS NOT 85
BEGIN SELECT RAISE(ABORT,'pre-content evidence requires the schema-85 transaction'); END;
CREATE TRIGGER backend_job_delivery_pre_content_capture_boundary
BEFORE INSERT ON backend_job_delivery_pre_content_history
WHEN (SELECT max(version) FROM __yoi_schema_migrations) IS NOT 85
BEGIN SELECT RAISE(ABORT,'pre-content evidence requires the schema-85 transaction'); END;
CREATE TRIGGER backend_job_pre_content_identity_no_update
BEFORE UPDATE OF workspace_id,job_id,purpose,input_revision,input_ref,resource_key,intent_fingerprint,current_attempt,state,result_digest,failure_category,failure_detail,updated_at,completed_at,captured_at ON backend_job_pre_content_history
BEGIN SELECT RAISE(ABORT,'immutable pre-content Job evidence'); END;
CREATE TRIGGER backend_job_pre_content_request_retirement
BEFORE UPDATE OF request_json,input_retired ON backend_job_pre_content_history
WHEN (
OLD.input_retired=0 AND NEW.input_retired=1
AND json_type(NEW.request_json,'$.input')='null'
AND json_remove(NEW.request_json,'$.input')=json_remove(OLD.request_json,'$.input')
AND EXISTS(SELECT 1 FROM backend_jobs j WHERE j.workspace_id=OLD.workspace_id AND j.job_id=OLD.job_id
AND COALESCE(json_type(j.request_json,'$.input'),'null')='null')
) IS NOT TRUE
BEGIN SELECT RAISE(ABORT,'pre-content Job body retires only with its owner'); END;
CREATE TRIGGER backend_job_pre_content_retire_with_owner
AFTER UPDATE OF request_json ON backend_jobs
WHEN COALESCE(json_type(NEW.request_json,'$.input'),'null')='null'
BEGIN
UPDATE backend_job_pre_content_history
SET request_json=json_set(request_json,'$.input',json('null')),input_retired=1
WHERE workspace_id=NEW.workspace_id AND job_id=NEW.job_id AND input_retired=0;
END;
CREATE TRIGGER backend_job_pre_content_no_input_restore
BEFORE UPDATE OF request_json ON backend_jobs
WHEN COALESCE(json_type(NEW.request_json,'$.input'),'null')!='null'
AND EXISTS(SELECT 1 FROM backend_job_pre_content_history h
WHERE h.workspace_id=OLD.workspace_id AND h.job_id=OLD.job_id AND h.input_retired=1)
BEGIN SELECT RAISE(ABORT,'retired pre-content Job input cannot regenerate'); END;
CREATE TRIGGER backend_job_attempt_pre_content_no_update
BEFORE UPDATE ON backend_job_attempt_pre_content_history
BEGIN SELECT RAISE(ABORT,'immutable pre-content attempt evidence'); END;
CREATE TRIGGER backend_job_delivery_pre_content_no_update
BEFORE UPDATE ON backend_job_delivery_pre_content_history
BEGIN SELECT RAISE(ABORT,'immutable pre-content delivery evidence'); END;
"#;
const RAW_CAPTURE_SQL: &str = r#"
INSERT INTO backend_job_pre_content_history
(workspace_id,job_id,purpose,input_revision,input_ref,resource_key,request_json,input_retired,intent_fingerprint,current_attempt,state,result_digest,failure_category,failure_detail,updated_at,completed_at)
SELECT workspace_id,job_id,purpose,input_revision,input_ref,resource_key,request_json,
CASE WHEN COALESCE(json_type(request_json,'$.input'),'null')='null' THEN 1 ELSE 0 END,
intent_fingerprint,current_attempt,state,result_digest,failure_category,failure_detail,updated_at,completed_at
FROM backend_jobs;
INSERT INTO backend_job_attempt_pre_content_history
(workspace_id,job_id,attempt_id,attempt,input_revision,state,runtime_id,worker_id,runtime_run_id,dispatched_at,deadline_at,result_digest,failure_category,failure_detail,worker_cleanup_state,worker_cleanup_failure_category,worker_cleanup_failure_detail,worker_cleanup_updated_at,worker_cleanup_completed_at,updated_at,completed_at)
SELECT workspace_id,job_id,attempt_id,attempt,input_revision,state,runtime_id,worker_id,runtime_run_id,dispatched_at,deadline_at,result_digest,failure_category,failure_detail,worker_cleanup_state,worker_cleanup_failure_category,worker_cleanup_failure_detail,worker_cleanup_updated_at,worker_cleanup_completed_at,updated_at,completed_at
FROM backend_job_attempts;
INSERT INTO backend_job_delivery_pre_content_history
(workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,failure_category,failure_detail,updated_at,delivered_at)
SELECT workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,failure_category,failure_detail,updated_at,delivered_at
FROM backend_job_deliveries;
"#;
/// Called exactly once INSIDE migrate_content_state_v85_to_v86's transaction,
/// before its Job SELECT/renames/updates. On error the owner must roll back the
/// entire schema-86 step. No open/read helper may call this entrypoint. The owner
/// holds the storage lifecycle lock until the final cutover is committed; partial
/// retained prefixes are resumable migration data, never execution admission.
pub(crate) fn capture_before_content_cutover(conn: &Connection) -> Result<()> {
require_transaction(conn)?;
if super::current_schema_version(conn)? != 85
|| !super::column_exists(conn, "backend_jobs", "input_revision")?
|| super::column_exists(conn, "backend_jobs", "input_digest")?
{
return Err(Error::Store(
"pre-content Job capture requires the unmodified schema-85 boundary".into(),
));
}
conn.execute_batch(RAW_SCHEMA)?;
conn.execute_batch(RAW_CAPTURE_SQL)?;
verify_pre_content(conn)
}
/// Reject intervening Job/attempt ownership writes rather than erase real newer
/// evidence while undoing the retained cutover's synthetic values. Delivery
/// reconciliation and input retirement remain independent and are not frozen.
fn validate_content_cutover_before_restore(conn: &Connection) -> Result<()> {
let drift: bool = conn.query_row(r#"
WITH reserved_jobs AS (
SELECT j.workspace_id,j.job_id FROM backend_job_pre_content_history j
WHERE EXISTS(SELECT 1 FROM backend_job_attempt_pre_content_history a WHERE a.workspace_id=j.workspace_id AND a.job_id=j.job_id)
AND NOT EXISTS(SELECT 1 FROM backend_job_attempt_pre_content_history a WHERE a.workspace_id=j.workspace_id AND a.job_id=j.job_id
AND (a.state!='reserved' OR a.runtime_id IS NOT NULL OR a.worker_cleanup_state IS NOT NULL))
)
SELECT EXISTS(
SELECT 1 FROM backend_jobs j JOIN backend_job_pre_content_history r USING(workspace_id,job_id)
LEFT JOIN reserved_jobs v USING(workspace_id,job_id)
WHERE j.state IS NOT (CASE WHEN r.state='pending' THEN CASE WHEN v.job_id IS NOT NULL THEN 'failed' ELSE 'unknown' END ELSE r.state END)
OR j.failure_category IS NOT (CASE WHEN r.state='pending' THEN 'content_contract_cutover' ELSE r.failure_category END)
OR j.failure_detail IS NOT r.failure_detail OR j.updated_at IS NOT r.updated_at
OR j.completed_at IS NOT (CASE WHEN r.state='pending' THEN COALESCE(r.completed_at,r.updated_at) ELSE r.completed_at END)
) OR EXISTS(
SELECT 1 FROM backend_job_attempts a JOIN backend_job_attempt_pre_content_history r USING(workspace_id,job_id,attempt_id)
LEFT JOIN reserved_jobs v USING(workspace_id,job_id)
WHERE a.state IS NOT (CASE WHEN r.state='reserved' AND v.job_id IS NOT NULL THEN 'failed'
WHEN r.state IN ('reserved','dispatching','dispatched') THEN 'unknown' ELSE r.state END)
OR a.failure_category IS NOT (CASE WHEN r.state IN ('reserved','dispatching','dispatched') THEN 'content_contract_cutover' ELSE r.failure_category END)
OR a.failure_detail IS NOT r.failure_detail OR a.updated_at IS NOT r.updated_at
OR a.completed_at IS NOT (CASE WHEN r.state IN ('reserved','dispatching','dispatched') THEN COALESCE(r.completed_at,r.updated_at) ELSE r.completed_at END)
OR a.worker_cleanup_state IS NOT (CASE WHEN r.state='reserved' AND v.job_id IS NOT NULL THEN 'completed'
WHEN r.runtime_id IS NOT NULL AND r.state IN ('dispatching','dispatched') AND r.worker_cleanup_state IS NULL THEN 'pending' ELSE r.worker_cleanup_state END)
OR a.worker_cleanup_failure_category IS NOT r.worker_cleanup_failure_category
OR a.worker_cleanup_failure_detail IS NOT r.worker_cleanup_failure_detail
OR a.worker_cleanup_updated_at IS NOT r.worker_cleanup_updated_at
OR a.worker_cleanup_completed_at IS NOT r.worker_cleanup_completed_at
)
"#,[],|row|row.get(0))?;
if drift {
return Err(Error::Store("pre-content Job/attempt changed after its retained cutover; preserve evidence and reconcile before final migration".into()));
}
Ok(())
}
fn require_transaction(conn: &Connection) -> Result<()> {
if conn.is_autocommit() {
return Err(Error::Store(
"Backend Job upgrade requires a caller-owned transaction".into(),
));
}
Ok(())
}
/// Fresh creation installs empty evidence and final guards, never captures or
/// fails Jobs. The fresh canonical state CHECKs are supplied by the owner DDL.
pub(super) fn install_fresh(conn: &Connection) -> Result<()> {
require_transaction(conn)?;
conn.execute_batch(RAW_SCHEMA)?;
install_live_checks(conn)?;
conn.execute_batch(SCHEMA)?;
conn.execute_batch(
"ALTER TABLE backend_job_attempts ADD COLUMN allocation_precluded_at TEXT;",
)?;
conn.execute_batch(super::backend_job_recovery_store::SCHEMA)?;
verify(conn)
}
pub(super) fn install_live_checks(conn: &Connection) -> Result<()> {
for (table, old, new) in STATE_CHECKS {
replace_state_check(conn, table, old, new)?;
}
Ok(())
}
/// The retained pre-123 reference composition must keep its original CHECKs,
/// even when the fresh owner DDL is updated to the current live contract.
pub(super) fn retained_checks(sql: &str) -> String {
STATE_CHECKS.iter().fold(
sql.replace(" allocation_precluded_at TEXT,\n", ""),
|sql, (_, old, new)| sql.replace(new, old),
)
}
const SNAPSHOT_SQL: &str = r#"
INSERT INTO backend_job_upgrade_history
(workspace_id,job_id,state,failure_category,failure_detail,updated_at,completed_at)
SELECT workspace_id,job_id,state,failure_category,failure_detail,updated_at,completed_at FROM backend_jobs;
INSERT INTO backend_job_attempt_upgrade_history
(workspace_id,job_id,attempt_id,state,failure_category,failure_detail,updated_at,completed_at)
SELECT workspace_id,job_id,attempt_id,state,failure_category,failure_detail,updated_at,completed_at FROM backend_job_attempts;
"#;
// Preserve result/input/identity and every execution timestamp, including old
// completed_at. Migration time belongs to history, not execution-end evidence.
const FAIL_JOBS_SQL: &str = r#"
UPDATE backend_jobs SET state='failed',failure_category=?1,failure_detail=?2
WHERE state NOT IN ('completed','failed');
"#;
const FAIL_ATTEMPTS_SQL: &str = r#"
UPDATE backend_job_attempts SET state='failed',failure_category=?1,failure_detail=?2
WHERE state NOT IN ('completed','failed');
"#;
// Restore only captured values, not inferred execution-end evidence from the
// retained content cutover. Live normalized input/digest/fingerprint remain in
// their current format; their original wire identity is retained in raw evidence.
const APPLY_CAPTURED_JOBS_SQL: &str = r#"
UPDATE backend_jobs AS j SET (state,failure_category,failure_detail,updated_at,completed_at) = (
SELECT CASE WHEN r.state IN ('completed','failed') THEN r.state ELSE 'failed' END,
CASE WHEN r.state IN ('completed','failed') THEN r.failure_category ELSE ?1 END,
CASE WHEN r.state IN ('completed','failed') THEN r.failure_detail ELSE ?2 END,
r.updated_at,r.completed_at
FROM backend_job_pre_content_history r WHERE r.workspace_id=j.workspace_id AND r.job_id=j.job_id
) WHERE EXISTS(SELECT 1 FROM backend_job_pre_content_history r WHERE r.workspace_id=j.workspace_id AND r.job_id=j.job_id);
"#;
const APPLY_CAPTURED_ATTEMPTS_SQL: &str = r#"
UPDATE backend_job_attempts AS a SET
(state,failure_category,failure_detail,updated_at,completed_at,
worker_cleanup_state,worker_cleanup_failure_category,worker_cleanup_failure_detail,worker_cleanup_updated_at,worker_cleanup_completed_at) = (
SELECT CASE WHEN r.state IN ('completed','failed') THEN r.state ELSE 'failed' END,
CASE WHEN r.state IN ('completed','failed') THEN r.failure_category ELSE ?1 END,
CASE WHEN r.state IN ('completed','failed') THEN r.failure_detail ELSE ?2 END,
r.updated_at,r.completed_at,
r.worker_cleanup_state,r.worker_cleanup_failure_category,r.worker_cleanup_failure_detail,r.worker_cleanup_updated_at,r.worker_cleanup_completed_at
FROM backend_job_attempt_pre_content_history r
WHERE r.workspace_id=a.workspace_id AND r.job_id=a.job_id AND r.attempt_id=a.attempt_id
) WHERE EXISTS(SELECT 1 FROM backend_job_attempt_pre_content_history r
WHERE r.workspace_id=a.workspace_id AND r.job_id=a.job_id AND r.attempt_id=a.attempt_id);
"#;
// Exact retained CHECKs only. Published Jobs-family tables may already have the
// new CHECK; the physical recognizer, not this function, owns family acceptance.
const STATE_CHECKS: [(&str, &str, &str); 2] = [
(
"backend_jobs",
"CHECK (state IN ('pending', 'completed', 'failed', 'unknown'))",
"CHECK (state IN ('pending', 'completed', 'failed'))",
),
(
"backend_job_attempts",
"CHECK (state IN ('reserved', 'dispatching', 'dispatched', 'completed', 'failed', 'unknown'))",
"CHECK (state IN ('reserved', 'dispatching', 'dispatched', 'completed', 'failed'))",
),
];
/// Apply once in the caller's validated Workspace migration transaction.
/// On ANY error the caller must roll back, including DDL and its version receipt.
/// A second invocation is an error, not a new sweep of post-upgrade Jobs.
pub(super) fn install(conn: &Connection) -> Result<()> {
require_transaction(conn)?;
let foreign_keys: bool = conn.query_row("PRAGMA foreign_keys", [], |row| row.get(0))?;
if foreign_keys {
return Err(Error::Store(
"Backend Job upgrade requires foreign_keys OFF before the transaction".into(),
));
}
for table in ["backend_jobs", "backend_job_attempts"] {
if !super::column_exists(conn, table, "input_digest")?
|| super::column_exists(conn, table, "input_revision")?
{
return Err(Error::Store(
"Backend Job final cutover requires the retained content normalization first"
.into(),
));
}
}
let raw_tables = [
"backend_job_pre_content_history",
"backend_job_attempt_pre_content_history",
"backend_job_delivery_pre_content_history",
];
let present = raw_tables
.into_iter()
.map(|name| super::table_exists(conn, name))
.collect::<Result<Vec<_>>>()?;
if present.iter().all(|exists| !exists) {
// Sources that were already schema 86+ cannot regain lost pre-86 facts.
conn.execute_batch(RAW_SCHEMA)?;
} else if !present.iter().all(|exists| *exists) {
return Err(Error::Store(
"incomplete pre-content Job evidence schema".into(),
));
}
verify_pre_content(conn)?;
validate_content_cutover_before_restore(conn)?;
let (history_schema, execution_guards) = SCHEMA
.split_once("-- FINAL_EXECUTION_GUARDS")
.ok_or_else(|| Error::Store("missing final Backend Job guard boundary".into()))?;
conn.execute_batch(history_schema)?;
conn.execute_batch(SNAPSHOT_SQL)?;
for sql in [
FAIL_JOBS_SQL,
FAIL_ATTEMPTS_SQL,
APPLY_CAPTURED_JOBS_SQL,
APPLY_CAPTURED_ATTEMPTS_SQL,
] {
conn.execute(sql, [FAILURE_CATEGORY, FAILURE_DETAIL])?;
}
for (table, old, new) in STATE_CHECKS {
replace_state_check(conn, table, old, new)?;
}
// A captured originally-unfinished row may already be Failed by schema 86.
// Install terminal immutability only after replacing that synthetic outcome.
conn.execute_batch(execution_guards)?;
// Old Failed/Reserved rows acquire no negative allocation proof from migration.
conn.execute_batch(
"ALTER TABLE backend_job_attempts ADD COLUMN allocation_precluded_at TEXT;",
)?;
conn.execute_batch(super::backend_job_recovery_store::SCHEMA)?;
verify(conn)
}
fn quoted(identifier: &str) -> String {
format!("\"{}\"", identifier.replace('"', "\"\""))
}
// Recreate under the SAME table name, without ALTER RENAME or writable_schema.
// This avoids rewriting FKs/views/triggers attached to other family tables. Copy
// every physical column, including published cleanup ownership/admission fields;
// don't replace family DDL with a baseline or infer any missing termination rows.
fn replace_state_check(conn: &Connection, table: &str, old: &str, new: &str) -> Result<()> {
let ddl: String = conn.query_row(
"SELECT sql FROM sqlite_schema WHERE type='table' AND name=?1",
[table],
|row| row.get(0),
)?;
if ddl.matches(new).count() == 1 && !ddl.contains(old) {
return Ok(());
}
if ddl.matches(old).count() != 1 || ddl.contains(new) {
return Err(Error::Store(format!(
"Backend Job upgrade: unrecognized state CHECK in {table}"
)));
}
let programs = conn.prepare(
"SELECT sql FROM sqlite_schema WHERE tbl_name=?1 AND type IN ('index','trigger') AND sql IS NOT NULL ORDER BY type,name",
)?.query_map([table], |row| row.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
let columns = conn
.prepare(&format!("PRAGMA table_xinfo({})", quoted(table)))?
.query_map([], |row| {
Ok((row.get::<_, String>(1)?, row.get::<_, i64>(6)?))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
if columns.iter().any(|(_, hidden)| *hidden != 0) {
return Err(Error::Store(format!(
"Backend Job upgrade: unsupported generated/hidden column in {table}"
)));
}
let columns = columns
.iter()
.map(|(name, _)| quoted(name))
.collect::<Vec<_>>()
.join(",");
let temporary = quoted(&format!("__yoi_upgrade_{table}"));
let table_name = quoted(table);
conn.execute_batch(&format!(
"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM main.{table_name};
DROP TABLE main.{table_name};"
))?;
conn.execute_batch(&ddl.replacen(old, new, 1))?;
conn.execute_batch(&format!(
"INSERT INTO main.{table_name} ({columns}) SELECT {columns} FROM temp.{temporary};
DROP TABLE temp.{temporary};"
))?;
for sql in programs {
conn.execute_batch(&sql)?;
}
Ok(())
}
/// Validate ownership and immutable content without normalizing or restoring any
/// body. This also runs before the schema-86 receipt, when input_revision is live.
/// Cleanup/delivery outcomes may legitimately evolve after migration; they are
/// historical evidence, not shadow operational state machines.
pub(super) fn verify_pre_content(conn: &Connection) -> Result<()> {
let invalid: bool = conn.query_row(r#"
SELECT EXISTS(
SELECT 1 FROM backend_job_pre_content_history r JOIN backend_jobs j
ON j.workspace_id=r.workspace_id AND j.job_id=r.job_id
WHERE j.purpose IS NOT r.purpose OR j.input_ref IS NOT r.input_ref
OR j.resource_key IS NOT r.resource_key OR j.current_attempt IS NOT r.current_attempt
OR j.result_digest IS NOT r.result_digest
OR (SELECT count(*) FROM backend_job_attempts a WHERE a.workspace_id=r.workspace_id AND a.job_id=r.job_id)
!= (SELECT count(*) FROM backend_job_attempt_pre_content_history a WHERE a.workspace_id=r.workspace_id AND a.job_id=r.job_id)
) OR EXISTS(
SELECT 1 FROM backend_job_attempt_pre_content_history r JOIN backend_job_attempts a
ON a.workspace_id=r.workspace_id AND a.job_id=r.job_id AND a.attempt_id=r.attempt_id
WHERE a.attempt IS NOT r.attempt OR a.runtime_id IS NOT r.runtime_id OR a.worker_id IS NOT r.worker_id
OR a.runtime_run_id IS NOT r.runtime_run_id OR a.dispatched_at IS NOT r.dispatched_at
OR a.deadline_at IS NOT r.deadline_at OR a.result_digest IS NOT r.result_digest
) OR EXISTS(
SELECT 1 FROM backend_job_delivery_pre_content_history r JOIN backend_job_deliveries d
ON d.workspace_id=r.workspace_id AND d.delivery_id=r.delivery_id
WHERE d.job_id IS NOT r.job_id OR d.attempt_id IS NOT r.attempt_id
OR d.target_runtime_id IS NOT r.target_runtime_id OR d.target_worker_id IS NOT r.target_worker_id
)
"#, [], |row| row.get(0))?;
if invalid {
return Err(Error::Store(
"pre-content Job identity/result/attempt coverage drift".into(),
));
}
let mut statement = conn.prepare("SELECT r.request_json,j.request_json,r.input_retired FROM backend_job_pre_content_history r JOIN backend_jobs j ON j.workspace_id=r.workspace_id AND j.job_id=r.job_id")?;
let mut rows = statement.query([])?;
while let Some(row) = rows.next()? {
let original: serde_json::Value = serde_json::from_str(&row.get::<_, String>(0)?)
.map_err(|_| Error::Store("invalid pre-content request evidence".into()))?;
let current: serde_json::Value = serde_json::from_str(&row.get::<_, String>(1)?)
.map_err(|_| Error::Store("invalid live request beside pre-content evidence".into()))?;
let retired: bool = row.get(2)?;
let original_input = original.get("input");
let current_input = current.get("input");
let original_empty = original_input.is_none_or(serde_json::Value::is_null);
let current_empty = current_input.is_none_or(serde_json::Value::is_null);
if (retired && (!original_empty || !current_empty))
|| (!retired && (original_empty || original_input != current_input))
{
return Err(Error::Store(
"pre-content immutable input lost or regenerated".into(),
));
}
}
Ok(())
}
/// Read-only postcondition; never sweeps or repairs post-migration Jobs.
pub(super) fn verify(conn: &Connection) -> Result<()> {
verify_pre_content(conn)?;
for (table, old, new) in STATE_CHECKS {
let ddl: String = conn.query_row(
"SELECT sql FROM sqlite_schema WHERE type='table' AND name=?1",
[table],
|row| row.get(0),
)?;
if ddl.contains(old) || ddl.matches(new).count() != 1 {
return Err(Error::Store(format!(
"Backend Job upgrade left a legacy live-state CHECK in {table}"
)));
}
}
for (table, history, raw_history, identity) in [
(
"backend_jobs",
"backend_job_upgrade_history",
"backend_job_pre_content_history",
"a.job_id=h.job_id",
),
(
"backend_job_attempts",
"backend_job_attempt_upgrade_history",
"backend_job_attempt_pre_content_history",
"a.job_id=h.job_id AND a.attempt_id=h.attempt_id",
),
] {
let raw_identity = identity.replace("h.", "r.");
let basis = "CASE WHEN r.job_id IS NULL THEN h.state ELSE r.state END";
let category =
"CASE WHEN r.job_id IS NULL THEN h.failure_category ELSE r.failure_category END";
let detail = "CASE WHEN r.job_id IS NULL THEN h.failure_detail ELSE r.failure_detail END";
let completed = "CASE WHEN r.job_id IS NULL THEN h.completed_at ELSE r.completed_at END";
let invalid: bool = conn.query_row(
&format!(
"SELECT EXISTS(SELECT 1 FROM {table} a JOIN {history} h
ON a.workspace_id=h.workspace_id AND {identity}
LEFT JOIN {raw_history} r ON a.workspace_id=r.workspace_id AND {raw_identity}
WHERE a.state NOT IN ('completed','failed') OR a.completed_at IS NOT ({completed})
OR (({basis}) IN ('completed','failed')
AND (a.state IS NOT ({basis}) OR a.failure_category IS NOT ({category})
OR a.failure_detail IS NOT ({detail})))
OR (({basis}) NOT IN ('completed','failed')
AND (a.state!='failed' OR a.failure_category IS NOT ?1 OR a.failure_detail IS NOT ?2)))"
),
[FAILURE_CATEGORY, FAILURE_DETAIL],
|row| row.get(0),
)?;
if invalid {
return Err(Error::Store(format!(
"Backend Job upgrade left an unretired row in {table}"
)));
}
}
let violations: i64 =
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| {
row.get(0)
})?;
if violations != 0 {
return Err(Error::Store(
"Backend Job upgrade left foreign-key violations".into(),
));
}
Ok(())
}
/// Reuse the storage owner's exact published-family fixtures in its normal suite.
#[cfg(test)]
pub(super) fn assert_published_fixture_for_test(conn: &Connection) {
tests::assert_published_fixture(conn);
}
#[cfg(test)]
#[path = "backend_job_upgrade_tests.rs"]
mod tests;
crates/workspace-server/src/backend_job_upgrade.sql
+76
View File
@@ -0,0 +1,76 @@
-- These rows mark the migration boundary, including already terminal Jobs.
-- Do not duplicate request/result bodies: their existing retention/erasure owner
-- remains authoritative. Historical state spellings are not live execution states.
CREATE TABLE backend_job_upgrade_history (
workspace_id TEXT NOT NULL,
job_id TEXT NOT NULL,
state TEXT NOT NULL,
failure_category TEXT,
failure_detail TEXT,
updated_at TEXT NOT NULL,
completed_at TEXT,
migrated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
PRIMARY KEY (workspace_id, job_id),
FOREIGN KEY (workspace_id, job_id)
REFERENCES backend_jobs(workspace_id, job_id) ON DELETE CASCADE
);
CREATE TABLE backend_job_attempt_upgrade_history (
workspace_id TEXT NOT NULL,
job_id TEXT NOT NULL,
attempt_id TEXT NOT NULL,
state TEXT NOT NULL,
failure_category TEXT,
failure_detail TEXT,
updated_at TEXT NOT NULL,
completed_at TEXT,
migrated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
PRIMARY KEY (workspace_id, job_id, attempt_id),
FOREIGN KEY (workspace_id, job_id, attempt_id)
REFERENCES backend_job_attempts(workspace_id, job_id, attempt_id) ON DELETE CASCADE
);
CREATE TRIGGER backend_job_upgrade_history_no_update
BEFORE UPDATE ON backend_job_upgrade_history
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job upgrade history'); END;
CREATE TRIGGER backend_job_attempt_upgrade_history_no_update
BEFORE UPDATE ON backend_job_attempt_upgrade_history
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job attempt upgrade history'); END;
-- FINAL_EXECUTION_GUARDS
-- Install these only after all captured pre-content values have been composed.
-- Input retirement and independently evidenced cleanup are deliberately separate.
CREATE TRIGGER backend_job_upgrade_no_resume
BEFORE UPDATE ON backend_jobs
WHEN EXISTS (SELECT 1 FROM backend_job_upgrade_history h
WHERE h.workspace_id=OLD.workspace_id AND h.job_id=OLD.job_id)
AND OLD.state IN ('completed','failed')
AND (NEW.state IS NOT OLD.state OR NEW.current_attempt IS NOT OLD.current_attempt
OR NEW.result_json IS NOT OLD.result_json OR NEW.result_digest IS NOT OLD.result_digest
OR NEW.failure_category IS NOT OLD.failure_category OR NEW.failure_detail IS NOT OLD.failure_detail
OR NEW.completed_at IS NOT OLD.completed_at)
BEGIN SELECT RAISE(ABORT, 'pre-upgrade Backend Job cannot resume'); END;
CREATE TRIGGER backend_job_upgrade_no_attempt
BEFORE INSERT ON backend_job_attempts
WHEN EXISTS (SELECT 1 FROM backend_job_upgrade_history h
WHERE h.workspace_id=NEW.workspace_id AND h.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT, 'pre-upgrade Backend Job cannot create attempts'); END;
CREATE TRIGGER backend_job_upgrade_no_attempt_reparent
BEFORE UPDATE OF workspace_id,job_id ON backend_job_attempts
WHEN (NEW.workspace_id IS NOT OLD.workspace_id OR NEW.job_id IS NOT OLD.job_id)
AND EXISTS (SELECT 1 FROM backend_job_upgrade_history h
WHERE h.workspace_id=NEW.workspace_id AND h.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT, 'pre-upgrade Backend Job cannot adopt attempts'); END;
CREATE TRIGGER backend_job_attempt_upgrade_no_resume
BEFORE UPDATE ON backend_job_attempts
WHEN EXISTS (SELECT 1 FROM backend_job_attempt_upgrade_history h
WHERE h.workspace_id=OLD.workspace_id AND h.job_id=OLD.job_id AND h.attempt_id=OLD.attempt_id)
AND OLD.state IN ('completed','failed')
AND (NEW.workspace_id IS NOT OLD.workspace_id OR NEW.job_id IS NOT OLD.job_id
OR NEW.attempt_id IS NOT OLD.attempt_id OR NEW.attempt IS NOT OLD.attempt
OR NEW.input_digest IS NOT OLD.input_digest OR NEW.state IS NOT OLD.state
OR NEW.runtime_id IS NOT OLD.runtime_id OR NEW.worker_id IS NOT OLD.worker_id
OR NEW.runtime_run_id IS NOT OLD.runtime_run_id OR NEW.dispatched_at IS NOT OLD.dispatched_at
OR NEW.deadline_at IS NOT OLD.deadline_at
OR NEW.result_json IS NOT OLD.result_json OR NEW.result_digest IS NOT OLD.result_digest
OR NEW.failure_category IS NOT OLD.failure_category OR NEW.failure_detail IS NOT OLD.failure_detail
OR NEW.completed_at IS NOT OLD.completed_at)
BEGIN SELECT RAISE(ABORT, 'pre-upgrade Backend Job attempt cannot resume'); END;
crates/workspace-server/src/backend_job_upgrade_tests.rs
+1671
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/db_executor.rs
+109
View File
@@ -53,10 +53,31 @@ tokio::task_local! {
static OPERATION_LIFETIME: Arc<dyn DbOperationLifetime>;
}
struct NestedOperationLifetime {
outer: Arc<dyn DbOperationLifetime>,
inner: Arc<dyn DbOperationLifetime>,
}
impl DbOperationLifetime for NestedOperationLifetime {
fn retain(&self) -> Box<dyn Send> {
Box::new((self.outer.retain(), self.inner.retain()))
}
}
pub(crate) async fn with_operation_lifetime<F: Future>(
lifetime: Arc<dyn DbOperationLifetime>,
future: F,
) -> F::Output {
// An inner attempt scope must not replace its Workspace admission lease. Carry both
// sources through actual SQL, synchronous continuations, and discarded-result cleanup.
let lifetime = OPERATION_LIFETIME
.try_with(|outer| {
Arc::new(NestedOperationLifetime {
outer: outer.clone(),
inner: lifetime.clone(),
}) as Arc<dyn DbOperationLifetime>
})
.unwrap_or(lifetime);
OPERATION_LIFETIME.scope(lifetime, future).await
}
@@ -591,6 +612,94 @@ mod tests {
);
}
#[tokio::test(flavor = "current_thread")]
async fn nested_workspace_and_attempt_lifetimes_survive_sql_child_and_discarded_reply_cleanup()
{
struct ReplyCleanup {
entered: Option<oneshot::Sender<()>>,
finish: mpsc::Receiver<()>,
}
impl Drop for ReplyCleanup {
fn drop(&mut self) {
self.entered.take().unwrap().send(()).unwrap();
self.finish.recv_timeout(Duration::from_secs(5)).unwrap();
}
}
let (dir, owner, mut parent_conn) = database();
let mut child_conn = Connection::open(dir.path().join("jobs.sqlite")).unwrap();
let parent = executor(1);
let child = executor(1);
let child_owner = child.clone();
let workspace = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let attempt = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let outer: Arc<dyn DbOperationLifetime> = Arc::new(Lease(workspace.clone()));
let inner: Arc<dyn DbOperationLifetime> = Arc::new(Lease(attempt.clone()));
let (entered, running) = oneshot::channel();
let (release, permit) = mpsc::channel();
let (reply_dropping, dropping) = oneshot::channel();
let (finish_reply, finish) = mpsc::channel();
let mut operation = Box::pin(with_operation_lifetime(
outer,
with_operation_lifetime(
inner,
parent.run(move || -> Result<()> {
entered.send(()).unwrap();
permit.recv_timeout(Duration::from_secs(5)).unwrap();
let tx =
parent_conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute("INSERT INTO writes VALUES (1)", [])?;
tx.commit()?;
let reply = child_owner.submit(move || {
let tx = child_conn
.transaction_with_behavior(TransactionBehavior::Immediate)
.unwrap();
let committed: i64 = tx
.query_row("SELECT count(*) FROM writes", [], |r| r.get(0))
.unwrap();
assert_eq!(committed, 1);
tx.execute("INSERT INTO writes VALUES (2)", []).unwrap();
tx.commit().unwrap();
// Failed reply delivery drops this result outside the SQL transaction.
Ok::<_, Error>(ReplyCleanup {
entered: Some(reply_dropping),
finish,
})
})?;
drop(reply);
Ok(())
}),
),
));
assert!(futures::poll!(&mut operation).is_pending());
observed(running).await.unwrap();
drop(operation);
for count in [&workspace, &attempt] {
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
}
release.send(()).unwrap();
observed(dropping).await.unwrap();
observed(parent.drain()).await.unwrap();
for count in [&workspace, &attempt] {
assert_eq!(
count.load(std::sync::atomic::Ordering::SeqCst),
1,
"both owners must survive the parent and cancelled waiter"
);
}
let mut drain = Box::pin(child.drain());
assert!(futures::poll!(&mut drain).is_pending());
assert_eq!(
owner
.query_row("SELECT count(*) FROM writes", [], |r| r.get::<_, i64>(0))
.unwrap(),
2
);
finish_reply.send(()).unwrap();
observed(drain).await.unwrap();
assert_eq!(workspace.load(std::sync::atomic::Ordering::SeqCst), 0);
assert_eq!(attempt.load(std::sync::atomic::Ordering::SeqCst), 0);
}
#[tokio::test]
async fn failed_transaction_rolls_back_and_releases_actual_job_capacity() {
let (_dir, owner, mut worker) = database();
crates/workspace-server/src/frozen_content_state_migration.rs
+1
View File
@@ -95,6 +95,7 @@ fn migrate_content_state_v85_to_v86(conn: &Connection) -> Result<()> {
// Converge on the canonical schema for both legacy creation paths.
tx.execute_batch("CREATE INDEX IF NOT EXISTS worker_create_reservations_worker ON worker_create_reservations(workspace_id, worker_id);")?;
if column_exists(&tx, "backend_jobs", "input_revision")? {
backend_job_upgrade::capture_before_content_cutover(&tx)?;
let jobs = {
let mut stmt =
tx.prepare("SELECT workspace_id,job_id,request_json FROM backend_jobs")?;
crates/workspace-server/src/frozen_content_state_tests.rs
+12 -16
View File
@@ -426,25 +426,25 @@ fn content_cutover_keeps_each_evaluation_provenance_and_deduplicates_only_identi
}
#[test]
fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_it() {
fn content_cutover_version_abort_retains_prior_cleanup_and_uncertain_resource_owners() {
for (state, expected_job, expected_attempt, expected_cleanup) in [
(
"reserved",
BackendJobState::Failed,
BackendJobAttemptState::Failed,
Some(BackendJobWorkerCleanupState::Completed),
None,
),
(
"dispatching",
BackendJobState::Unknown,
BackendJobAttemptState::Unknown,
Some(BackendJobWorkerCleanupState::Pending),
BackendJobState::Failed,
BackendJobAttemptState::Failed,
None,
),
(
"dispatched",
BackendJobState::Unknown,
BackendJobAttemptState::Unknown,
Some(BackendJobWorkerCleanupState::Pending),
BackendJobState::Failed,
BackendJobAttemptState::Failed,
None,
),
] {
let dir = tempfile::tempdir().unwrap();
@@ -490,15 +490,11 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
let next = store
.reserve_backend_job("space", &next, "2026-08-15T00:00:00Z")
.unwrap();
assert_eq!(next.resource_reused, state != "reserved");
assert_eq!(
next.job.request.job_id,
if state == "reserved" {
"next-job"
} else {
"legacy-job"
}
assert!(
next.resource_reused,
"migration failure is not termination evidence"
);
assert_eq!(next.job.request.job_id, "legacy-job");
}
}
crates/workspace-server/src/hosts.rs
+294
View File
@@ -1437,6 +1437,45 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
Ok(None)
}
/// Original allocation identity only, not execution or input acceptance.
fn lookup_job_worker_creation(
&self,
_worker_id: &str,
_create_fingerprint: &str,
_binding: &worker_runtime::catalog::BackendJobExecutionBinding,
) -> Result<Option<InternalWorkerSummary>, RuntimeRegistryError> {
Err(RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id().to_owned(),
code: "worker_creation_receipt_unsupported".into(),
message: "Runtime does not support original Job Worker identity lookup".into(),
})
}
fn backend_job_termination(
&self,
_worker_id: &str,
) -> Result<Option<worker_runtime::execution::WorkerExecutionTermination>, RuntimeRegistryError>
{
Err(RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id().to_owned(),
code: "worker_termination_receipt_unsupported".into(),
message: "Runtime does not support actual scope-termination receipt lookup".into(),
})
}
/// Read-only exact receipt lookup. Unsupported lookup is not proof of rejection.
fn reconcile_worker_input(
&self,
_worker_id: &str,
_request: WorkerInputRequest,
) -> Result<WorkerInputResult, RuntimeRegistryError> {
Err(RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id().to_owned(),
code: "worker_input_receipt_unsupported".into(),
message: "Runtime does not support read-only submission receipt lookup".into(),
})
}
fn send_input(&self, worker_id: &str, _request: WorkerInputRequest) -> WorkerInputResult {
WorkerInputResult {
state: InternalWorkerOperationState::Unsupported,
@@ -2075,6 +2114,40 @@ impl RuntimeRegistry {
runtime.send_protocol_method(worker_id, method)
}
pub fn lookup_job_worker_creation(
&self,
worker: &RuntimeWorkerRef,
create_fingerprint: &str,
binding: &worker_runtime::catalog::BackendJobExecutionBinding,
) -> Result<Option<InternalWorkerSummary>, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", worker.runtime_id.as_str())?;
validate_backend_identifier("worker_id", worker.worker_id.as_str())?;
self.runtime(worker.runtime_id.as_str())?
.lookup_job_worker_creation(worker.worker_id.as_str(), create_fingerprint, binding)
}
pub fn backend_job_termination(
&self,
worker: &RuntimeWorkerRef,
) -> Result<Option<worker_runtime::execution::WorkerExecutionTermination>, RuntimeRegistryError>
{
validate_backend_identifier("runtime_id", worker.runtime_id.as_str())?;
validate_backend_identifier("worker_id", worker.worker_id.as_str())?;
self.runtime(worker.runtime_id.as_str())?
.backend_job_termination(worker.worker_id.as_str())
}
pub fn reconcile_worker_input(
&self,
worker: &RuntimeWorkerRef,
request: WorkerInputRequest,
) -> Result<WorkerInputResult, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", worker.runtime_id.as_str())?;
validate_backend_identifier("worker_id", worker.worker_id.as_str())?;
self.runtime(worker.runtime_id.as_str())?
.reconcile_worker_input(worker.worker_id.as_str(), request)
}
pub fn send_input(
&self,
worker: &RuntimeWorkerRef,
@@ -3792,6 +3865,111 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
})
}
fn lookup_job_worker_creation(
&self,
worker_id: &str,
create_fingerprint: &str,
binding: &worker_runtime::catalog::BackendJobExecutionBinding,
) -> Result<Option<InternalWorkerSummary>, RuntimeRegistryError> {
let worker_ref =
self.worker_ref(worker_id)
.ok_or_else(|| RuntimeRegistryError::UnknownWorker {
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
})?;
self.runtime
.lookup_job_worker_creation(
&self.workspace_id,
&worker_ref,
create_fingerprint,
binding,
)
.map(|worker| worker.map(|worker| self.map_worker_detail(worker)))
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id.clone(),
code: "embedded_job_creation_lookup_failed".into(),
message: error.to_string(),
})
}
fn backend_job_termination(
&self,
worker_id: &str,
) -> Result<Option<worker_runtime::execution::WorkerExecutionTermination>, RuntimeRegistryError>
{
let worker_ref =
self.worker_ref(worker_id)
.ok_or_else(|| RuntimeRegistryError::UnknownWorker {
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
})?;
self.runtime
.execution_termination(&worker_ref)
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id.clone(),
code: "embedded_worker_termination_lookup_failed".into(),
message: error.to_string(),
})
}
fn reconcile_worker_input(
&self,
worker_id: &str,
request: WorkerInputRequest,
) -> Result<WorkerInputResult, RuntimeRegistryError> {
if !self.execution_enabled {
return Err(RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id.clone(),
code: "embedded_worker_execution_unavailable".into(),
message: "Runtime has no execution backend for receipt lookup".into(),
});
}
let worker_ref =
self.worker_ref(worker_id)
.ok_or_else(|| RuntimeRegistryError::UnknownWorker {
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
})?;
let input = EmbeddedWorkerInput {
kind: match request.kind {
WorkerInputKind::User => EmbeddedWorkerInputKind::User,
WorkerInputKind::UserIfIdle => EmbeddedWorkerInputKind::UserIfIdle,
WorkerInputKind::Notify => EmbeddedWorkerInputKind::Notify,
WorkerInputKind::Compact => EmbeddedWorkerInputKind::Compact,
WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer,
},
content: request.content,
source: request.source,
submission_request_id: request.submission_request_id,
segments: request.segments,
};
let receipt = self
.runtime
.reconcile_input(&worker_ref, input)
.map_err(|error| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: self.runtime_id.clone(),
code: "embedded_worker_receipt_lookup_failed".into(),
message: error.to_string(),
})?;
if receipt.is_accepted() {
Ok(WorkerInputResult {
state: InternalWorkerOperationState::Accepted,
disposition: WorkerInputDisposition::Accepted,
worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id),
runtime_run_id: receipt.submission.map(|ack| ack.submission_request_id),
diagnostics: Vec::new(),
})
} else {
Ok(embedded_input_rejected(
&self.runtime_id,
worker_id,
WorkerInputDisposition::Unknown,
diagnostic(
"embedded_worker_receipt_not_confirmed",
HostDiagnosticSeverity::Warning,
receipt.message_or_default(),
),
))
}
}
fn send_input(&self, worker_id: &str, request: WorkerInputRequest) -> WorkerInputResult {
if !self.execution_enabled {
return embedded_input_rejected(
@@ -8587,6 +8765,122 @@ mod tests {
assert!(initial_worker_input(&[]).is_none());
}
#[test]
fn registry_receipt_lookup_uses_embedded_read_boundary_without_input_dispatch() {
use worker_runtime::execution::{
WorkerExecutionBackend, WorkerExecutionOperation, WorkerExecutionResult,
};
#[derive(Default)]
struct ReceiptBackend {
creation: AcceptingExecutionBackend,
receipt: Mutex<Option<WorkerExecutionResult>>,
queries: Mutex<Vec<EmbeddedWorkerInput>>,
}
impl WorkerExecutionBackend for ReceiptBackend {
fn backend_id(&self) -> &str {
"receipt-only-test-backend"
}
fn spawn_worker(
&self,
request: worker_runtime::execution::WorkerExecutionSpawnRequest,
) -> worker_runtime::execution::WorkerExecutionSpawnResult {
self.creation.spawn_worker(request)
}
fn dispatch_input(
&self,
_: &EmbeddedWorkerRef,
_: EmbeddedWorkerInput,
) -> WorkerExecutionResult {
panic!("a receipt query must not dispatch input")
}
fn reconcile_submission(
&self,
_: &EmbeddedWorkerRef,
input: EmbeddedWorkerInput,
) -> WorkerExecutionResult {
self.queries.lock().unwrap().push(input);
self.receipt.lock().unwrap().clone().unwrap_or_else(|| {
WorkerExecutionResult::errored(
WorkerExecutionOperation::Input,
"no stored receipt",
)
})
}
}
let backend = Arc::new(ReceiptBackend::default());
let runtime = EmbeddedWorkerRuntime::new_memory_with_execution_backend(
"workspace-test",
backend.clone(),
)
.unwrap();
let mut spawn = embedded_spawn_request();
spawn.intent = WorkerSpawnIntent::BackendJob {
job_id: "job-1".into(),
attempt_id: "attempt-1".into(),
purpose: "receipt-only test".into(),
input_digest: format!("sha256:{}", "a".repeat(64)),
subjektiv_consolidation: false,
subjektiv_journal: false,
};
let binding = worker_runtime::catalog::BackendJobExecutionBinding {
job_id: "job-1".into(),
attempt_id: "attempt-1".into(),
input_digest: Some(format!("sha256:{}", "a".repeat(64))),
subjektiv_consolidation: false,
subjektiv_journal: false,
};
let created = runtime.spawn_worker(test_create_binding(), spawn);
let worker = created.worker.unwrap().worker;
let registry = RuntimeRegistry::for_workspace(runtime);
let saved = registry
.lookup_job_worker_creation(&worker, "sha256:test-create", &binding)
.unwrap()
.unwrap();
assert_eq!(saved.worker, worker);
let mut foreign = binding.clone();
foreign.attempt_id = "attempt-foreign".into();
assert!(
registry
.lookup_job_worker_creation(&worker, "sha256:test-create", &foreign)
.is_err()
);
assert!(
registry
.lookup_job_worker_creation(&worker, "foreign-create", &binding)
.is_err()
);
let request = || WorkerInputRequest {
source: None,
kind: WorkerInputKind::User,
content: "frozen input".into(),
submission_request_id: Some("stored-request".into()),
segments: None,
};
let absent = registry.reconcile_worker_input(&worker, request()).unwrap();
assert_eq!(absent.disposition, WorkerInputDisposition::Unknown);
*backend.receipt.lock().unwrap() = Some(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
"stored-request",
"stored-submission",
protocol::SubmissionDisposition::Started,
));
let accepted = registry.reconcile_worker_input(&worker, request()).unwrap();
assert_eq!(accepted.state, InternalWorkerOperationState::Accepted);
assert_eq!(accepted.runtime_run_id.as_deref(), Some("stored-request"));
*backend.receipt.lock().unwrap() = Some(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input,
"foreign-request",
"stored-submission",
protocol::SubmissionDisposition::Started,
));
let foreign = registry.reconcile_worker_input(&worker, request()).unwrap();
assert_eq!(foreign.disposition, WorkerInputDisposition::Unknown);
let queries = backend.queries.lock().unwrap();
assert_eq!(queries.len(), 3);
assert!(queries.iter().all(|input| input.content == "frozen input"
&& input.submission_request_id.as_deref() == Some("stored-request")));
}
#[test]
fn embedded_runtime_with_execution_backend_routes_input_and_updates_status() {
let runtime = EmbeddedWorkerRuntime::new_memory_with_execution_backend(
crates/workspace-server/src/latest_schema.sql
+3 -2
View File
@@ -799,7 +799,7 @@ CREATE TABLE backend_jobs (
resource_key TEXT,
request_json TEXT NOT NULL,
intent_fingerprint TEXT NOT NULL,
state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown')),
state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed')),
current_attempt INTEGER NOT NULL CHECK (current_attempt > 0 AND current_attempt <= 3),
result_json TEXT,
result_digest TEXT,
@@ -823,7 +823,7 @@ CREATE TABLE backend_job_attempts (
attempt_id TEXT NOT NULL,
attempt INTEGER NOT NULL CHECK (attempt > 0 AND attempt <= 3),
input_digest TEXT NOT NULL,
state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatching', 'dispatched', 'completed', 'failed', 'unknown')),
state TEXT NOT NULL CHECK (state IN ('reserved', 'dispatching', 'dispatched', 'completed', 'failed')),
runtime_id TEXT,
worker_id TEXT,
runtime_run_id TEXT,
@@ -843,6 +843,7 @@ CREATE TABLE backend_job_attempts (
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
completed_at TEXT,
allocation_precluded_at TEXT,
PRIMARY KEY (workspace_id, job_id, attempt_id),
UNIQUE (workspace_id, job_id, attempt),
UNIQUE (workspace_id, runtime_id, worker_id),
crates/workspace-server/src/lib.rs
+1
View File
@@ -3,6 +3,7 @@
pub mod auth;
pub mod authority;
pub mod backend_job;
mod backend_job_host_effects;
pub mod companion;
pub mod config;
pub mod config_adoption;
crates/workspace-server/src/server.rs
+371 -1615
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/server/backend_job_recovery.rs
+1217
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/server/subjektiv_job.rs
+41
View File
@@ -241,6 +241,47 @@ pub(super) fn dispatch(
})
}
/// Read the existing candidate dispositions and terminal surface outcome for this
/// exact consolidation attempt. Recovery does not rerun candidate decisions or
/// surface generation, and migration-retired Jobs never reach the domain lookup.
pub(super) fn lookup_result(
api: &WorkspaceApi,
job: &crate::backend_job::BackendJobRecord,
attempt: &BackendJobAttemptRecord,
) -> Result<Option<serde_json::Value>> {
if api
.store
.backend_job_was_migrated(api.workspace_id(), &job.request.job_id)?
{
return Ok(None);
}
let Some(grant) = &job.request.grants.subjektiv_consolidation else {
return Ok(None);
};
if job.workspace_id != api.workspace_id()
|| attempt.workspace_id != job.workspace_id
|| attempt.job_id != job.request.job_id
|| attempt.attempt != job.current_attempt
|| attempt.attempt_id != crate::backend_job::attempt_id(&attempt.job_id, attempt.attempt)
|| attempt.input_digest != job.input_digest
{
return Err(Error::WorkspacePermissionDenied(
"Consolidation recovery owner mismatch".into(),
));
}
job.load_executable_request()?;
subject_authorization::authorize_job_input(api, &job.request).map_err(|error| error.error)?;
let store = open_subjektiv_store(api).map_err(|error| error.error)?;
subjektiv::job::lookup_result(
&store,
&grant.subject_id,
&grant.candidate_ids,
&job.request.job_id,
&attempt.attempt_id,
)
.map_err(Error::from)
}
pub(super) fn validate_result(
api: &WorkspaceApi,
request: &BackendJobRequest,
crates/workspace-server/src/server/subjektiv_journal.rs
+640 -389
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/server/tests/backend_job_tests.rs
+1664
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs
+673 -365
View File
File diff suppressed because it is too large Load Diff
crates/workspace-server/src/storage_format_tests.rs
+119 -52
View File
@@ -110,13 +110,18 @@ fn assert_accepted_without_domain_changes(conn: &Connection, current: i64) {
verify_history(conn, current).unwrap();
if current >= 114 {
let plan = published_plan(conn, current).unwrap().unwrap();
assert_eq!(plan.migrations.len(), 1);
assert_eq!(plan.migrations.len(), 2);
assert_eq!(plan.migrations[0].version, 122);
assert_eq!(
plan.migrations[0].name,
crate::config_adoption::MIGRATION_NAME
);
assert_eq!(plan.target_schema_version, 122);
assert_eq!(plan.migrations[1].version, 123);
assert_eq!(
plan.migrations[1].name,
super::super::backend_job_upgrade::MIGRATION_NAME
);
assert_eq!(plan.target_schema_version, 123);
}
assert_eq!(rows, saved_rows(conn));
assert_eq!(shape, snapshot(conn).unwrap());
@@ -171,59 +176,121 @@ fn converged_shapes_keep_all_real_predecessor_receipts_and_erasure_fences() {
}
}
// Snapshot every retained value except the three fields the new migration owns.
// Existing published-family history tables remain byte-for-byte in this check.
fn immutable_rows(conn: &Connection) -> Vec<(String, Vec<String>)> {
let mut result = Vec::new();
for (name, _) in saved_rows(conn) {
if name == "__yoi_schema_migrations" {
continue;
}
let columns = super::super::table_columns(conn, &name)
.unwrap()
.into_iter()
.filter(|column| {
!["backend_jobs", "backend_job_attempts"].contains(&name.as_str())
|| !["state", "failure_category", "failure_detail"].contains(&column.as_str())
})
.map(|column| format!("\"{}\"", column.replace('"', "\"\"")))
.collect::<Vec<_>>()
.join(",");
let mut statement = conn
.prepare(&format!("SELECT {columns} FROM \"{name}\""))
.unwrap();
let count = statement.column_count();
let mut rows = statement
.query_map([], |row| {
(0..count)
.map(|index| row.get::<_, rusqlite::types::Value>(index))
.collect::<rusqlite::Result<Vec<_>>>()
})
.unwrap()
.map(|row| format!("{:?}", row.unwrap()))
.collect::<Vec<_>>();
rows.sort();
result.push((name, rows));
}
result
}
fn exercise_published_runner(stage: Stage, baseline: i64, predecessor: Option<(Family, usize)>) {
let source = fixture(stage, baseline, predecessor);
let expected = immutable_rows(&source);
let original_receipts = super::super::workspace_schema_migration_history(&source).unwrap();
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("catalog.db");
let mut destination = Connection::open(&path).unwrap();
Backup::new(&source, &mut destination)
.unwrap()
.run_to_completion(128, std::time::Duration::ZERO, None)
.unwrap();
drop(destination);
let mut committed_history = None;
for _ in 0..2 {
let conn = Connection::open(&path).unwrap();
// Actual production open/ordered runner, including the new migration.
super::super::prepare_connection(&conn).unwrap();
assert_eq!(super::super::current_schema_version(&conn).unwrap(), 123);
let actual = immutable_rows(&conn);
for (table, rows) in &expected {
assert_eq!(
&actual.iter().find(|(name, _)| name == table).unwrap().1,
rows,
"{table}, from {stage:?}, baseline {baseline}"
);
}
let outcome:(String,String,Option<String>)=conn.query_row("SELECT state,failure_category,completed_at FROM backend_jobs WHERE job_id='saved-job'",[],|row|Ok((row.get(0)?,row.get(1)?,row.get(2)?))).unwrap();
assert_eq!(
outcome,
(
"failed".into(),
super::super::backend_job_upgrade::FAILURE_CATEGORY.into(),
None
)
);
let mut history = super::super::workspace_schema_migration_history(&conn).unwrap();
assert_eq!(
history.pop(),
Some((
123,
super::super::backend_job_upgrade::MIGRATION_NAME.into()
))
);
assert_eq!(
history.pop(),
Some((122, crate::config_adoption::MIGRATION_NAME.into()))
);
assert_eq!(history, original_receipts);
verify_history(&conn, 123).unwrap();
let saved = saved_rows(&conn);
if let Some(previous) = &committed_history {
assert_eq!(previous, &saved);
} else {
committed_history = Some(saved);
}
}
}
#[test]
fn published_database_runs_normal_store_preparation_and_reopens_without_downgrade() {
for stage in [
Stage::Published(Family::Permissions, 2),
Stage::Published(Family::Removal, 4),
Stage::Published(Family::Jobs, 5),
Stage::Converged(119),
Stage::Converged(120),
Stage::Converged(121),
] {
let current = match stage {
Stage::Published(_, count) => 113 + count as i64,
Stage::Converged(version) => version,
Stage::Base => 113,
};
let source = fixture(stage, 113, None);
let expected = saved_rows(&source);
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("catalog.db");
let mut destination = Connection::open(&path).unwrap();
Backup::new(&source, &mut destination)
.unwrap()
.run_to_completion(128, std::time::Duration::ZERO, None)
.unwrap();
drop(destination);
for _ in 0..2 {
let conn = Connection::open(&path).unwrap();
// Production store owner, not a separate test migration implementation.
super::super::prepare_connection(&conn).unwrap();
assert_eq!(super::super::current_schema_version(&conn).unwrap(), 122);
for (table, rows) in &expected {
if table != "__yoi_schema_migrations" {
assert_eq!(
saved_rows(&conn)
.into_iter()
.find(|(name, _)| name == table)
.unwrap()
.1,
*rows,
"{table}, from {current}"
);
}
for family in FAMILIES {
for count in 1..=family.names().len() {
let stage = Stage::Published(family, count);
for baseline in 113..=113 + count as i64 {
exercise_published_runner(stage, baseline, None);
}
let mut history = super::super::workspace_schema_migration_history(&conn).unwrap();
assert_eq!(
history.pop(),
Some((122, crate::config_adoption::MIGRATION_NAME.into()))
);
assert_eq!(
history,
super::super::workspace_schema_migration_history(&source).unwrap()
);
verify_history(&conn, 122).unwrap();
}
}
for current in 119..=121 {
for baseline in 119..=current {
exercise_published_runner(Stage::Converged(current), baseline, None);
}
for origin in
std::iter::once(None).chain(FAMILIES.into_iter().flat_map(|family| {
(1..=family.names().len()).map(move |count| Some((family, count)))
}))
{
exercise_published_runner(Stage::Converged(current), 113, origin);
}
}
}
crates/workspace-server/src/storage_formats.rs
+168 -12
View File
@@ -75,6 +75,26 @@ pub(super) fn verify_history(conn: &Connection, current: i64) -> Result<()> {
}
fn verify_history_records(conn: &Connection, current: i64, rows: &[(i64, String)]) -> Result<()> {
verify_raw_capture_shape(conn, current, rows)?;
if current == 123 {
verify_execution_shape(conn)?;
super::backend_job_upgrade::verify(conn)?;
if rows == [(123, super::SCHEMA_BASELINE_NAME.to_owned())] {
verify_configuration_shape(conn)?;
require_shape(conn, Stage::Base)?;
return verify_103_features(conn, current, rows);
}
if rows.last().map(|(version, name)| (*version, name.as_str()))
!= Some((123, super::backend_job_upgrade::MIGRATION_NAME))
{
return Err(invalid("unrecognized Backend Job execution receipt"));
}
let prefix = &rows[..rows.len() - 1];
if prefix.last().map(|row| row.0) != Some(122) {
return Err(invalid("missing Backend Job execution predecessor"));
}
return verify_history_records(conn, 122, prefix);
}
if current == 122 {
verify_configuration_shape(conn)?;
if rows == [(122, super::SCHEMA_BASELINE_NAME.to_owned())] {
@@ -216,15 +236,18 @@ pub(super) fn published_plan(
verify_history(conn, current)?;
Ok(Some(super::WorkspaceSchemaMigrationPlan {
current_schema_version: current,
target_schema_version: 122,
migrations: if current < 122 {
vec![super::WorkspaceSchemaMigrationStep {
version: 122,
name: crate::config_adoption::MIGRATION_NAME.to_owned(),
}]
} else {
vec![]
},
target_schema_version: 123,
migrations: [
(122, crate::config_adoption::MIGRATION_NAME),
(123, super::backend_job_upgrade::MIGRATION_NAME),
]
.into_iter()
.filter(|(version, _)| *version > current)
.map(|(version, name)| super::WorkspaceSchemaMigrationStep {
version,
name: name.to_owned(),
})
.collect(),
}))
}
@@ -347,6 +370,139 @@ fn build_references() -> Result<References> {
})
}
fn extension_shape(sql: &str) -> Result<Shape> {
let reference = Connection::open_in_memory()?;
reference.execute_batch("CREATE TABLE backend_jobs(workspace_id TEXT,job_id TEXT,request_json TEXT); CREATE TABLE backend_job_attempts(workspace_id TEXT,job_id TEXT,attempt_id TEXT); CREATE TABLE backend_job_deliveries(workspace_id TEXT,delivery_id TEXT);")?;
reference.execute_batch(sql)?;
let mut shape = snapshot(&reference)?;
for name in [
"backend_jobs",
"backend_job_attempts",
"backend_job_deliveries",
] {
shape.remove(name);
}
Ok(shape)
}
fn require_extension(conn: &Connection, sql: &str) -> Result<Shape> {
let expected = extension_shape(sql)?;
let actual = snapshot(conn)?;
let tables: BTreeSet<_> = expected
.iter()
.filter(|(_, object)| object.0 == "table")
.map(|(_, object)| object.1.clone())
.collect();
for (name, object) in &actual {
if tables.contains(&object.1) && !expected.contains_key(name) {
return Err(invalid(format!("unreceipted Job evidence object: {name}")));
}
}
for (name, object) in &expected {
if actual.get(name) != Some(object) {
return Err(invalid(format!("Job evidence schema drift: {name}")));
}
}
Ok(expected)
}
fn verify_raw_capture_shape(conn: &Connection, current: i64, rows: &[(i64, String)]) -> Result<()> {
let expected = extension_shape(super::backend_job_upgrade::RAW_SCHEMA)?;
let actual = snapshot(conn)?;
let mut execution = extension_shape(super::backend_job_upgrade::SCHEMA)?;
execution.extend(extension_shape(super::backend_job_recovery_store::SCHEMA)?);
if current < 123
&& super::current_schema_version(conn)? < 123
&& execution.keys().any(|name| actual.contains_key(name))
{
return Err(invalid("Backend Job execution objects before receipt 123"));
}
let present = expected.keys().any(|name| actual.contains_key(name));
if !present {
return Ok(());
}
if current < 86 {
return Err(invalid("pre-content capture before its schema-86 boundary"));
}
require_extension(conn, super::backend_job_upgrade::RAW_SCHEMA)?;
let populated: bool = conn.query_row(
"SELECT EXISTS(SELECT 1 FROM backend_job_pre_content_history)",
[],
|row| row.get(0),
)?;
if populated
&& !(rows.first().is_some_and(|row| row.0 <= 85) && rows.iter().any(|row| row.0 == 86))
{
return Err(invalid(
"populated pre-content capture without retained schema-86 history",
));
}
super::backend_job_upgrade::verify_pre_content(conn)
}
fn verify_execution_shape(conn: &Connection) -> Result<()> {
require_extension(conn, super::backend_job_upgrade::RAW_SCHEMA)?;
require_extension(conn, super::backend_job_upgrade::SCHEMA)?;
require_extension(conn, super::backend_job_recovery_store::SCHEMA)?;
Ok(())
}
// Ignore only separately validated evidence programs, not arbitrary extra Job
// triggers. The retained family's CHECK is transformed only after receipt 123;
// all other retained columns, programs and original receipt names remain exact.
fn legacy_shape(conn: &Connection) -> Result<Shape> {
let mut shape = snapshot(conn)?;
for sql in [
super::backend_job_upgrade::RAW_SCHEMA,
super::backend_job_upgrade::SCHEMA,
super::backend_job_recovery_store::SCHEMA,
] {
for name in extension_shape(sql)?.keys() {
shape.remove(name);
}
}
Ok(shape)
}
fn upgraded_reference(conn: &Connection, mut shape: Shape) -> Result<Shape> {
if super::current_schema_version(conn)? != 123 {
return Ok(shape);
}
for table in ["backend_jobs", "backend_job_attempts"] {
if let Some((_, _, tokens)) = shape.get_mut(table) {
if table == "backend_job_attempts" {
// This exact nullable declaration belongs only to receipt 123.
// Canonical declarations are sorted; allocation precedes attempt.
let start = tokens
.iter()
.position(|token| token == "(")
.ok_or_else(|| invalid("missing attempt declarations"))?;
tokens.splice(
start + 1..start + 1,
[
"word:allocation_precluded_at".into(),
"word:text".into(),
",".into(),
],
);
}
// These exact retained state declarations put the legacy value last.
let start = tokens
.windows(5)
.position(|w| w == ["word:check", "(", "word:state", "word:in", "("]);
if let Some(start) = start {
if let Some(end) = tokens[start + 5..].iter().position(|token| token == ")") {
let end = start + 5 + end;
if end >= 2 && tokens[end - 1] == "literal:unknown" && tokens[end - 2] == "," {
tokens.drain(end - 2..end);
}
}
}
}
}
Ok(shape)
}
fn require_shape(conn: &Connection, stage: Stage) -> Result<()> {
let references = references()?;
let selected = |shape: Shape| -> Shape {
@@ -355,8 +511,8 @@ fn require_shape(conn: &Connection, stage: Stage) -> Result<()> {
.filter(|(_, object)| references.tables.contains(&object.1))
.collect()
};
let expected = selected(references.stages[&stage].clone());
let actual = selected(snapshot(conn)?);
let expected = selected(upgraded_reference(conn, references.stages[&stage].clone())?);
let actual = selected(legacy_shape(conn)?);
if actual != expected {
let changed: BTreeSet<_> = actual
.keys()
@@ -373,7 +529,7 @@ fn reject_early_tail(conn: &Connection, current: i64) -> Result<()> {
return Ok(());
}
let references = references()?;
let actual = snapshot(conn)?;
let actual = legacy_shape(conn)?;
let base = &references.stages[&Stage::Base];
for (name, object) in &actual {
if references.tables.contains(&object.1) && !base.contains_key(name) {
crates/workspace-server/src/store.rs
+404 -88
View File
@@ -129,10 +129,15 @@ pub use ticket::{
};
const OLDEST_SCHEMA_VERSION: i64 = 50;
#[path = "backend_job_recovery_store.rs"]
mod backend_job_recovery_store;
#[path = "backend_job_upgrade.rs"]
pub(crate) mod backend_job_upgrade;
#[path = "storage_formats.rs"]
mod storage_formats;
pub use backend_job_recovery_store::BackendJobReconciliation;
const LATEST_SCHEMA_VERSION: i64 = 122;
const LATEST_SCHEMA_VERSION: i64 = 123;
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
@@ -510,6 +515,11 @@ const MIGRATIONS: &[Migration] = &[
name: crate::config_adoption::MIGRATION_NAME,
apply: migrate_configuration_adoption_to_v122,
},
Migration {
version: 123,
name: backend_job_upgrade::MIGRATION_NAME,
apply: migrate_backend_job_execution_to_v123,
},
];
// Both schema-103 identities shipped. Keep the alternate implementation and accept
@@ -1961,7 +1971,8 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
) -> Result<BackendJobReservation>;
fn get_backend_job(&self, workspace_id: &str, job_id: &str)
-> Result<Option<BackendJobRecord>>;
/// Pending/unknown Jobs and terminal Jobs with unfinished Worker cleanup
fn backend_job_was_migrated(&self, workspace_id: &str, job_id: &str) -> Result<bool>;
/// Pending Jobs and terminal Jobs with unfinished Worker cleanup
/// retain their resource. This is a query, not an independent lock ledger.
fn find_active_backend_job_for_resource(
&self,
@@ -1981,6 +1992,70 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
job_id: &str,
attempt_id: &str,
) -> Result<Option<BackendJobAttemptRecord>>;
/// Full ordered execution history for exact-owner release CAS.
fn list_backend_job_attempts(
&self,
workspace_id: &str,
job_id: &str,
) -> Result<Vec<BackendJobAttemptRecord>>;
fn record_backend_job_reconciliation(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
failure_category: &str,
failure_detail: &str,
next_action: &str,
now: &str,
) -> Result<BackendJobAttemptRecord>;
fn probe_backend_job_recovery(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
) -> Result<bool>;
fn record_backend_job_execution_fence(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
controller_evidence: &str,
host_evidence: &str,
now: &str,
) -> Result<BackendJobAttemptRecord>;
fn has_backend_job_execution_fence(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
) -> Result<bool>;
/// Host-only saved non-Journal result reconciliation; no model/finalizer rerun.
fn reconcile_backend_job_domain_result(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
submission: &BackendJobResultSubmission,
now: &str,
) -> Result<BackendJobAttemptRecord>;
fn get_backend_job_reconciliation(
&self,
workspace_id: &str,
job_id: &str,
attempt_id: &str,
) -> Result<Option<BackendJobReconciliation>>;
fn claim_backend_job_worker_cleanup_owned(
&self,
workspace_id: &str,
expected_history: &[BackendJobAttemptRecord],
expected: &BackendJobAttemptRecord,
now: &str,
) -> Result<(BackendJobAttemptRecord, bool)>;
fn finish_backend_job_worker_cleanup_owned(
&self,
workspace_id: &str,
expected_history: &[BackendJobAttemptRecord],
expected: &BackendJobAttemptRecord,
state: BackendJobWorkerCleanupState,
failure_category: Option<&str>,
failure_detail: Option<&str>,
now: &str,
) -> Result<BackendJobAttemptRecord>;
fn list_recoverable_backend_job_attempts(
&self,
workspace_id: &str,
@@ -2032,8 +2107,8 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
completion: Option<&BackendJobResultSubmission>,
now: &str,
) -> Result<BackendJobAttemptRecord>;
/// Bounded keyset over current Journal Unknown attempts, independent of source liveness.
fn list_backend_journal_unknown_attempts(
/// Bounded keyset over current Journal Pending attempts, independent of source liveness.
fn list_backend_journal_pending_attempts(
&self,
workspace_id: &str,
after_job_id: Option<&str>,
@@ -3786,6 +3861,57 @@ impl SqliteWorkspaceStore {
)?))
}
/// Read only: recover the original create reservation, never create/restore a
/// Worker. The workspace-store caller must separately check registry Job binding.
pub(crate) fn lookup_backend_job_allocation(
&self,
workspace_id: &str,
job: &BackendJobRecord,
attempt: &BackendJobAttemptRecord,
) -> Result<Option<(WorkerCreateReservation, String)>> {
if job.workspace_id != workspace_id
|| attempt.workspace_id != workspace_id
|| job.request.job_id != attempt.job_id
|| job.current_attempt != attempt.attempt
|| job.input_digest != attempt.input_digest
|| job.state != BackendJobState::Pending
|| attempt.state.terminal()
|| attempt.attempt_id
!= crate::backend_job::attempt_id(&job.request.job_id, attempt.attempt)
{
return Err(Error::InvalidInput(
"allocation lookup requires the exact current Job intent".into(),
));
}
job.load_executable_request()?;
let key = crate::backend_job::allocation_key(&attempt.attempt_id);
self.with_conn(|conn| {
let saved = conn.query_row("SELECT worker_id,runtime_id,request_fingerprint,create_fingerprint,memory_language,state,singleton_key,singleton_generation FROM worker_create_reservations WHERE workspace_id=?1 AND allocation_key=?2",params![workspace_id,key],|row|Ok((row.get::<_,String>(0)?,row.get::<_,String>(1)?,row.get::<_,Option<String>>(2)?,row.get::<_,String>(3)?,row.get::<_,Option<String>>(4)?,row.get::<_,String>(5)?,row.get::<_,Option<String>>(6)?,row.get::<_,Option<i64>>(7)?))).optional()?;
let Some((worker_id,runtime,request_fingerprint,create_fingerprint,language,state,singleton,generation))=saved else { return Ok(None); };
if state == "removed" || singleton.is_some() || generation.is_some() {
return Err(Error::RepositoryConflict("original Job allocation is removed or has different create intent".into()));
}
// The trusted reservation fixes the original create intent, including
// its then-selected Profile. Reading that evidence does not resolve
// today's Profile/default or issue any new execution grant.
let request_fingerprint = request_fingerprint.filter(|fingerprint| !fingerprint.is_empty())
.ok_or_else(|| Error::Store("original Job create fingerprint missing".into()))?;
let snapshot = manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: workspace_id.into(),
language: language.ok_or_else(||Error::Store("Job allocation Memory snapshot missing".into()))?,
};
validate_workspace_memory_settings_snapshot(&snapshot,workspace_id)?;
if create_fingerprint != bound_worker_create_fingerprint(&request_fingerprint,&snapshot) {
return Err(Error::RepositoryConflict("Job allocation create fingerprint mismatch".into()));
}
let worker_id = worker_id.parse::<WorkerId>().map_err(|_|Error::Store("invalid original Job Worker identity".into()))?;
if attempt.worker.as_ref().is_some_and(|worker| worker.runtime_id != runtime || worker.worker_id != worker_id.to_string()) {
return Err(Error::RepositoryConflict("Job allocation differs from saved execution binding".into()));
}
Ok(Some((WorkerCreateReservation{worker_id,create_fingerprint,memory_settings:snapshot,singleton:None},runtime)))
})
}
pub(crate) fn reserved_worker_create_runtime(
&self,
workspace_id: &str,
@@ -7968,6 +8094,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<BackendJobReservation> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?;
let mut job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
@@ -8030,6 +8157,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
self.with_conn(|conn| read_backend_job(conn, workspace_id, job_id))
}
fn backend_job_was_migrated(&self, workspace_id: &str, job_id: &str) -> Result<bool> {
self.with_conn(|conn| Ok(conn.query_row(
"SELECT EXISTS(SELECT 1 FROM backend_job_upgrade_history WHERE workspace_id=?1 AND job_id=?2)",
params![workspace_id, job_id], |row| row.get(0),
)?))
}
fn find_active_backend_job_for_resource(
&self,
workspace_id: &str,
@@ -8069,6 +8203,122 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
self.with_conn(|conn| read_backend_job_attempt(conn, workspace_id, job_id, attempt_id))
}
fn list_backend_job_attempts(
&self,
workspace_id: &str,
job_id: &str,
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn(|conn| backend_job_recovery_store::history(conn, workspace_id, job_id))
}
fn record_backend_job_reconciliation(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
failure_category: &str,
failure_detail: &str,
next_action: &str,
now: &str,
) -> Result<BackendJobAttemptRecord> {
backend_job_recovery_store::record(
self,
workspace_id,
expected,
failure_category,
failure_detail,
next_action,
now,
)
}
fn probe_backend_job_recovery(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
) -> Result<bool> {
backend_job_recovery_store::probe(self, workspace_id, expected)
}
fn record_backend_job_execution_fence(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
controller_evidence: &str,
host_evidence: &str,
now: &str,
) -> Result<BackendJobAttemptRecord> {
backend_job_recovery_store::record_fence(
self,
workspace_id,
expected,
controller_evidence,
host_evidence,
now,
)
}
fn has_backend_job_execution_fence(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
) -> Result<bool> {
self.with_conn(|conn| backend_job_recovery_store::has_fence(conn, workspace_id, expected))
}
fn reconcile_backend_job_domain_result(
&self,
workspace_id: &str,
expected: &BackendJobAttemptRecord,
submission: &BackendJobResultSubmission,
now: &str,
) -> Result<BackendJobAttemptRecord> {
backend_job_recovery_store::reconcile_domain_result(
self,
workspace_id,
expected,
submission,
now,
)
}
fn get_backend_job_reconciliation(
&self,
workspace_id: &str,
job_id: &str,
attempt_id: &str,
) -> Result<Option<BackendJobReconciliation>> {
self.with_conn(|conn| conn.query_row("SELECT cause,detail,next_action,updated_at FROM backend_job_reconciliation WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3",params![workspace_id,job_id,attempt_id],|row|Ok(BackendJobReconciliation{cause:row.get(0)?,detail:row.get(1)?,next_action:row.get(2)?,updated_at:row.get(3)?})).optional().map_err(Error::from))
}
fn claim_backend_job_worker_cleanup_owned(
&self,
workspace_id: &str,
expected_history: &[BackendJobAttemptRecord],
expected: &BackendJobAttemptRecord,
now: &str,
) -> Result<(BackendJobAttemptRecord, bool)> {
backend_job_recovery_store::claim_cleanup_owned(
self,
workspace_id,
expected_history,
expected,
now,
)
}
fn finish_backend_job_worker_cleanup_owned(
&self,
workspace_id: &str,
expected_history: &[BackendJobAttemptRecord],
expected: &BackendJobAttemptRecord,
state: BackendJobWorkerCleanupState,
failure_category: Option<&str>,
failure_detail: Option<&str>,
now: &str,
) -> Result<BackendJobAttemptRecord> {
backend_job_recovery_store::finish_cleanup_owned(
self,
workspace_id,
expected_history,
expected,
state,
failure_category,
failure_detail,
now,
)
}
fn list_recoverable_backend_job_attempts(
&self,
workspace_id: &str,
@@ -8076,15 +8326,10 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
// A dispatch claim has no externally meaningful identity until the
// deterministic Worker allocation is bound. Startup can therefore
// replay an interrupted claim through the same allocation key.
tx.execute(
"UPDATE backend_job_attempts SET state = 'reserved' WHERE workspace_id = ?1 AND state = 'dispatching' AND worker_id IS NULL",
params![workspace_id],
)?;
// Preserve the last confirmed dispatch fact. An unbound claim may
// already own an external allocation; the Server reconciles its ID.
let mut statement = tx.prepare(
"SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('reserved', 'dispatched') ORDER BY created_at LIMIT ?2",
"SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j USING(workspace_id,job_id) WHERE a.workspace_id = ?1 AND a.state IN ('reserved', 'dispatching', 'dispatched') AND a.attempt=j.current_attempt AND j.state='pending' AND j.input_disposition='available' AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=a.workspace_id AND h.job_id=a.job_id) ORDER BY a.created_at LIMIT ?2",
)?;
let identities = statement.query_map(params![workspace_id, limit as i64], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
@@ -8108,6 +8353,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<(BackendJobReservation, bool)> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?;
let job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
@@ -8149,7 +8395,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn(|conn| {
let mut statement = conn.prepare(
"SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='reserved' AND j.input_disposition='available' ORDER BY a.created_at, a.attempt_id LIMIT ?2",
"SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='reserved' AND j.input_disposition='available' AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=a.workspace_id AND h.job_id=a.job_id) ORDER BY a.created_at, a.attempt_id LIMIT ?2",
)?;
let identities = statement.query_map(params![workspace_id, limit as i64], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
@@ -8172,6 +8418,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<(WorkerRegistryProjectionCommit, bool)> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?;
let job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
@@ -8237,6 +8484,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
}
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?;
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
@@ -8323,6 +8571,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
}
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
backend_job_recovery_store::require_current_contract(&tx, workspace_id, &submission.job_id)?;
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
return Err(Error::InvalidInput(
"Backend Job attempt no longer accepts results".to_string(),
@@ -8349,16 +8598,16 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
})
}
fn list_backend_journal_unknown_attempts(
fn list_backend_journal_pending_attempts(
&self,
workspace_id: &str,
after_job_id: Option<&str>,
limit: usize,
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn(|conn| {
let mut stmt = conn.prepare("SELECT a.job_id,a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='unknown' AND j.state='unknown' AND a.attempt=j.current_attempt AND json_type(j.request_json,'$.grants.subjektiv_journal')='object' AND (?2 IS NULL OR a.job_id>?2) ORDER BY a.job_id LIMIT ?3")?;
let mut stmt = conn.prepare("SELECT a.job_id,a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='dispatched' AND j.state='pending' AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=a.workspace_id AND h.job_id=a.job_id) AND a.attempt=j.current_attempt AND json_type(j.request_json,'$.grants.subjektiv_journal')='object' AND (?2 IS NULL OR a.job_id>?2) ORDER BY a.job_id LIMIT ?3")?;
let ids = stmt.query_map(params![workspace_id,after_job_id,limit.clamp(1,100) as i64], |r| Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
ids.into_iter().map(|(job,id)| read_backend_job_attempt(conn,workspace_id,&job,&id)?.ok_or_else(|| Error::Store("Journal Unknown attempt disappeared".into()))).collect()
ids.into_iter().map(|(job,id)| read_backend_job_attempt(conn,workspace_id,&job,&id)?.ok_or_else(|| Error::Store("Journal Pending attempt disappeared".into()))).collect()
})
}
@@ -8385,6 +8634,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<BackendJobAttemptRecord> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
backend_job_recovery_store::require_current_contract(&tx, workspace_id, &expected.job_id)?;
let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?;
@@ -8392,7 +8642,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
if &attempt != expected || job.current_attempt != attempt.attempt
|| job.request.grants.subjektiv_journal.is_none()
|| attempt.input_digest != job.input_digest
|| !matches!(job.state, BackendJobState::Unknown | BackendJobState::Completed | BackendJobState::Failed) {
|| !matches!(job.state, BackendJobState::Pending | BackendJobState::Completed | BackendJobState::Failed) {
return Err(Error::InvalidInput("Journal reconciliation binding changed".into()));
}
if let Some(submission) = completion {
@@ -8401,25 +8651,25 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
}
let (digest, json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
if attempt.state == BackendJobAttemptState::Completed && attempt.result_digest.as_deref() == Some(digest.as_str()) { tx.commit()?; return Ok(attempt); }
if attempt.state != BackendJobAttemptState::Unknown || job.state != BackendJobState::Unknown {
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
return Err(Error::InvalidInput("Journal completion cannot replace definitive outcome".into()));
}
tx.execute("UPDATE backend_job_attempts SET state='completed',result_json=?4,result_digest=?5,failure_category=NULL,failure_detail=NULL,updated_at=?6,completed_at=?6 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='unknown'",params![workspace_id,expected.job_id,expected.attempt_id,json,digest,now])?;
tx.execute("UPDATE backend_jobs SET state='completed',result_json=?3,result_digest=?4,failure_category=NULL,failure_detail=NULL,updated_at=?5,completed_at=?5 WHERE workspace_id=?1 AND job_id=?2 AND state='unknown'",params![workspace_id,expected.job_id,json,digest,now])?;
tx.execute("UPDATE backend_job_attempts SET state='completed',result_json=?4,result_digest=?5,failure_category=NULL,failure_detail=NULL,worker_cleanup_state=COALESCE(worker_cleanup_state,'pending'),worker_cleanup_updated_at=CASE WHEN worker_cleanup_state IS NULL THEN ?6 ELSE worker_cleanup_updated_at END,updated_at=?6,completed_at=?6 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='dispatched'",params![workspace_id,expected.job_id,expected.attempt_id,json,digest,now])?;
tx.execute("UPDATE backend_jobs SET state='completed',result_json=?3,result_digest=?4,failure_category=NULL,failure_detail=NULL,updated_at=?5,completed_at=?5 WHERE workspace_id=?1 AND job_id=?2 AND state='pending'",params![workspace_id,expected.job_id,json,digest,now])?;
notification_routing::jobs::capture(&tx,workspace_id,&expected.job_id,&expected.attempt_id)?;
} else {
if attempt.state == BackendJobAttemptState::Failed && attempt.failure_category.as_deref() == Some("journal_attempt_fenced") { tx.commit()?; return Ok(attempt); }
if attempt.state != BackendJobAttemptState::Unknown || job.state != BackendJobState::Unknown
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending
|| attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed)
|| attempt.worker_cleanup_completed_at.is_none() {
return Err(Error::InvalidInput("partial Journal Unknown requires completed canonical cleanup fence".into()));
return Err(Error::InvalidInput("partial Journal Pending requires completed canonical cleanup fence".into()));
}
let purged = if let Some(worker) = &attempt.worker {
tx.query_row("SELECT EXISTS(SELECT 1 FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='succeeded' AND session_disposition='purge')",params![workspace_id,worker.runtime_id,worker.worker_id],|r|r.get::<_,bool>(0))?
} else { false };
if !purged { return Err(Error::InvalidInput("partial Journal Unknown has no committed stop/Purge removal receipt".into())); }
tx.execute("UPDATE backend_job_attempts SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed; retained operations remain valid',updated_at=?4 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='unknown'",params![workspace_id,expected.job_id,expected.attempt_id,now])?;
tx.execute("UPDATE backend_jobs SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed',updated_at=?3 WHERE workspace_id=?1 AND job_id=?2 AND state='unknown'",params![workspace_id,expected.job_id,now])?;
if !purged { return Err(Error::InvalidInput("partial Journal Pending has no committed stop/Purge removal receipt".into())); }
tx.execute("UPDATE backend_job_attempts SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed; retained operations remain valid',updated_at=?4 WHERE workspace_id=?1 AND job_id=?2 AND attempt_id=?3 AND state='dispatched'",params![workspace_id,expected.job_id,expected.attempt_id,now])?;
tx.execute("UPDATE backend_jobs SET state='failed',failure_category='journal_attempt_fenced',failure_detail='Canonical Worker cleanup and domain generation fence confirmed',updated_at=?3 WHERE workspace_id=?1 AND job_id=?2 AND state='pending'",params![workspace_id,expected.job_id,now])?;
}
let result = read_backend_job_attempt(&tx,workspace_id,&expected.job_id,&expected.attempt_id)?.ok_or_else(|| Error::Store("Reconciled Journal attempt missing".into()))?;
tx.commit()?; Ok(result)
@@ -8436,12 +8686,9 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
failure_detail: &str,
now: &str,
) -> Result<BackendJobAttemptRecord> {
if !matches!(
state,
BackendJobAttemptState::Failed | BackendJobAttemptState::Unknown
) {
if !matches!(state, BackendJobAttemptState::Failed) {
return Err(Error::InvalidInput(
"Backend Job failure completion requires failed or unknown state".to_string(),
"Backend Job failure completion requires failed state".to_string(),
));
}
let detail = bounded_failure_detail(failure_detail);
@@ -8465,16 +8712,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
tx.commit()?;
return Ok(attempt);
}
backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?;
tx.execute(
"UPDATE backend_job_attempts SET state = ?4, failure_category = ?5, failure_detail = ?6, worker_cleanup_state = CASE WHEN worker_id IS NULL THEN 'completed' ELSE 'pending' END, worker_cleanup_failure_category = NULL, worker_cleanup_failure_detail = NULL, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = CASE WHEN worker_id IS NULL THEN ?7 ELSE NULL END, completed_at = ?7, updated_at = ?7 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND state IN ('reserved', 'dispatching', 'dispatched')",
"UPDATE backend_job_attempts SET state = ?4, failure_category = ?5, failure_detail = ?6, worker_cleanup_state = CASE WHEN state='reserved' AND worker_id IS NULL THEN 'completed' ELSE 'pending' END, worker_cleanup_failure_category = NULL, worker_cleanup_failure_detail = NULL, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = CASE WHEN state='reserved' AND worker_id IS NULL THEN ?7 ELSE NULL END, allocation_precluded_at = CASE WHEN state='reserved' AND runtime_id IS NULL AND worker_id IS NULL AND runtime_run_id IS NULL AND dispatched_at IS NULL THEN ?7 ELSE NULL END, completed_at = ?7, updated_at = ?7 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND state IN ('reserved', 'dispatching', 'dispatched')",
params![workspace_id, job_id, attempt_id, state.as_str(), failure_category, detail, now],
)?;
if attempt.attempt == job.current_attempt && job.state == BackendJobState::Pending {
let job_state = match state {
BackendJobAttemptState::Failed => BackendJobState::Failed,
BackendJobAttemptState::Unknown => BackendJobState::Unknown,
_ => unreachable!(),
};
let job_state = BackendJobState::Failed;
tx.execute(
"UPDATE backend_jobs SET state = ?3, failure_category = ?4, failure_detail = ?5, completed_at = ?6, updated_at = ?6 WHERE workspace_id = ?1 AND job_id = ?2 AND state = 'pending'",
params![workspace_id, job_id, job_state.as_str(), failure_category, detail, now],
@@ -8490,35 +8734,12 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
fn recover_backend_job_worker_cleanups(
&self,
workspace_id: &str,
now: &str,
_now: &str,
limit: usize,
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
tx.execute(
"UPDATE backend_job_attempts SET worker_cleanup_state = 'pending', worker_cleanup_failure_category = 'cleanup_interrupted', worker_cleanup_failure_detail = 'Backend restart interrupted Worker cleanup; retrying canonical removal', worker_cleanup_updated_at = ?2 WHERE workspace_id = ?1 AND worker_cleanup_state = 'executing'",
params![workspace_id, now],
)?;
let mut statement = tx.prepare(
"SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('completed', 'failed', 'unknown') AND runtime_id IS NOT NULL AND worker_cleanup_state IN ('pending', 'failed') ORDER BY worker_cleanup_updated_at, created_at LIMIT ?2",
)?;
let identities = statement
.query_map(params![workspace_id, limit as i64], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
drop(statement);
let attempts = identities
.into_iter()
.map(|(job_id, attempt_id)| {
read_backend_job_attempt(&tx, workspace_id, &job_id, &attempt_id)?.ok_or_else(
|| Error::Store("recoverable Backend Job Worker cleanup disappeared".to_string()),
)
})
.collect::<Result<Vec<_>>>()?;
tx.commit()?;
Ok(attempts)
})
// Reopening observes retained owners; it does not revoke an executing
// cleanup or authorize a new one by resetting its state.
self.list_backend_job_worker_cleanups(workspace_id, limit)
}
fn list_backend_job_worker_cleanups(
@@ -8528,7 +8749,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn(|conn| {
let mut statement = conn.prepare(
"SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('completed', 'failed', 'unknown') AND runtime_id IS NOT NULL AND worker_cleanup_state IN ('pending', 'failed') ORDER BY worker_cleanup_updated_at, created_at LIMIT ?2",
"SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j USING(workspace_id,job_id) WHERE a.workspace_id = ?1 AND a.state IN ('completed', 'failed') AND a.runtime_id IS NOT NULL AND (a.worker_cleanup_state IN ('pending', 'failed', 'executing') OR (a.state='completed' AND a.worker_cleanup_state='completed' AND a.attempt=j.current_attempt AND j.input_disposition='available' AND json_type(j.request_json,'$.grants.subjektiv_journal')='object')) AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=a.workspace_id AND h.job_id=a.job_id) ORDER BY a.worker_cleanup_updated_at, a.created_at LIMIT ?2",
)?;
let identities = statement
.query_map(params![workspace_id, limit as i64], |row| {
@@ -8555,6 +8776,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<(BackendJobAttemptRecord, bool)> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
if !attempt.state.terminal() || attempt.worker.is_none() {
@@ -8607,6 +8829,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let failure_detail = failure_detail.map(bounded_failure_detail);
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
backend_job_recovery_store::require_current_contract(&tx, workspace_id, job_id)?;
let current = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
if current.worker_cleanup_state == Some(state) {
@@ -8620,7 +8843,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
}
let completed_at = (state == BackendJobWorkerCleanupState::Completed).then_some(now);
let changed = tx.execute(
"UPDATE backend_job_attempts SET worker_cleanup_state = ?4, worker_cleanup_failure_category = ?5, worker_cleanup_failure_detail = ?6, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = ?8 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND worker_cleanup_state = 'executing'",
"UPDATE backend_job_attempts SET worker_cleanup_state = ?4, worker_cleanup_failure_category = ?5, worker_cleanup_failure_detail = ?6, worker_cleanup_updated_at = ?7, worker_cleanup_completed_at = ?8 WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3 AND worker_cleanup_state = 'executing' AND NOT EXISTS(SELECT 1 FROM backend_job_upgrade_history h WHERE h.workspace_id=backend_job_attempts.workspace_id AND h.job_id=backend_job_attempts.job_id)",
params![workspace_id, job_id, attempt_id, state.as_str(), failure_category, failure_detail, now, completed_at],
)?;
if changed != 1 {
@@ -12809,7 +13032,7 @@ fn read_active_backend_job_for_resource(
// effects block even after Worker removal; cleanup is not outcome resolution.
let id = conn.query_row(
"SELECT job.job_id FROM backend_jobs job WHERE job.workspace_id = ?1 AND job.resource_key = ?2
AND (job.state IN ('pending', 'unknown') OR EXISTS (
AND (job.state='pending' OR EXISTS (
SELECT 1 FROM backend_job_attempts attempt WHERE attempt.workspace_id = job.workspace_id
AND attempt.job_id = job.job_id
AND attempt.worker_cleanup_state IS NOT 'completed'))
@@ -12921,7 +13144,7 @@ fn read_backend_job_attempt(
attempt_id: &str,
) -> Result<Option<BackendJobAttemptRecord>> {
let row = conn.query_row(
"SELECT attempt, input_digest, state, runtime_id, worker_id, runtime_run_id, dispatched_at, deadline_at, result_json, result_digest, failure_category, failure_detail, worker_cleanup_state, worker_cleanup_failure_category, worker_cleanup_failure_detail, worker_cleanup_updated_at, worker_cleanup_completed_at, created_at, updated_at, completed_at FROM backend_job_attempts WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3",
"SELECT attempt, input_digest, state, runtime_id, worker_id, runtime_run_id, dispatched_at, deadline_at, result_json, result_digest, failure_category, failure_detail, worker_cleanup_state, worker_cleanup_failure_category, worker_cleanup_failure_detail, worker_cleanup_updated_at, worker_cleanup_completed_at, created_at, updated_at, completed_at, allocation_precluded_at FROM backend_job_attempts WHERE workspace_id = ?1 AND job_id = ?2 AND attempt_id = ?3",
params![workspace_id, job_id, attempt_id],
|row| Ok((
row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?,
@@ -12930,7 +13153,7 @@ fn read_backend_job_attempt(
row.get::<_, Option<String>>(9)?, row.get::<_, Option<String>>(10)?, row.get::<_, Option<String>>(11)?,
row.get::<_, Option<String>>(12)?, row.get::<_, Option<String>>(13)?, row.get::<_, Option<String>>(14)?,
row.get::<_, Option<String>>(15)?, row.get::<_, Option<String>>(16)?, row.get::<_, String>(17)?,
row.get::<_, String>(18)?, row.get::<_, Option<String>>(19)?,
row.get::<_, String>(18)?, row.get::<_, Option<String>>(19)?, row.get::<_, Option<String>>(20)?,
)),
).optional()?;
let Some((
@@ -12954,6 +13177,7 @@ fn read_backend_job_attempt(
created_at,
updated_at,
completed_at,
allocation_precluded_at,
)) = row
else {
return Ok(None);
@@ -12987,6 +13211,7 @@ fn read_backend_job_attempt(
worker,
runtime_run_id,
dispatched_at,
allocation_precluded_at,
deadline_at,
result,
result_digest,
@@ -16074,6 +16299,36 @@ fn migrate_plugin_services_v111_to_v112(conn: &Connection) -> Result<()> {
Ok(())
}
fn migrate_backend_job_execution_to_v123(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 122 {
return Err(Error::Store(
"Backend Job execution migration requires schema 122".into(),
));
}
storage_formats::verify_history(conn, 122)?;
let foreign_keys: bool = conn.query_row("PRAGMA foreign_keys", [], |row| row.get(0))?;
conn.execute_batch("PRAGMA foreign_keys=OFF;")?;
let migration = (|| -> Result<()> {
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
backend_job_upgrade::install(&tx)?;
tx.execute(
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(123,?1)",
[backend_job_upgrade::MIGRATION_NAME],
)?;
storage_formats::verify_history(&tx, 123)?;
tx.commit()?;
Ok(())
})();
let restore = conn
.execute_batch(if foreign_keys {
"PRAGMA foreign_keys=ON;"
} else {
"PRAGMA foreign_keys=OFF;"
})
.map_err(Error::from);
migration.and(restore)
}
fn migrate_configuration_adoption_to_v122(conn: &Connection) -> Result<()> {
let current = current_schema_version(conn)?;
if !(113..=121).contains(&current) {
@@ -16192,6 +16447,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
create_workspace_schema_through_v113(conn)?;
conn.execute_batch(content_history_store::SCHEMA)?;
conn.execute_batch(crate::config_adoption::SCHEMA)?;
backend_job_upgrade::install_fresh(conn)?;
Ok(())
}
@@ -16229,7 +16485,11 @@ fn migrate_plugin_plans_v106_to_v107(conn: &Connection) -> Result<()> {
// Fresh composition uses the posts contract. Retained fixtures pass frozen host
// and component DDL to the shared cross-domain constraint composition below.
fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
create_workspace_schema_structure_from_sql(conn, include_str!("latest_schema.sql"), true)?;
create_workspace_schema_structure_from_sql(
conn,
&backend_job_upgrade::retained_checks(include_str!("latest_schema.sql")),
true,
)?;
conn.execute_batch(include_str!("activation_schema.sql"))?;
conn.execute_batch(include_str!("passkey_results_schema.sql"))?;
conn.execute_batch(crate::plugin_plan::SCHEMA_SQL)?;
@@ -27498,20 +27758,32 @@ INSERT INTO worker_registry (
}
#[tokio::test]
async fn backend_job_input_unknown_retention_and_legacy_invalidation_do_not_reinterpret_outcome()
async fn backend_job_input_pending_retention_and_legacy_invalidation_do_not_reinterpret_outcome()
{
use crate::backend_job::{JobInputDisposition, JobInputRetirementReason};
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
seed_resource_job_workspace(&store).await;
for (id, state) in [
("unfinished", None),
("unknown-input", Some(BackendJobAttemptState::Unknown)),
("reconciling-input", None),
("failed-input", Some(BackendJobAttemptState::Failed)),
] {
let original = resource_job_request(id, id, "candidate");
let reservation = store
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap();
if id == "reconciling-input" {
store
.record_backend_job_reconciliation(
"resource-workspace",
&reservation.attempt,
"receipt_missing",
"acceptance lookup temporarily unavailable",
"reconcile_saved_acceptance",
RESOURCE_JOB_NOW,
)
.unwrap();
}
if let Some(state) = state {
store
.finish_backend_job_attempt(
@@ -27685,12 +27957,24 @@ INSERT INTO worker_registry (
migrate_job_input_retention_v102_to_v103(&conn).unwrap();
drop(conn);
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let migrated = reopened
.get_backend_job("resource-workspace", "upgrade")
.unwrap()
.unwrap();
assert_eq!(migrated.state, BackendJobState::Failed);
assert_eq!(
migrated.failure_category.as_deref(),
Some("version_upgrade_execution_aborted")
);
let mut preserved = migrated.clone();
preserved.state = before.job.state;
preserved.failure_category = before.job.failure_category.clone();
preserved.failure_detail = before.job.failure_detail.clone();
assert_eq!(preserved, before.job);
assert!(
reopened
.get_backend_job("resource-workspace", "upgrade")
.backend_job_was_migrated("resource-workspace", "upgrade")
.unwrap()
.unwrap(),
before.job
);
let erased = reopened
.reclaim_backend_job_input(
@@ -27701,7 +27985,8 @@ INSERT INTO worker_registry (
)
.unwrap();
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
assert_eq!(erased.state, BackendJobState::Pending);
assert_eq!(erased.state, BackendJobState::Failed);
assert_eq!(erased.failure_category, migrated.failure_category);
assert_eq!(
reopened
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
@@ -27926,18 +28211,50 @@ INSERT INTO worker_registry (
assert_eq!(second.job.request, request);
assert_eq!(second.attempt.attempt, 2);
bind_resource_job_worker(&store, &second, "unknown-worker");
store
.finish_backend_job_attempt(
let expected = store
.get_backend_job_attempt(
"resource-workspace",
&request.job_id,
&second.attempt.attempt_id,
BackendJobAttemptState::Unknown,
"unknown_effects",
"domain outcome unknown",
)
.unwrap()
.unwrap();
let pending = store
.record_backend_job_reconciliation(
"resource-workspace",
&expected,
"receipt_unavailable",
"domain completion must be reconciled",
"read_saved_domain_completion",
RESOURCE_JOB_NOW,
)
.unwrap();
let history = store
.list_backend_job_attempts("resource-workspace", &request.job_id)
.unwrap();
let (executing, claimed) = store
.claim_backend_job_worker_cleanup_owned(
"resource-workspace",
&history,
&pending,
RESOURCE_JOB_NOW,
)
.unwrap();
assert!(claimed);
let history = store
.list_backend_job_attempts("resource-workspace", &request.job_id)
.unwrap();
store
.finish_backend_job_worker_cleanup_owned(
"resource-workspace",
&history,
&executing,
BackendJobWorkerCleanupState::Completed,
None,
None,
RESOURCE_JOB_NOW,
)
.unwrap();
finish_resource_job_cleanup(&store, &second);
drop(store);
let restarted = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let active = restarted
@@ -27947,7 +28264,7 @@ INSERT INTO worker_registry (
)
.unwrap()
.unwrap();
assert_eq!(active.state, BackendJobState::Unknown);
assert_eq!(active.state, BackendJobState::Pending);
assert_eq!(active.request, request);
assert!(
restarted
@@ -28946,13 +29263,12 @@ INSERT INTO worker_registry (
.unwrap();
assert_eq!(second.attempt.attempt, 2);
store
.finish_backend_job_attempt(
.record_backend_job_reconciliation(
workspace_id,
&retry_request.job_id,
&second.attempt.attempt_id,
BackendJobAttemptState::Unknown,
"worker_unknown",
"worker unavailable",
&second.attempt,
"worker_unavailable",
"worker acceptance must be reconciled",
"reconcile_saved_acceptance",
"2026-09-01T00:01:05Z",
)
.unwrap();
crates/workspace-server/src/subject_deletion_store.rs
+9 -6
View File
@@ -872,13 +872,16 @@ mod tests {
&crate::backend_job::allocation_key(&second.attempt.attempt_id),
None,
);
s.finish_backend_job_attempt(
let current = s
.get_backend_job_attempt("w", "job", &second.attempt.attempt_id)
.unwrap()
.unwrap();
s.record_backend_job_reconciliation(
"w",
"job",
&second.attempt.attempt_id,
BackendJobAttemptState::Unknown,
"unknown",
"unknown outcome",
&current,
"submission_receipt_unconfirmed",
"fixture dispatch reply lost",
"Read the exact original allocation receipt",
NOW,
)
.unwrap();
crates/workspace-server/src/ticket_item_checker.rs
+1
View File
@@ -442,6 +442,7 @@ mod tests {
state: BackendJobAttemptState::Completed,
worker: Some(RuntimeWorkerRef::new("runtime", "checker")),
runtime_run_id: Some("run".into()),
allocation_precluded_at: None,
dispatched_at: Some("1".into()),
deadline_at: "3".into(),
result: Some(result.clone()),
tools/state-contract-name-exceptions.json
+93
View File
@@ -1,4 +1,97 @@
[
{
"path": "crates/standalone/src/job_store/recovery.rs",
"kind": "saved_data",
"reason": "旧standalone Job schema 1/2の実migration fixtureに必要な保存列・JSON key。旧入力tokenの保持と新digestへの既存移行を検証する境界だけに残し、現行実行の状態・CASへ流用しない。",
"lines": [
{
"sha256": "969de6ed2a496597ba3d80cc06e76d273e73a641e438eee10cb8b708696f29ad",
"count": 1
},
{
"sha256": "c69cf30db18c35ff26a854bf47ecd4f67bc70513ab41a7b74b78e232e2e34a68",
"count": 1
},
{
"sha256": "f88b9c3ac64944b6e957f9d85a54081925f70cbd2563d936f2a5f9d746093940",
"count": 2
}
]
},
{
"path": "crates/workspace-server/src/backend_job_upgrade.rs",
"kind": "saved_data",
"reason": "schema85以前のJob/全attemptの原保存列をschema86の本文変換前に同transactionで捕捉・検証する読み取り/移行境界。旧tokenを履歴として保ち、raw本文は既Job消去ownerと連動してretireする。現行Job入力digestの別名や新しい実行状態を作らない。",
"lines": [
{
"sha256": "b80515c50a007075bcde4268cd5362d4f03e8e1b768b3f5d7d9400b408252de1",
"count": 2
},
{
"sha256": "3476b390e729656e898805367f93d86368a1660d7fc59142f588767470eb6b4f",
"count": 1
},
{
"sha256": "1fe9f975d2ae4ee3186faefdbde6f29a93c43423a1993b7def62cd757c903472",
"count": 1
},
{
"sha256": "37c5715b8ac1eb001ebd75d90078751313b1c0bff0717446a4a17481efd7267a",
"count": 1
},
{
"sha256": "acf5ef6e7e4fef26678263c18fe4bf29fbac42046dc94cc0a727e94e7942da32",
"count": 1
},
{
"sha256": "f121db497084c3889dba8545ee27e0225db94cfde2d662ea922c723a1fea31ff",
"count": 1
},
{
"sha256": "656f37d2121e4c564555684926b6917d273d9ca97e9b3112c6733925397410f6",
"count": 1
},
{
"sha256": "3265977c091a93b4f70af0c9f93fa19f15b0022ec5c90400b29e885dd4b941e0",
"count": 1
},
{
"sha256": "f621d8c70aecdc042ed8dfa68bcf6f7d69b23eb3f2dcc32464dbba25175d5c40",
"count": 1
}
]
},
{
"path": "crates/workspace-server/src/backend_job_upgrade_tests.rs",
"kind": "saved_data",
"reason": "実旧schemaのJob入力列・request JSONと全attempt原値をfixtureに挿入し、正規runner前後の保持・rollback/retry/reopenを比較する保存互換試験。現行producer/実行型には旧keyを導入しない。",
"lines": [
{
"sha256": "430e40af773fc80bc14662b74fa4cd960aaf30b78b9b1b02be32004ada56d886",
"count": 1
},
{
"sha256": "030aed0d240185b411ef3a190dbd718e00df8e91f0ac799ff39fa524a6f704de",
"count": 2
},
{
"sha256": "d03c3392bb2443aa9384c3340960c07aacc605af847dc3525ea934d928bc9844",
"count": 1
},
{
"sha256": "2cf9adfc96b54eecc5545a04e5d651c8a03f9d78eb1f8bf9dc981a5d91a76e2e",
"count": 1
},
{
"sha256": "7253a381f932f6fce62caae83f6c1cc25553dbb662a795638d83f19101790f00",
"count": 1
},
{
"sha256": "c72df513101625c8bff4c62b7826fc9f4ef02cb207a6d302fe1ccecb09583ec9",
"count": 1
}
]
},
{
"path": "crates/subjektiv/src/storage_format_tests.rs",
"kind": "saved_data",