feat(journal): execute frozen body inputs with durable delivery and erasure fences

This commit is contained in:
2026-10-27 08:03:43 +09:00
parent 6776973e4c
commit 897c025b85
46 changed files with 10368 additions and 2180 deletions
@@ -476,7 +476,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
(99, "Explicit hosting administrators and self-registration policy".into()),
(100, "Provisional users and scoped passkey activation".into()),
(101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into())
(102, "Durable failed Subject creation recovery".into()),
(103, "Job input retirement independent of immutable receipts".into())
]
);
for (table, rows) in retained.iter().zip(original.iter()) {
@@ -463,10 +463,52 @@ impl BackendJobRequest {
}
}
/// Availability of the immutable input body, independent of execution outcome.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum JobInputDisposition {
Available,
Reclaimed,
Erased,
Invalidated,
}
impl JobInputDisposition {
pub fn as_str(self) -> &'static str {
match self {
Self::Available => "available",
Self::Reclaimed => "reclaimed",
Self::Erased => "erased",
Self::Invalidated => "invalidated",
}
}
}
/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum JobInputRetirementReason {
Completed,
ExplicitErasure,
LegacyInvalidation,
}
impl JobInputRetirementReason {
pub fn disposition(self) -> JobInputDisposition {
match self {
Self::Completed => JobInputDisposition::Reclaimed,
Self::ExplicitErasure => JobInputDisposition::Erased,
Self::LegacyInvalidation => JobInputDisposition::Invalidated,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct BackendJobRecord {
pub workspace_id: String,
/// Receipt metadata. After retirement `input` is null; use the verified loader to execute.
pub request: BackendJobRequest,
pub input_digest: String,
pub input_disposition: JobInputDisposition,
pub intent_fingerprint: String,
pub state: BackendJobState,
pub current_attempt: u8,
@@ -479,6 +521,28 @@ pub struct BackendJobRecord {
pub completed_at: Option<String>,
}
impl BackendJobRecord {
/// Verify the original immutable body and envelope before any execution or retry.
/// No Session read or replay request may repair unavailable/corrupt input.
pub fn load_executable_request(&self) -> Result<&BackendJobRequest> {
if self.input_disposition != JobInputDisposition::Available {
return Err(Error::InvalidInput(format!(
"Backend Job input is {}",
self.input_disposition.as_str()
)));
}
self.request.validate()?;
if self.request.input_digest()? != self.input_digest
|| self.request.fingerprint()? != self.intent_fingerprint
{
return Err(Error::Store(
"Backend Job immutable input or intent fingerprint mismatch".into(),
));
}
Ok(&self.request)
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum BackendJobWorkerCleanupState {
@@ -665,6 +729,8 @@ mod tests {
"workspace_id": "workspace-a",
"request": request(),
"intent_fingerprint": request().fingerprint().unwrap(),
"input_digest": request().input_digest().unwrap(),
"input_disposition": "available",
"state": "pending",
"current_attempt": 2,
"created_at": "created",
@@ -1124,3 +1190,17 @@ mod tests {
}
}
}
pub(crate) const INPUT_RETENTION_TRIGGERS: &str = r#"
CREATE TRIGGER backend_job_identity_immutable BEFORE UPDATE OF
workspace_id, job_id, purpose, input_digest, input_ref, resource_key, intent_fingerprint ON backend_jobs
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job identity'); END;
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON backend_jobs
WHEN (
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
AND json_type(NEW.request_json,'$.input') = 'null'
) IS NOT TRUE
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job input retirement'); END;
"#;
@@ -71,6 +71,11 @@ fn legacy_content_database(path: &Path) -> (Connection, String) {
conn.execute("INSERT INTO workspace_config_tree_revisions VALUES('space',?1,?2,'toolchain','projection',?3,?4,'{}')",
params![ordinal,snapshot.digest,manifest,created]).unwrap();
}
seed_legacy_content_job(&conn);
(conn, snapshot.digest)
}
fn seed_legacy_content_job(conn: &Connection) {
let request = BackendJobRequest {
job_id: "legacy-job".into(),
purpose: "migration_test".into(),
@@ -93,7 +98,19 @@ fn legacy_content_database(path: &Path) -> (Connection, String) {
VALUES('space','legacy-job','legacy-job:attempt:1',1,'old-number','dispatched','runtime','worker','2026-08-14T00:00:00Z','2026-08-13T00:00:00Z','2026-08-13T00:00:00Z');
INSERT INTO backend_job_deliveries(workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,created_at,updated_at)
VALUES('space','delivery','legacy-job','legacy-job:attempt:1','runtime','target','pending','created','updated');") .unwrap();
(conn, snapshot.digest)
}
// Replay/resource checks use the current Store only after the full retained chain.
// Keep the focused partial DDL fixture above for the cutover's schema assertions.
fn legacy_content_job_database(path: &Path) -> Connection {
super::tests::prepare_retained_schema(path, 85);
let conn = Connection::open(path).unwrap();
configure_sqlite(&conn).unwrap();
conn.execute_batch("INSERT INTO accounts VALUES('owner','user','owner','Owner','created','updated');
INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at)
VALUES('space','owner','Space','active','created','updated');").unwrap();
seed_legacy_content_job(&conn);
conn
}
#[test]
@@ -432,7 +449,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
] {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("content.db");
let (conn, _) = legacy_content_database(&path);
let conn = legacy_content_job_database(&path);
conn.execute(
"UPDATE backend_job_attempts SET state=?1,
runtime_id=CASE WHEN ?1='reserved' THEN NULL ELSE runtime_id END,
@@ -441,11 +458,22 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
)
.unwrap();
migrate_content_state_v85_to_v86(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 86);
let cutover: (String, String, Option<String>) = conn.query_row(
"SELECT j.state,a.state,a.worker_cleanup_state FROM backend_jobs j JOIN backend_job_attempts a USING(workspace_id,job_id)",
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
).unwrap();
assert_eq!(
cutover,
if state == "reserved" {
("failed".into(), "failed".into(), Some("completed".into()))
} else {
("unknown".into(), "unknown".into(), Some("pending".into()))
}
);
apply_migrations(&conn).unwrap();
drop(conn);
let store = SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
};
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let job = store
.get_backend_job("space", "legacy-job")
.unwrap()
@@ -478,7 +506,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("content.db");
let (conn, _) = legacy_content_database(&path);
let conn = legacy_content_job_database(&path);
let result = serde_json::json!({"finding":"preserved"});
let (digest, encoded) = job::result_digest(&result, 4096).unwrap();
conn.execute("UPDATE backend_jobs SET state='completed',result_json=?1,result_digest=?2,completed_at='finished'",params![encoded,digest]).unwrap();
@@ -490,11 +518,23 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() {
)
.unwrap();
migrate_content_state_v85_to_v86(&conn).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 86);
let cutover: (String, String, String, String) = conn.query_row(
"SELECT j.state,j.result_json,a.worker_cleanup_state,a.worker_cleanup_completed_at FROM backend_jobs j JOIN backend_job_attempts a USING(workspace_id,job_id)",
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
).unwrap();
assert_eq!(
cutover,
(
"completed".into(),
encoded,
"completed".into(),
"cleaned".into()
)
);
apply_migrations(&conn).unwrap();
drop(conn);
let store = SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor::shared(),
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
};
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let job = store
.get_backend_job("space", "legacy-job")
.unwrap()
+47
View File
@@ -1326,6 +1326,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
Err("Session source permanent erasure is unavailable on this Runtime".into())
}
fn session_recording_capture(
&self,
_request: runtime_api::SessionRecordingCaptureRequest,
) -> Result<runtime_api::SessionRecordingCaptureAvailability, String> {
Err("committed recording capture is unavailable on this Runtime".into())
}
fn session_material_catalog(
&self,
_request: runtime_api::SessionMaterialCatalogRequest,
@@ -2336,6 +2343,21 @@ impl RuntimeRegistry {
})
}
pub fn session_recording_capture(
&self,
runtime_id: &str,
request: runtime_api::SessionRecordingCaptureRequest,
) -> Result<runtime_api::SessionRecordingCaptureAvailability, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.session_recording_capture(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "session_recording_capture_failed".into(),
message,
})
}
pub fn session_material_catalog(
&self,
runtime_id: &str,
@@ -3441,6 +3463,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
})
}
fn session_recording_capture(
&self,
request: runtime_api::SessionRecordingCaptureRequest,
) -> Result<runtime_api::SessionRecordingCaptureAvailability, String> {
if request.workspace_id != self.workspace_id {
return Err("recording capture Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
Ok(self
.runtime
.session_recording_capture_scoped(&scope, &request))
}
fn session_material_catalog(
&self,
request: runtime_api::SessionMaterialCatalogRequest,
@@ -5677,6 +5712,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
.map_err(|diagnostic| diagnostic.message)
}
fn session_recording_capture(
&self,
request: runtime_api::SessionRecordingCaptureRequest,
) -> Result<runtime_api::SessionRecordingCaptureAvailability, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.session_recording_capture(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn session_material_catalog(
&self,
request: runtime_api::SessionMaterialCatalogRequest,
@@ -381,6 +381,10 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
DROP TABLE ticket_implementation_results;
DROP TABLE ticket_implementation_delegations;
DROP TABLE merge_request_source_observations;
DROP TRIGGER backend_job_input_retirement;
DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
@@ -517,6 +521,10 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
// Reconstruct the prior composition before its new business capture existed.
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TRIGGER backend_job_input_retirement;
DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
@@ -679,7 +687,11 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
if historical {
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TABLE subject_create_recoveries;
DROP TRIGGER backend_job_input_retirement;
DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
@@ -2,8 +2,8 @@
//! lifecycle, are the durable source. Fixed Worker recipients never follow leases.
use super::*;
use crate::backend_job::{
BackendJobAttemptState, BackendJobDeliveryRecord, BackendJobDeliveryState,
BackendJobResultAcceptance, BackendJobState,
BackendJobAttemptState, BackendJobDeliveryRecord, BackendJobDeliveryState, BackendJobRequest,
BackendJobState,
};
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -103,6 +103,32 @@ pub(in crate::store) fn install(conn: &Connection) -> Result<()> {
Ok(())
}
struct NotificationResultSnapshot {
request: BackendJobRequest,
result: Option<serde_json::Value>,
result_digest: Option<String>,
}
// The schema-96 import runs before later Job lifecycle migrations. Read only
// the retained result envelope, not the executable Job's current-schema columns.
fn read_result_snapshot(
conn: &Connection,
workspace: &str,
job: &str,
) -> Result<NotificationResultSnapshot> {
let (request, result, result_digest): (String, Option<String>, Option<String>) = conn
.query_row(
"SELECT request_json, result_json, result_digest FROM backend_jobs WHERE workspace_id=?1 AND job_id=?2",
params![workspace, job],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
)?;
Ok(NotificationResultSnapshot {
request: decode(&request)?,
result: result.map(|value| decode(&value)).transpose()?,
result_digest,
})
}
pub(in crate::store) fn capture(
conn: &Connection,
workspace: &str,
@@ -115,8 +141,7 @@ pub(in crate::store) fn capture(
attempt_id: attempt.into(),
delivery_id: id.clone(),
};
let job = read_backend_job(conn, workspace, job)?
.ok_or_else(|| Error::Store("Job notification source missing".into()))?;
let job = read_result_snapshot(conn, workspace, job)?;
let target = &job.request.notification_target;
let facts = serde_json::json!({"principal_kind":if target.is_some(){Some("worker")}else{None}, "principal_id":null,"runtime_id":target.as_ref().map(|w|&w.runtime_id),"worker_id":target.as_ref().map(|w|&w.worker_id),"status":null,"case_state":"open","released":false,"backend_job":source,"recipient_kind":"fixed_worker","job_purpose":job.request.purpose,"result_digest":job.result_digest});
conn.execute("INSERT OR IGNORE INTO notification_sources(workspace_id,ticket_id,input_event_id,role,assignment_id,kind,facts_json) VALUES(?1,NULL,?2,'backend_job','','backend_job_result',?3)",params![workspace,id,json(&facts)?])?;
@@ -290,22 +315,13 @@ fn import_legacy(conn: &Connection, workspace: &str, job_id: &str, attempt_id: &
| "notification_too_large",
) => ("defer", NotificationReason::SourceUnavailable),
_ => {
let job = read_backend_job(conn, workspace, job_id)?
.ok_or_else(|| Error::Store("Legacy Job result missing".into()))?;
let attempt = read_backend_job_attempt(conn, workspace, job_id, attempt_id)?
.ok_or_else(|| Error::Store("Legacy Job attempt missing".into()))?;
let acceptance = BackendJobResultAcceptance {
job,
attempt,
replayed: true,
};
let facts = if acceptance.job.request.notification_target.as_ref() != Some(&old.target)
{
let snapshot = read_result_snapshot(conn, workspace, job_id)?;
let facts = if snapshot.request.notification_target.as_ref() != Some(&old.target) {
JobNotificationFacts::Failed(NotificationFailureCode::PreparationFailed)
} else {
legacy_result_facts(&acceptance)
legacy_result_facts(&snapshot, job_id, attempt_id)
};
let decision = policy(acceptance.job.request.notification_target.is_some(), &facts);
let decision = policy(snapshot.request.notification_target.is_some(), &facts);
match facts {
JobNotificationFacts::Result(body) => content = Some(body),
JobNotificationFacts::Failed(code) => failure = Some(code),
@@ -338,28 +354,26 @@ fn import_legacy(conn: &Connection, workspace: &str, job_id: &str, attempt_id: &
Ok(())
}
fn legacy_result_facts(acceptance: &BackendJobResultAcceptance) -> JobNotificationFacts {
let content = if acceptance.job.request.purpose == crate::ticket_item_checker::PURPOSE {
let Some(result) = &acceptance.job.result else {
fn legacy_result_facts(
snapshot: &NotificationResultSnapshot,
job_id: &str,
attempt_id: &str,
) -> JobNotificationFacts {
let content = if snapshot.request.purpose == crate::ticket_item_checker::PURPOSE {
let Some(result) = &snapshot.result else {
return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult);
};
match crate::ticket_item_checker::validate_result(&acceptance.job.request.input, result) {
match crate::ticket_item_checker::validate_result(&snapshot.request.input, result) {
Ok(result) if result.findings.is_empty() => return JobNotificationFacts::NoFindings,
Ok(_) => {}
Err(_) => return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult),
}
crate::ticket_item_checker::render_result(acceptance)
crate::ticket_item_checker::render_result(&snapshot.request.input, result)
} else {
Ok(format!(
"Backend Job `{}` completed attempt `{}` for `{}`. Durable result digest: {}.",
acceptance.job.request.job_id,
acceptance.attempt.attempt_id,
acceptance.job.request.purpose,
acceptance
.job
.result_digest
.as_deref()
.unwrap_or("unavailable")
"Backend Job `{job_id}` completed attempt `{attempt_id}` for `{}`. Durable result digest: {}.",
snapshot.request.purpose,
snapshot.result_digest.as_deref().unwrap_or("unavailable")
))
};
match content {
@@ -789,6 +789,7 @@ CREATE TABLE "worker_registry" (
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
);
CREATE TABLE backend_jobs (
input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated')),
workspace_id TEXT NOT NULL,
job_id TEXT NOT NULL,
purpose TEXT NOT NULL,
@@ -1642,6 +1643,19 @@ CREATE TABLE IF NOT EXISTS ticket_assignment_work_releases (
FOREIGN KEY (workspace_id, assignment_id)
REFERENCES ticket_worker_assignments(workspace_id, assignment_id) ON DELETE CASCADE
);
CREATE TRIGGER backend_job_identity_immutable BEFORE UPDATE OF
workspace_id, job_id, purpose, input_digest, input_ref, resource_key, intent_fingerprint ON backend_jobs
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job identity'); END;
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON backend_jobs
WHEN (
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
AND json_type(NEW.request_json,'$.input') = 'null'
) IS NOT TRUE
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job input retirement'); END;
-- Ticket case assignment work constraints (schema 91).
DROP VIEW IF EXISTS ticket_active_worker_assignments;
CREATE VIEW ticket_active_worker_assignments AS
+80 -1
View File
@@ -4635,11 +4635,79 @@ impl WorkspaceApi {
&self,
reservation: BackendJobReservation,
) -> Result<BackendJobReservation> {
// Reread even a terminal/stale reservation before deciding whether it can
// dispatch or replay: an explicit erasure may have retired its saved input.
let journal_gate = reservation
.job
.request
.grants
.subjektiv_journal
.as_ref()
.map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id))
.transpose()?;
let _journal_guard = journal_gate
.as_ref()
.map(|gate| {
gate.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
})
.transpose()?;
let current_job = self
.store
.get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)?
.ok_or_else(|| Error::Store("reserved Backend Job disappeared".into()))?;
let current_attempt = self
.store
.get_backend_job_attempt(
&self.config.workspace_id,
&reservation.job.request.job_id,
&reservation.attempt.attempt_id,
)?
.ok_or_else(|| Error::Store("reserved Backend attempt disappeared".into()))?;
let reservation = BackendJobReservation {
job: current_job,
attempt: current_attempt,
replayed: reservation.replayed,
resource_reused: reservation.resource_reused,
};
if matches!(
reservation.job.input_disposition,
crate::backend_job::JobInputDisposition::Erased
| crate::backend_job::JobInputDisposition::Invalidated
) {
return Err(Error::WorkspacePermissionDenied(
"erased or invalidated Backend Job input cannot dispatch".into(),
));
}
if reservation.job.state != BackendJobState::Pending
|| reservation.attempt.state != BackendJobAttemptState::Reserved
|| reservation.attempt.worker.is_some()
{
return Ok(reservation);
return Ok(BackendJobReservation {
replayed: true,
..reservation
});
}
reservation.job.load_executable_request()?;
if subjektiv_journal::migrate_reserved_legacy(self, &reservation)? {
let job = self
.store
.get_backend_job(self.workspace_id(), &reservation.job.request.job_id)?
.ok_or_else(|| Error::Store("retired legacy Job missing".into()))?;
let attempt = self
.store
.get_backend_job_attempt(
self.workspace_id(),
&reservation.job.request.job_id,
&reservation.attempt.attempt_id,
)?
.ok_or_else(|| Error::Store("retired legacy attempt missing".into()))?;
return Ok(BackendJobReservation {
job,
attempt,
replayed: reservation.replayed,
resource_reused: reservation.resource_reused,
});
}
subjektiv_journal::prepare(self, &reservation)?;
let (reservation, claimed) = self.store.claim_backend_job_attempt_dispatch(
@@ -4844,6 +4912,7 @@ impl WorkspaceApi {
.to_string(),
});
}
subjektiv_journal::attest_delivery(self, &reservation)?;
let job = self
.store
.get_backend_job(&self.config.workspace_id, &request.job_id)?
@@ -5220,6 +5289,16 @@ impl WorkspaceApi {
subjektiv_journal::close(self, &job.request, &current)?;
if current.state == BackendJobAttemptState::Completed {
subjektiv_journal::drain_after_completion(self, &job.request);
if job.input_disposition
== crate::backend_job::JobInputDisposition::Available
{
self.store.reclaim_backend_job_input(
self.workspace_id(),
job_id,
&job.input_digest,
crate::backend_job::JobInputRetirementReason::Completed,
)?;
}
}
}
}
@@ -4,7 +4,7 @@ use super::*;
use crate::backend_job::{
BackendJobGrants, BackendJobRecord, SubjektivJournalGrant, journal_serialization_key,
};
use subjektiv::journal::recording::{JournalMaterial, select_materials_bounded};
use subjektiv::journal::recording::JournalInput;
use subjektiv::journal::transport::*;
use subjektiv::journal::{self, *};
@@ -92,20 +92,82 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) -
};
let store = store(api)?;
let host = host(api, &store, grant)?;
store
.prepare_journal_generation(
let input = saved_input(&reservation.job.request, grant)?;
let g = store
.prepare_journal_input(
&host,
&grant.journal_id,
&input,
subjektiv::api::JobAttemptBinding {
job_id: reservation.job.request.job_id.clone(),
attempt_id: reservation.attempt.attempt_id.clone(),
},
grant.selected_ranges.clone(),
)
.map_err(domain)?;
input.validate_generation(&g).map_err(domain)?;
Ok(())
}
fn legacy_descriptor(request: &BackendJobRequest) -> bool {
let Some(grant) = request.grants.subjektiv_journal.as_ref() else {
return false;
};
let Some(object) = request.input.as_object() else {
return false;
};
object.len() == 2
&& object.get("journal_id").and_then(|v| v.as_str()) == Some(&grant.journal_id)
&& object.get("selected_ranges")
== serde_json::to_value(&grant.selected_ranges).ok().as_ref()
}
/// Retire only the explicitly recognized descriptor wire. Malformed body snapshots are
/// failures, not migration requests and never a reason to reselect from current Sessions.
pub(super) fn migrate_reserved_legacy(
api: &WorkspaceApi,
reservation: &BackendJobReservation,
) -> Result<bool> {
if !legacy_descriptor(&reservation.job.request) {
return Ok(false);
}
reservation.job.load_executable_request()?;
close(api, &reservation.job.request, &reservation.attempt)?;
api.store.finish_backend_job_attempt(
api.workspace_id(),
&reservation.job.request.job_id,
&reservation.attempt.attempt_id,
BackendJobAttemptState::Failed,
"journal_descriptor_invalidated",
"Descriptor-only Journal input retired; saved checkpoints retained",
&Utc::now().to_rfc3339(),
)?;
api.store.reclaim_backend_job_input(
api.workspace_id(),
&reservation.job.request.job_id,
&reservation.job.input_digest,
crate::backend_job::JobInputRetirementReason::LegacyInvalidation,
)?;
let queued = api.clone();
let request = reservation.job.request.clone();
api.host_executors.host()?.dispatch(move || {
drain_after_completion(&queued, &request);
})?;
Ok(true)
}
fn saved_input(request: &BackendJobRequest, grant: &SubjektivJournalGrant) -> Result<JournalInput> {
let input = JournalInput::decode(&request.input).map_err(domain)?;
if input.subject_id != grant.subject_id
|| input.journal_id != grant.journal_id
|| input.capture.session_id != grant.session_id
|| input.selected_ranges != grant.selected_ranges
{
return Err(Error::WorkspacePermissionDenied(
"Journal saved input/capability mismatch".into(),
));
}
Ok(input)
}
fn live(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
@@ -125,6 +187,7 @@ fn live(
.store
.get_backend_job(api.workspace_id(), &binding.job_id)?
.ok_or_else(|| Error::WorkspacePermissionDenied("Journal Job missing".into()))?;
job.load_executable_request()?;
let attempt = api
.store
.get_backend_job_attempt(api.workspace_id(), &binding.job_id, &binding.attempt_id)?
@@ -180,15 +243,44 @@ pub(super) fn execute(
worker: &RuntimeWorkerRef,
request: JournalJobRequest,
) -> Result<JournalJobResponse> {
let (grant, _, _) = live(api, worker)?;
let gate = lifecycle_lock(api, &grant.subject_id)?;
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
// Dispatch holds this same gate until matching Runtime acceptance is attested.
// Revalidate after waiting: erasure, cancellation or replacement may have won.
let (grant, job, attempt) = live(api, worker)?;
let store = store(api)?;
let host = host(api, &store, &grant)?;
let g = generation(&store, &host, &grant, &job, &attempt)?;
let input = saved_input(&job.request, &grant)?;
input.validate_generation(&g).map_err(domain)?;
let ctx = JournalJobContext::validated(&store, &host, &g.id, g.job.clone()).map_err(domain)?;
journal::service::execute_bounded(&store, &ctx, request, |source, start, limit| {
read_source(api, &grant, source, start, limit)
.map_err(|e| JournalError::Conflict(e.to_string()))
})
journal::service::execute_snapshot(&store, &ctx, &attempt.input_digest, request).map_err(domain)
}
/// Called only after matching tracked Runtime acceptance of the exact rendered saved input.
/// The durable request and Worker binding alone do not establish body delivery.
pub(super) fn attest_delivery(
api: &WorkspaceApi,
reservation: &BackendJobReservation,
) -> Result<()> {
let Some(grant) = &reservation.job.request.grants.subjektiv_journal else {
return Ok(());
};
let input = saved_input(&reservation.job.request, grant)?;
let store = store(api)?;
let host = host(api, &store, grant)?;
let g = generation(&store, &host, grant, &reservation.job, &reservation.attempt)?;
input.validate_generation(&g).map_err(domain)?;
let ctx = JournalJobContext::validated(&store, &host, &g.id, g.job.clone()).map_err(domain)?;
journal::service::attest_input_delivery(
&store,
&ctx,
&reservation.attempt.input_digest,
&input.selected_ranges,
)
.map_err(domain)
}
@@ -196,21 +288,29 @@ pub(super) fn finish(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
) -> Result<JournalJobCompletion> {
let (grant, _, _) = live(api, worker)?;
let gate = lifecycle_lock(api, &grant.subject_id)?;
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
// Dispatch holds this same gate until matching Runtime acceptance is attested.
// Revalidate after waiting: erasure, cancellation or replacement may have won.
let (grant, job, attempt) = live(api, worker)?;
let store = store(api)?;
let host = host(api, &store, &grant)?;
let input = saved_input(&job.request, &grant)?;
// prepare is an exact binding lookup too, including the saved completion of a closed generation.
let g = store
.prepare_journal_generation(
.prepare_journal_input(
&host,
&grant.journal_id,
&input,
subjektiv::api::JobAttemptBinding {
job_id: job.request.job_id,
attempt_id: attempt.attempt_id,
},
grant.selected_ranges.clone(),
)
.map_err(domain)?;
input.validate_generation(&g).map_err(domain)?;
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
.map_err(domain)?;
journal::service::finish(&store, &ctx).map_err(domain)
@@ -277,21 +377,6 @@ pub(super) fn close(
Ok(())
}
fn read_source(
api: &WorkspaceApi,
grant: &SubjektivJournalGrant,
source: &JournalSource,
start: u64,
limit: usize,
) -> Result<JournalMaterial> {
let store = store(api)?;
let attr = store
.session_attribution(&grant.session_id)
.map_err(|e| Error::Store(e.to_string()))?
.ok_or_else(|| Error::WorkspacePermissionDenied("Session attribution absent".into()))?;
read_canonical(&material_reader(api, &attr)?, source, start, limit)
}
pub(super) fn bind_host_session(
api: &WorkspaceApi,
attr: crate::subjektiv::SubjectSessionAttribution,
@@ -524,22 +609,63 @@ fn dispatch(
.close_journal_generation(&host, &g.id, &g.job)
.map_err(domain)?;
}
let reader = material_reader(api, &attribution)?;
let materials = select_materials_bounded(
&store,
&host,
journal_id,
96 * 1024,
|source, start, limit| {
read_canonical(&reader, source, start, limit)
.map_err(|e| JournalError::Conflict(e.to_string()))
},
)
let mut after = None;
loop {
let jobs = api.store.list_backend_journal_jobs(
api.workspace_id(),
&attribution.session_id,
after.as_deref(),
100,
)?;
for job in &jobs {
if job
.request
.grants
.subjektiv_journal
.as_ref()
.is_none_or(|g| g.journal_id != journal_id)
|| job.input_disposition != crate::backend_job::JobInputDisposition::Available
{
continue;
}
if job.state == BackendJobState::Pending || job.state == BackendJobState::Unknown {
return Ok(());
}
if job.state == BackendJobState::Failed && !legacy_descriptor(&job.request) {
job.load_executable_request()?;
saved_input(
&job.request,
job.request
.grants
.subjektiv_journal
.as_ref()
.expect("checked Journal grant"),
)?;
let reservation = api.store.reserve_backend_job_retry(
api.workspace_id(),
&job.request.job_id,
&Utc::now().to_rfc3339(),
)?;
return queue_reservation(api, reservation);
}
}
if jobs.len() < 100 {
break;
}
after = jobs.last().map(|j| j.request.job_id.clone());
}
let input = JournalInput::build(&store, &host, journal_id, |order, offset, budget| {
capture_input(api, &attribution, order, offset, budget)
.map_err(|e| JournalError::Conflict(e.to_string()))
})
.map_err(domain)?;
if materials.is_empty() {
if input.selected_ranges.is_empty() {
return Ok(());
}
let request = recording_request(&host, attribution, journal_id, materials)?;
store
.accept_journal_sources(&host, "Session activity", &input.selected_ranges)
.map_err(domain)?;
let request = recording_request(&host, attribution, journal_id, input)?;
// The intent is saved before generation binding or dispatch. If binding/dispatch fails,
// Reserved recovery repeats this exact intent, never creates a replacement blindly.
api.resolve_backend_job_profile(&request)?;
@@ -548,6 +674,10 @@ fn dispatch(
&request,
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
)?;
queue_reservation(api, reservation)
}
fn queue_reservation(api: &WorkspaceApi, reservation: BackendJobReservation) -> Result<()> {
let queued = api.clone();
if let Err(error)=api.host_executors.host()?.dispatch(move || {
if let Err(error)=queued.dispatch_backend_job_reservation(reservation) {
@@ -578,33 +708,21 @@ fn admit_sources(
));
}
let mut id = store.journal_for_session(&host).map_err(domain)?;
let reader = material_reader(api, &attribution)?;
if reader.not_materialized {
if store.journal_retention_required(&host).map_err(domain)? {
return Err(Error::RepositoryConflict(
"Admitted Journal original Session is not materialized".into(),
));
}
retain_source(
api.runtime.as_ref(),
api.workspace_id(),
&attribution,
material_source(api, &attribution)?,
true,
)?;
return Ok(JournalRecordingResponse::NoNewSources);
}
let (order, offset) = if let Some(id) = id.as_deref() {
let journal = store.read_journal(&host, id).map_err(domain)?;
(journal.process_order, journal.process_byte)
} else {
(0, 0)
};
let capture = capture_input(api, &attribution, order, offset, 48 * 1024)?;
retain_source(
api.runtime.as_ref(),
api.workspace_id(),
&attribution,
material_source(api, &attribution)?,
false,
capture.segment_id == "not-materialized",
)?;
let sources: Vec<_> = reader
.descriptors()
.map(|d| source_from_descriptor(&attribution.session_id, d))
.collect();
let sources = journal::recording::capture_ranges(&capture).map_err(domain)?;
for batch in sources.chunks(MAX_JOURNAL_RANGES) {
id = Some(
store
@@ -995,7 +1113,7 @@ pub(super) fn release_subject(
{
// Legacy saved-before-admission Sessions are not optional. Prove canonical
// source identity first, then bind only this original Host-attributed Session.
let _canonical = material_reader(api, &attr)?;
let _canonical = capture_input(api, &attr, 0, 0, 48 * 1024)?;
let host =
JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
if host
@@ -1281,80 +1399,6 @@ fn material_source(
},
)
}
fn material_reader<'a>(
api: &'a WorkspaceApi,
attribution: &crate::subjektiv::SubjectSessionAttribution,
) -> Result<MaterialReader<'a>> {
MaterialReader::open(
api.runtime.as_ref(),
api.workspace_id(),
&attribution.runtime_id,
material_source(api, attribution)?,
&attribution.session_id,
)
}
fn source_from_descriptor(
session: &str,
d: &session_store::public_material::PublicMaterialDescriptor,
) -> JournalSourceRange {
JournalSourceRange {
source: JournalSource {
session_id: session.into(),
pointer: JournalSourcePointer {
segment_id: d.identity.segment_id,
entry_ref: d.identity.entry_ref.clone(),
},
order: d.canonical_order,
content_sha256: d.sha256.clone(),
total_bytes: d.total_bytes,
},
bytes: JournalByteRange {
start: 0,
end: d.total_bytes,
},
}
}
fn read_canonical(
reader: &MaterialReader<'_>,
source: &JournalSource,
start: u64,
limit: usize,
) -> Result<JournalMaterial> {
if reader.session_id != source.session_id {
return Err(Error::WorkspacePermissionDenied(
"Journal source Session mismatch".into(),
));
}
let descriptor = reader
.descriptors()
.find(|d| source_from_descriptor(&source.session_id, d).source == *source)
.ok_or_else(|| {
Error::Store("Journal original public source changed or unavailable".into())
})?;
// The reader streams the complete canonical digest while returning only this bounded window.
let page = reader
.read_content(descriptor, start, limit)
.map_err(|e| Error::Store(e.to_string()))?;
if page.sha256 != source.content_sha256
|| page.total_bytes != source.total_bytes
|| page.next_offset != start.saturating_add(page.content.len() as u64)
{
return Err(Error::RepositoryConflict(
"Journal canonical source identity changed".into(),
));
}
Ok(JournalMaterial {
range: JournalSourceRange {
source: source.clone(),
bytes: JournalByteRange {
start,
end: page.next_offset,
},
},
text: page.content,
})
}
/// Close the first-run save/notification gap: Runtime can commit an initial turn before
/// the create handler has finalized the Backend registry required by recording callbacks.
/// Reconcile only after creation succeeds, using the ordinary current-owner/Feature checks.
@@ -1475,7 +1519,7 @@ pub(super) fn recording_request(
host: &JournalHostContext,
attribution: crate::subjektiv::SubjectSessionAttribution,
journal_id: &str,
materials: Vec<journal::recording::JournalMaterial>,
input: JournalInput,
) -> Result<BackendJobRequest> {
let binding_id = host
.writer_binding_id()
@@ -1488,18 +1532,88 @@ pub(super) fn recording_request(
subject_id: attribution.subject_id,
session_id: attribution.session_id,
journal_id: journal_id.into(),
selected_ranges: materials.iter().map(|m| m.range.clone()).collect(),
selected_ranges: input.selected_ranges.clone(),
};
Ok(BackendJobRequest {
job_id:format!("journal-recording:{}",Uuid::new_v4()),purpose:"subjektiv_journal".into(),
input_ref:format!("subjektiv://{}/journal/{journal_id}",grant.subject_id),
input:serde_json::json!({"journal_id":journal_id,"selected_ranges":grant.selected_ranges}),
instruction:"Record ONLY this frozen original public selected_ranges prefix. The input contains descriptors, NOT source bodies. First ReadJournalState and use bounded ReadJournalText Source until every byte of every selected range is returned; record operations are refused until then. ReadJournalText also exposes initial/current position/open text. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, update current position and CheckpointJournal using retained record receipts. Never infer event completion from turn/task/job/compaction boundaries. Preserve exact operation ID/full input after unknown outcomes. SubmitBackendJobResult only after the full selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(),
profile:"builtin:subjektiv-journal-recording".into(),
grants:BackendJobGrants {subjektiv_journal:Some(grant),..Default::default()},
serialization_key:Some(journal_serialization_key(journal_id)),source_worker:None,notification_target:None,
limits:crate::backend_job::BackendJobLimits {timeout_seconds:600,..Default::default()},
})
let request = BackendJobRequest {
job_id: format!("journal-recording:{}",Uuid::new_v4()), purpose: "subjektiv_journal".into(),
input_ref: format!("subjektiv://{}/journal/{journal_id}",grant.subject_id),
input: serde_json::to_value(input).map_err(|e| Error::Store(e.to_string()))?,
instruction: "Record ONLY the immutable body-bearing capture supplied in this input. These exact canonical source bytes, identities and ranges are the whole input for this Job; do not fetch original Sessions or use ReadJournalText Source. First ReadJournalState; initial/current Journal position/open text remain separately paged by ReadJournalText. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, current position and receipt-backed CheckpointJournal. Input delivery is not understanding or reflection. Never infer event completion from turn/task/job/compaction boundaries. Preserve exact operation ID/full input after Unknown. SubmitBackendJobResult only after the complete selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(),
profile: "builtin:subjektiv-journal-recording".into(),
grants: BackendJobGrants { subjektiv_journal: Some(grant), ..Default::default() },
serialization_key: Some(journal_serialization_key(journal_id)), source_worker: None, notification_target: None,
limits: crate::backend_job::BackendJobLimits { timeout_seconds: 600, ..Default::default() },
};
saved_input(
&request,
request
.grants
.subjektiv_journal
.as_ref()
.expect("constructed grant"),
)?;
request.validate()?;
Ok(request)
}
pub(super) fn capture_input(
api: &WorkspaceApi,
attr: &crate::subjektiv::SubjectSessionAttribution,
order: u64,
offset: u64,
budget: usize,
) -> Result<session_store::capture::CaptureDto> {
match api
.runtime
.session_recording_capture(
&attr.runtime_id,
runtime_api::SessionRecordingCaptureRequest {
workspace_id: api.workspace_id().into(),
source: material_source(api, attr)?,
expected_session_id: attr.session_id.clone(),
start: session_store::capture::RecordingPosition {
canonical_order: order,
byte_offset: offset,
},
max_entries: MAX_JOURNAL_RANGES,
max_serialized_bytes: budget,
},
)
.map_err(RuntimeRegistryError::into_error)?
{
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => {
if slice.capture.session_id != attr.session_id
|| serde_json::to_vec(&slice)
.map_err(|e| Error::Store(e.to_string()))?
.len()
> budget
{
return Err(Error::RepositoryConflict(
"recording capture identity/budget mismatch".into(),
));
}
Ok(slice.capture)
}
runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id }
if session_id == attr.session_id && order == 0 && offset == 0 =>
{
Ok(session_store::capture::CaptureDto {
version: 1,
session_id,
segment_id: "not-materialized".into(),
view_entry_count: 0,
entries: vec![],
})
}
runtime_api::SessionRecordingCaptureAvailability::Unavailable { reason, .. } => {
Err(Error::Store(format!(
"committed Journal capture unavailable ({reason:?}); no latest-source fallback"
)))
}
_ => Err(Error::Store(
"committed Journal capture identity mismatch; no latest-source fallback".into(),
)),
}
}
fn removal_attribution(
@@ -1575,6 +1689,16 @@ fn finish_erased_release(
"explicit erase tombstone missing".into(),
));
}
for session in store
.subject_session_attributions(&attr.subject_id)
.map_err(|e| Error::Store(e.to_string()))?
.into_iter()
.filter(|session| {
session.runtime_id == attr.runtime_id && session.worker_id == attr.worker_id
})
{
require_erased_job_copies(api, &session.session_id)?;
}
// Pre-binding bodies have no write token. This stable receipt label is not a writer
// grant: the domain erase operation records an inactive lifetime without binding Sessions.
let erased_body_id = format!("erased-body:{}", attr.worker_id);
@@ -1835,183 +1959,6 @@ pub(super) async fn finish_route(
}
/// A bounded Runtime transport view, not a second history index or a local-path fallback.
/// Reopening it observes a new catalog; each generation still compares its frozen source identity.
struct MaterialReader<'a> {
runtime: &'a RuntimeRegistry,
workspace_id: &'a str,
runtime_id: String,
source: runtime_api::SessionPublicSource,
session_id: String,
not_materialized: bool,
descriptors: Vec<session_store::public_material::PublicMaterialDescriptor>,
}
fn material_descriptor(
d: runtime_api::SessionMaterialDescriptor,
) -> Result<session_store::public_material::PublicMaterialDescriptor> {
Ok(session_store::public_material::PublicMaterialDescriptor {
identity: session_store::public_material::PublicMaterialIdentity {
segment_id: Uuid::parse_str(&d.segment_id).map_err(|e| Error::Store(e.to_string()))?,
entry_ref: d.entry_ref,
},
canonical_order: d.canonical_order,
origin: serde_json::from_str(&d.origin_json).map_err(|e| Error::Store(e.to_string()))?,
sha256: d.sha256,
total_bytes: d.total_bytes,
})
}
impl<'a> MaterialReader<'a> {
fn open(
runtime: &'a RuntimeRegistry,
workspace_id: &'a str,
runtime_id: &str,
source: runtime_api::SessionPublicSource,
session_id: &str,
) -> Result<Self> {
let mut descriptors = Vec::new();
let mut offset = 0;
let mut condition = None;
loop {
let page = match runtime
.session_material_catalog(
runtime_id,
runtime_api::SessionMaterialCatalogRequest {
workspace_id: workspace_id.into(),
source: source.clone(),
expected_session_id: session_id.into(),
expected_catalog_sha256: condition.clone(),
offset,
limit: 100,
},
)
.map_err(|e| e.into_error())?
{
runtime_api::SessionMaterialCatalogAvailability::NotMaterialized {
session_id: observed,
} => {
if observed != session_id || offset != 0 || condition.is_some() {
return Err(Error::RepositoryConflict(
"Journal Session materialization proof changed during catalog read"
.into(),
));
}
return Ok(Self {
runtime,
workspace_id,
runtime_id: runtime_id.into(),
source,
session_id: session_id.into(),
not_materialized: true,
descriptors,
});
}
runtime_api::SessionMaterialCatalogAvailability::Page { page } => page,
runtime_api::SessionMaterialCatalogAvailability::Unavailable {
reason,
message,
} => {
return Err(Error::Store(format!(
"Journal original material unavailable ({reason:?}): {message}"
)));
}
};
if page.session_id != session_id
|| condition
.as_ref()
.is_some_and(|c| c != &page.catalog_sha256)
|| page.descriptors.len() > 100
|| descriptors.len().saturating_add(page.descriptors.len()) > 1_000_000
{
return Err(Error::RepositoryConflict(
"Journal canonical Session catalog changed or exceeded its bound".into(),
));
}
condition = Some(page.catalog_sha256);
for d in page.descriptors {
let d = material_descriptor(d)?;
if d.canonical_order != descriptors.len() as u64 {
return Err(Error::RepositoryConflict(
"Journal canonical catalog order is discontinuous".into(),
));
}
descriptors.push(d);
}
match page.next_offset {
Some(next) if next > offset && next == descriptors.len() as u64 => offset = next,
None => break,
_ => {
return Err(Error::RepositoryConflict(
"Journal catalog continuation did not advance".into(),
));
}
}
}
Ok(Self {
runtime,
workspace_id,
runtime_id: runtime_id.into(),
source,
session_id: session_id.into(),
not_materialized: false,
descriptors,
})
}
fn descriptors(
&self,
) -> impl Iterator<Item = &session_store::public_material::PublicMaterialDescriptor> {
self.descriptors.iter()
}
fn read_content(
&self,
descriptor: &session_store::public_material::PublicMaterialDescriptor,
offset: u64,
limit: usize,
) -> Result<session_store::public_material::PublicMaterialContent> {
let page = match self
.runtime
.session_material_read(
&self.runtime_id,
runtime_api::SessionMaterialReadRequest {
workspace_id: self.workspace_id.into(),
source: self.source.clone(),
expected_session_id: self.session_id.clone(),
expected_descriptor: runtime_api::SessionMaterialDescriptor {
segment_id: descriptor.identity.segment_id.to_string(),
entry_ref: descriptor.identity.entry_ref.clone(),
canonical_order: descriptor.canonical_order,
origin_json: serde_json::to_string(&descriptor.origin)
.map_err(|e| Error::Store(e.to_string()))?,
sha256: descriptor.sha256.clone(),
total_bytes: descriptor.total_bytes,
},
byte_offset: offset,
max_content_bytes: limit.clamp(4, 16 * 1024),
},
)
.map_err(|e| e.into_error())?
{
runtime_api::SessionMaterialReadAvailability::Page { page } => page,
runtime_api::SessionMaterialReadAvailability::Unavailable { reason, message } => {
return Err(Error::Store(format!(
"Journal original material read unavailable ({reason:?}): {message}"
)));
}
};
if material_descriptor(page.descriptor)? != *descriptor
|| page.content.len() > limit.clamp(4, 16 * 1024)
{
return Err(Error::RepositoryConflict(
"Journal canonical material identity changed".into(),
));
}
Ok(session_store::public_material::PublicMaterialContent {
content: page.content,
next_offset: page.next_offset,
total_bytes: descriptor.total_bytes,
sha256: descriptor.sha256.clone(),
})
}
}
fn erase_source(
runtime: &RuntimeRegistry,
workspace: &str,
@@ -2145,17 +2092,29 @@ pub(super) fn reconcile_unknown(
let host = host(api, &store, grant)?;
let erased = store.journal_session_erased(&host).map_err(domain)?;
if !erased {
let g = store
.prepare_journal_generation(
&host,
&grant.journal_id,
subjektiv::api::JobAttemptBinding {
job_id: job.request.job_id.clone(),
attempt_id: attempt.attempt_id.clone(),
},
grant.selected_ranges.clone(),
)
.map_err(domain)?;
job.load_executable_request()?;
let binding = subjektiv::api::JobAttemptBinding {
job_id: job.request.job_id.clone(),
attempt_id: attempt.attempt_id.clone(),
};
let g = if legacy_descriptor(&job.request) {
// Only reconcile already-saved legacy receipts; never redispatch descriptors.
store
.prepare_journal_generation(
&host,
&grant.journal_id,
binding,
grant.selected_ranges.clone(),
)
.map_err(domain)?
} else {
let input = saved_input(&job.request, grant)?;
let generation = store
.prepare_journal_input(&host, &input, binding)
.map_err(domain)?;
input.validate_generation(&generation).map_err(domain)?;
generation
};
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
.map_err(domain)?;
match journal::service::finish(&store, &ctx) {
@@ -2187,6 +2146,14 @@ pub(super) fn reconcile_unknown(
);
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) {
drain_after_completion(api, &job.request);
if job.input_disposition == crate::backend_job::JobInputDisposition::Available {
api.store.reclaim_backend_job_input(
api.workspace_id(),
&job.request.job_id,
&job.input_digest,
crate::backend_job::JobInputRetirementReason::Completed,
)?;
}
} else {
api.schedule_backend_job_worker_cleanup(&current);
}
@@ -2226,8 +2193,8 @@ pub(super) fn reconcile_unknown(
&Utc::now().to_rfc3339(),
)?;
if !erased {
// The old immutable input may include reflected bytes. Select the remaining
// prefix afresh only AFTER definitive lifecycle/domain fences; never replace Unknown.
// Resume the same saved body input only after canonical execution/domain fences.
// dispatch selects its retained intent before it can capture any newer source.
let attr = store
.session_attribution(&grant.session_id)
.map_err(|e| Error::Store(e.to_string()))?
@@ -2256,6 +2223,30 @@ fn reconcile_session_jobs(
.store
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
if !erased
&& legacy_descriptor(&job.request)
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available
{
job.load_executable_request()?;
close(api, &job.request, &attempt)?;
if !attempt.state.terminal() {
attempt = api.store.finish_backend_job_attempt(
api.workspace_id(),
&job.request.job_id,
&id,
BackendJobAttemptState::Failed,
"journal_descriptor_invalidated",
"Descriptor-only Journal input retired; saved checkpoints retained",
&Utc::now().to_rfc3339(),
)?;
}
api.store.reclaim_backend_job_input(
api.workspace_id(),
&job.request.job_id,
&job.input_digest,
crate::backend_job::JobInputRetirementReason::LegacyInvalidation,
)?;
}
if erased && !attempt.state.terminal() {
// The domain tombstone precedes this cancellation and also fences any
// crash window before it. No erased Job can prepare or use tools again.
@@ -2269,6 +2260,30 @@ fn reconcile_session_jobs(
&Utc::now().to_rfc3339(),
)?;
}
if erased {
api.store.reclaim_backend_job_input(
api.workspace_id(),
&job.request.job_id,
&job.input_digest,
crate::backend_job::JobInputRetirementReason::ExplicitErasure,
)?;
// Every historical execution can own a delivered copy, not just the current
// attempt or an unacknowledged result. Cleanup is canonical Purge and retryable.
for number in 1..=job.current_attempt {
let historical_id = crate::backend_job::attempt_id(&job.request.job_id, number);
let historical = api
.store
.get_backend_job_attempt(
api.workspace_id(),
&job.request.job_id,
&historical_id,
)?
.ok_or_else(|| Error::Store("historical Journal attempt missing".into()))?;
if historical.state.terminal() {
api.schedule_backend_job_worker_cleanup(&historical);
}
}
}
if attempt.state == BackendJobAttemptState::Unknown {
reconcile_unknown(api, job, &attempt)?;
}
@@ -2284,6 +2299,41 @@ fn reconcile_session_jobs(
Ok(())
}
fn require_erased_job_copies(api: &WorkspaceApi, session: &str) -> Result<()> {
let mut after = None;
loop {
let jobs = api.store.list_backend_journal_jobs(
api.workspace_id(),
session,
after.as_deref(),
100,
)?;
for job in &jobs {
if job.input_disposition != crate::backend_job::JobInputDisposition::Erased {
return Err(Error::RepositoryConflict(
"Journal input erasure remains incomplete".into(),
));
}
for number in 1..=job.current_attempt {
let id = crate::backend_job::attempt_id(&job.request.job_id, number);
let attempt = api
.store
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
.ok_or_else(|| Error::Store("erased Journal attempt missing".into()))?;
if attempt.worker.is_some()
&& attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed)
{
return Err(Error::RepositoryConflict("Journal delivered-copy Purge remains pending; retry the same erasure after cleanup".into()));
}
}
}
if jobs.len() < 100 {
return Ok(());
}
after = jobs.last().map(|j| j.request.job_id.clone());
}
}
impl WorkspaceApi {
/// Host-only erasure saves the domain fence, cancels Jobs, then erases originals.
/// Runtime requires a stopped source Worker. Physical failure leaves the domain
@@ -2301,6 +2351,10 @@ impl WorkspaceApi {
.ok_or_else(|| {
Error::WorkspacePermissionDenied("Session attribution missing".into())
})?;
let gate = lifecycle_lock(self, &attr.subject_id)?;
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
store
.erase_journal_session(&host, request_id)
@@ -2311,7 +2365,8 @@ impl WorkspaceApi {
self.workspace_id(),
&attr,
material_source(self, &attr)?,
)
)?;
require_erased_job_copies(self, &attr.session_id)
}
}
@@ -14,10 +14,33 @@ use std::sync::{Arc, Mutex};
#[derive(Clone, Default)]
struct ConnectedJournalModel {
dialogs: Arc<Mutex<Vec<Request>>>,
journal_inputs: Arc<Mutex<Vec<String>>>,
}
impl ConnectedJournalModel {
fn journal_response(request: &Request) -> Vec<Event> {
// Consume the actual Controller-rendered input, not a fixture generation or
// a Source tool response. Decode also checks complete bodies/ranges/digests.
let envelope = request
.items
.iter()
.filter_map(Item::as_text)
.find(|text| text.lines().any(|line| line.starts_with("input_json: ")))
.expect("rendered immutable Backend Job envelope");
let input_json = envelope
.lines()
.find_map(|line| line.strip_prefix("input_json: "))
.unwrap();
let input_value: Value = serde_json::from_str(input_json).unwrap();
let frozen = subjektiv::journal::recording::JournalInput::decode(&input_value).unwrap();
assert_eq!(
serde_json::to_string(&input_value).unwrap(),
input_json,
"render preserves the exact persisted JSON bytes"
);
assert!(frozen.capture.entries.iter().all(|entry| matches!(
&entry.body, session_store::capture::CaptureBody::CanonicalPublic { text, .. } if !text.is_empty())));
let ranges = serde_json::to_value(&frozen.selected_ranges).unwrap();
// Runtime Controllers deliberately retain alias-routed core schemas for their
// lifetime even with zero attachments. Verify invocation authority, not schema
// absence (standalone Internal Jobs use a different composition).
@@ -89,7 +112,7 @@ impl ConnectedJournalModel {
"request_id":format!("checkpoint:{}",state["value"]["generation_id"].as_str().unwrap()),
"expected_state":position["value"]["state_token"],
"action":{"action":"checkpoint","record_operation_ids":[format!("record:{}",state["value"]["generation_id"].as_str().unwrap())]},
"covered_ranges":state["value"]["selected_ranges"],
"covered_ranges":ranges,
}),
)
} else if let Some(record) = result("record") {
@@ -106,41 +129,14 @@ impl ConnectedJournalModel {
)
} else if let Some(state) = result("state") {
let state = &state["value"];
// Deliver the actual frozen Runtime source through authenticated Job tools, rather
// than injecting a fixture Journal or claiming unseen source was processed.
for (index, range) in state["selected_ranges"]
.as_array()
.unwrap()
.iter()
.enumerate()
{
let mut offset = range["bytes"]["start"].as_u64().unwrap();
let mut token = Value::Null;
loop {
let id = format!("source-{index}-{offset}");
if let Some(page) = result(&id) {
let page = &page["value"];
if let Some(next) = page["next_byte"].as_u64() {
assert!(next > offset);
offset = next;
token = page["content_token"].clone();
} else {
break;
}
} else {
assert!(
request
.tools
.iter()
.any(|tool| tool.name == "ReadJournalText")
);
return vec![Event::tool_use_start(0, &id, "ReadJournalText"), Event::tool_input_delta(0, json!({
"target":{"target":"source","pointer":range["source"]["pointer"]},
"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token,
}).to_string()), Event::tool_use_stop(0)];
}
}
}
assert_eq!(
state["selected_ranges"], ranges,
"mutation coverage must come from the rendered frozen input"
);
assert_eq!(
state["initial_journal"]["state_token"],
input_value["initial_state_token"]
);
let (tool, action) = if state["current_open"].is_null() {
(
"CreateJournalEntry",
@@ -158,7 +154,7 @@ impl ConnectedJournalModel {
json!({
"request_id":format!("record:{}",state["generation_id"].as_str().unwrap()),
"expected_state":state["current_journal"]["state_token"],
"action":action,"covered_ranges":state["selected_ranges"],
"action":action,"covered_ranges":ranges,
}),
)
} else {
@@ -209,6 +205,18 @@ impl LlmClient for ConnectedJournalModel {
.iter()
.any(|tool| tool.name == "ReadJournalState")
{
let envelope = request
.items
.iter()
.filter_map(Item::as_text)
.find(|text| text.lines().any(|line| line.starts_with("input_json: ")))
.expect("actual Journal input in model render")
.to_owned();
let mut inputs = self.journal_inputs.lock().unwrap();
if !inputs.contains(&envelope) {
inputs.push(envelope);
}
drop(inputs);
Self::journal_response(&request)
} else if request.tools.iter().any(|tool| {
matches!(
@@ -458,6 +466,29 @@ async fn backend_real_runtime_journal_release_and_next_body_persisted_startup()
.store
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &old_attr.session_id, None, 100)
.unwrap();
// Compare actual render with the durable identity, even if completion already
// reclaimed the input body. Reconstituting only for comparison is test-local.
for saved in &recorded {
let inputs = model.journal_inputs.lock().unwrap();
let prefix = format!("job_id: {}", saved.request.job_id);
let rendered = inputs
.iter()
.find(|text| text.lines().any(|line| line == prefix))
.expect("every saved Journal Job reached the real Controller");
let actual = rendered
.lines()
.find_map(|line| line.strip_prefix("input_json: "))
.unwrap();
let mut original = saved.request.clone();
original.input = serde_json::from_str(actual).unwrap();
assert_eq!(original.input_digest().unwrap(), saved.input_digest);
let attempt = crate::backend_job::attempt_id(&saved.request.job_id, saved.current_attempt);
assert_eq!(*rendered, original.worker_input(&attempt).unwrap());
let input = subjektiv::journal::recording::JournalInput::decode(&original.input).unwrap();
assert_eq!(input.capture.session_id, old_attr.session_id);
assert!(actual.contains("User chose B rather than A"));
assert!(actual.contains("Partial B implementation"));
}
let uncertain_job = &recorded[0].request.job_id;
let conn = rusqlite::Connection::open(&api.config.database_path).unwrap();
let saved_result: (String, String) = conn.query_row(
File diff suppressed because it is too large Load Diff
+438 -14
View File
@@ -119,7 +119,7 @@ pub use ticket::{
};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 102;
const LATEST_SCHEMA_VERSION: i64 = 103;
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
@@ -437,6 +437,11 @@ const MIGRATIONS: &[Migration] = &[
name: "Durable failed Subject creation recovery",
apply: migrate_subject_create_recovery_v101_to_v102,
},
Migration {
version: 103,
name: "Job input retirement independent of immutable receipts",
apply: migrate_job_input_retention_v102_to_v103,
},
];
#[derive(Clone, Copy)]
@@ -1771,6 +1776,15 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
request: &BackendJobRequest,
now: &str,
) -> Result<BackendJobReservation>;
/// Host-only retirement; fence domain mutations and delivered snapshots first for erasure.
/// Completed reclamation additionally requires canonical Worker cleanup for all attempts.
fn reclaim_backend_job_input(
&self,
workspace_id: &str,
job_id: &str,
expected_input_digest: &str,
reason: crate::backend_job::JobInputRetirementReason,
) -> Result<BackendJobRecord>;
fn reserve_backend_job_retry(
&self,
workspace_id: &str,
@@ -7358,6 +7372,48 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
})
}
fn reclaim_backend_job_input(
&self,
workspace_id: &str,
job_id: &str,
expected_input_digest: &str,
reason: crate::backend_job::JobInputRetirementReason,
) -> Result<BackendJobRecord> {
use crate::backend_job::{JobInputDisposition, JobInputRetirementReason};
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
if job.input_digest != expected_input_digest {
return Err(Error::InvalidInput("Backend Job input digest mismatch".into()));
}
let desired = reason.disposition();
if job.input_disposition == desired {
tx.commit()?;
reclaim_backend_job_input_storage(conn)?;
return Ok(job);
}
if reason == JobInputRetirementReason::Completed {
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))?
.ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?;
if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed
|| job.result.is_none() || backend_job_has_unfinished_cleanup(&tx, workspace_id, job_id)?
{ return Err(Error::InvalidInput("input reclaim requires known completed result and completed Worker cleanup".into())); }
}
if job.input_disposition != JobInputDisposition::Available
&& reason != JobInputRetirementReason::ExplicitErasure
{ return Err(Error::InvalidInput("retired Backend Job input cannot be restored or reclassified".into())); }
let mut metadata = job.request.clone();
metadata.input = serde_json::Value::Null;
tx.execute("UPDATE backend_jobs SET request_json=?3, input_disposition=?4 WHERE workspace_id=?1 AND job_id=?2", params![workspace_id,job_id,serde_json::to_string(&metadata).map_err(|e| Error::Store(e.to_string()))?,desired.as_str()])?;
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::Store("retired Backend Job disappeared".into()))?;
tx.commit()?;
reclaim_backend_job_input_storage(conn)?;
Ok(job)
})
}
fn reserve_backend_job_retry(
&self,
workspace_id: &str,
@@ -7368,6 +7424,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let mut job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
job.load_executable_request()?;
let previous_id = attempt_id(job_id, job.current_attempt);
let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)?
.ok_or_else(|| Error::Store("Backend Job current attempt is missing".to_string()))?;
@@ -7407,7 +7464,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
}
tx.execute(
"INSERT INTO backend_job_attempts (workspace_id, job_id, attempt_id, attempt, input_digest, state, deadline_at, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, 'reserved', ?6, ?7, ?7)",
params![workspace_id, job_id, next_id, next as i64, job.request.input_digest()?, deadline, now],
params![workspace_id, job_id, next_id, next as i64, job.input_digest, deadline, now],
)?;
job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::Store("retried Backend Job is missing".to_string()))?;
@@ -7446,7 +7503,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
"SELECT job.job_id FROM backend_jobs job JOIN backend_job_attempts attempt
ON attempt.workspace_id = job.workspace_id AND attempt.job_id = job.job_id
AND attempt.attempt = job.current_attempt
WHERE job.workspace_id = ?1 AND job.state = 'pending'
WHERE job.workspace_id = ?1 AND job.state = 'pending' AND job.input_disposition = 'available'
AND attempt.state = 'dispatched' AND attempt.runtime_id = ?2 AND attempt.worker_id = ?3
AND attempt.runtime_run_id IS NOT NULL",
params![workspace_id, worker.runtime_id.as_str(), worker.worker_id.as_str()],
@@ -7506,6 +7563,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
if attempt.state != BackendJobAttemptState::Reserved {
@@ -7543,7 +7601,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<Vec<BackendJobAttemptRecord>> {
self.with_conn(|conn| {
let mut statement = conn.prepare(
"SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state = 'reserved' ORDER BY created_at, attempt_id LIMIT ?2",
"SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='reserved' AND j.input_disposition='available' ORDER BY a.created_at, a.attempt_id LIMIT ?2",
)?;
let identities = statement.query_map(params![workspace_id, limit as i64], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
@@ -7568,9 +7626,10 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
if attempt.attempt != job.current_attempt || attempt.input_digest != job.request.input_digest()? {
if attempt.attempt != job.current_attempt || attempt.input_digest != job.input_digest {
return Err(Error::InvalidInput(
"Backend Job attempt is stale for the current input digest".to_string(),
));
@@ -7629,6 +7688,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
}
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| {
Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`"))
@@ -7682,7 +7743,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?;
let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
if submission.input_digest != job.request.input_digest()?
if submission.input_digest != job.input_digest
|| attempt.input_digest != submission.input_digest
|| attempt.attempt != job.current_attempt
{
@@ -7709,6 +7770,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
tx.commit()?;
return Ok(BackendJobResultAcceptance { job, attempt, replayed: true });
}
job.load_executable_request()?;
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
return Err(Error::InvalidInput(
"Backend Job attempt no longer accepts results".to_string(),
@@ -7776,7 +7838,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
// Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence.
if &attempt != expected || job.current_attempt != attempt.attempt
|| job.request.grants.subjektiv_journal.is_none()
|| attempt.input_digest != job.request.input_digest()?
|| attempt.input_digest != job.input_digest
|| !matches!(job.state, BackendJobState::Unknown | BackendJobState::Completed | BackendJobState::Failed) {
return Err(Error::InvalidInput("Journal reconciliation binding changed".into()));
}
@@ -12109,6 +12171,20 @@ fn read_memory_staging_resolution_record(
})
}
// The receipt/disposition commit precedes physical cleanup. An error here is
// safely retried with the same expected digest/reason, not a new execution.
fn reclaim_backend_job_input_storage(conn: &Connection) -> Result<()> {
conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?;
let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?;
if busy != 0 {
return Err(Error::Store(
"Backend Job input retired; SQLite body cleanup blocked by a reader, retry retirement"
.into(),
));
}
Ok(())
}
fn backend_job_deadline(now: &str, timeout_seconds: u32) -> Result<String> {
let now = chrono::DateTime::parse_from_rfc3339(now)
.map_err(|error| Error::InvalidInput(format!("invalid Backend Job timestamp: {error}")))?;
@@ -12172,7 +12248,7 @@ pub(crate) fn terminal_journal_job_worker(
Ok(job.request.grants.subjektiv_journal.is_some()
&& attempt.state.terminal()
&& attempt.worker.as_ref() == Some(worker)
&& attempt.input_digest == job.request.input_digest()?)
&& attempt.input_digest == job.input_digest)
}
fn read_backend_job(
@@ -12181,13 +12257,13 @@ fn read_backend_job(
job_id: &str,
) -> Result<Option<BackendJobRecord>> {
let row = conn.query_row(
"SELECT request_json, intent_fingerprint, state, current_attempt, result_json, result_digest, failure_category, failure_detail, created_at, updated_at, completed_at FROM backend_jobs WHERE workspace_id = ?1 AND job_id = ?2",
"SELECT request_json, intent_fingerprint, state, current_attempt, result_json, result_digest, failure_category, failure_detail, created_at, updated_at, completed_at, input_digest, input_disposition FROM backend_jobs WHERE workspace_id = ?1 AND job_id = ?2",
params![workspace_id, job_id],
|row| Ok((
row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?,
row.get::<_, i64>(3)?, row.get::<_, Option<String>>(4)?, row.get::<_, Option<String>>(5)?,
row.get::<_, Option<String>>(6)?, row.get::<_, Option<String>>(7)?, row.get::<_, String>(8)?,
row.get::<_, String>(9)?, row.get::<_, Option<String>>(10)?,
row.get::<_, String>(9)?, row.get::<_, Option<String>>(10)?, row.get::<_, String>(11)?, row.get::<_, String>(12)?,
)),
).optional()?;
let Some((
@@ -12202,11 +12278,20 @@ fn read_backend_job(
created_at,
updated_at,
completed_at,
input_digest,
input_disposition,
)) = row
else {
return Ok(None);
};
let request: BackendJobRequest = serde_json::from_str(&request_json)
let request_value: serde_json::Value = serde_json::from_str(&request_json)
.map_err(|error| Error::Store(format!("invalid stored Backend Job request: {error}")))?;
if request_value.get("input").is_none() {
return Err(Error::Store(
"stored Backend Job input field is missing".into(),
));
}
let request: BackendJobRequest = serde_json::from_value(request_value)
.map_err(|error| Error::Store(format!("invalid stored Backend Job request: {error}")))?;
let result = result_json
.map(|value| serde_json::from_str(&value))
@@ -12218,6 +12303,9 @@ fn read_backend_job(
workspace_id: workspace_id.to_string(),
request,
intent_fingerprint,
input_digest,
input_disposition: serde_json::from_value(serde_json::Value::String(input_disposition))
.map_err(|e| Error::Store(format!("invalid Job input disposition: {e}")))?,
state: BackendJobState::parse(&state)?,
current_attempt,
result,
@@ -15130,6 +15218,24 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> {
Ok(())
}
fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 102 {
return Err(Error::Store(
"Job input retention migration requires schema 102".into(),
));
}
let tx = conn.unchecked_transaction()?;
tx.execute_batch("ALTER TABLE backend_jobs ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));")?;
tx.execute_batch(crate::backend_job::INPUT_RETENTION_TRIGGERS)?;
tx.execute(
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(103,?1)",
["Job input retirement independent of immutable receipts"],
)?;
verify_workspace_resource_constraints(&tx)?;
tx.commit()?;
Ok(())
}
fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? != 101 {
return Err(Error::Store(
@@ -18534,7 +18640,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
assert!(conn.execute("INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) VALUES('invalid','missing','Invalid','active','1','1')", []).is_err());
}
fn prepare_retained_schema(path: &Path, version: i64) {
pub(super) fn prepare_retained_schema(path: &Path, version: i64) {
// Cache only empty preconditions, built from frozen DDL and real retained
// migrations. The test's seeded rows and migration under test are never shared.
static TEMPLATES: std::sync::OnceLock<Mutex<std::collections::BTreeMap<i64, Connection>>> =
@@ -18914,6 +19020,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
WorkspaceSchemaMigrationStep {version:103, name:"Job input retirement independent of immutable receipts".into()},
]
);
@@ -19026,6 +19133,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(100, "Provisional users and scoped passkey activation".to_string()),
(101, "Durable explicit Subject body release".to_string()),
(102, "Durable failed Subject creation recovery".to_string()),
(103, "Job input retirement independent of immutable receipts".to_string()),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -21293,7 +21401,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
drop(conn);
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
reopened
.with_conn(|c| verify_schema_history(c, 102))
.with_conn(|c| verify_schema_history(c, LATEST_SCHEMA_VERSION))
.unwrap();
}
@@ -22232,7 +22340,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(99, "Explicit hosting administrators and self-registration policy".into()),
(100, "Provisional users and scoped passkey activation".into()),
(101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into())]);
(102, "Durable failed Subject creation recovery".into()),
(103, "Job input retirement independent of immutable receipts".into())]);
for (table, rows) in unchanged.iter().zip(before.iter()) {
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
}
@@ -26150,6 +26259,321 @@ INSERT INTO worker_registry (
.unwrap();
}
// Storage component tests: seed already-saved result/cleanup receipts, without
// constructing an execution Worker or claiming a model/dispatch integration.
fn seed_completed_job_receipts(store: &SqliteWorkspaceStore, id: &str, cleanup: &str) {
let result = serde_json::json!({"stored":true});
let (digest, json) = result_digest(&result, job::ABSOLUTE_MAX_RESULT_BYTES).unwrap();
store.with_conn(|conn| {
conn.execute("UPDATE backend_job_attempts SET state='completed',result_json=?2,result_digest=?3,worker_cleanup_state=?4,worker_cleanup_completed_at=?5 WHERE job_id=?1", params![id,json,digest,cleanup,RESOURCE_JOB_NOW])?;
conn.execute("UPDATE backend_jobs SET state='completed',result_json=?2,result_digest=?3,completed_at=?4 WHERE job_id=?1", params![id,json,digest,RESOURCE_JOB_NOW])?;
Ok(())
}).unwrap();
}
#[tokio::test]
async fn backend_job_input_reclaim_preserves_receipts_reopen_and_exact_replay() {
use crate::backend_job::{JobInputDisposition, JobInputRetirementReason};
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("input.sqlite");
let store = SqliteWorkspaceStore::open_api_fixture(&path).unwrap();
seed_resource_job_workspace(&store).await;
let original = resource_job_request("reclaim", "subject-a", "candidate");
let reservation = store
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap();
assert!(
store
.reclaim_backend_job_input(
"resource-workspace",
"reclaim",
&reservation.job.input_digest,
JobInputRetirementReason::Completed
)
.is_err()
);
seed_completed_job_receipts(&store, "reclaim", "pending");
assert!(
store
.reclaim_backend_job_input(
"resource-workspace",
"reclaim",
&reservation.job.input_digest,
JobInputRetirementReason::Completed
)
.is_err()
);
store.with_conn(|conn| { conn.execute("UPDATE backend_job_attempts SET worker_cleanup_state='completed' WHERE job_id='reclaim'",[])?; Ok(()) }).unwrap();
let before = store
.get_backend_job("resource-workspace", "reclaim")
.unwrap()
.unwrap();
let attempt = store
.get_backend_job_attempt(
"resource-workspace",
"reclaim",
&reservation.attempt.attempt_id,
)
.unwrap()
.unwrap();
assert!(
store
.reclaim_backend_job_input(
"resource-workspace",
"reclaim",
"wrong",
JobInputRetirementReason::Completed
)
.is_err()
);
let retired = store
.reclaim_backend_job_input(
"resource-workspace",
"reclaim",
&before.input_digest,
JobInputRetirementReason::Completed,
)
.unwrap();
assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed);
assert!(retired.request.input.is_null());
assert!(retired.load_executable_request().is_err());
assert_eq!(retired.input_digest, original.input_digest().unwrap());
assert_eq!(retired.intent_fingerprint, before.intent_fingerprint);
assert_eq!(retired.request.grants, original.grants);
assert_eq!(retired.result, before.result);
assert_eq!(
store
.get_backend_job_attempt("resource-workspace", "reclaim", &attempt.attempt_id)
.unwrap()
.unwrap(),
attempt
);
drop(store);
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let replay = reopened
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap();
assert!(replay.replayed);
assert_eq!(replay.job, retired);
assert_eq!(
replay.outcome().input_digest,
original.input_digest().unwrap()
);
let mut changed = original.clone();
changed.input = serde_json::json!({"changed":true});
assert!(
reopened
.reserve_backend_job("resource-workspace", &changed, RESOURCE_JOB_NOW)
.is_err()
);
assert!(
reopened
.reserve_backend_job_retry("resource-workspace", "reclaim", RESOURCE_JOB_NOW)
.is_err()
);
assert!(
reopened
.claim_backend_job_attempt_dispatch(
"resource-workspace",
"reclaim",
&attempt.attempt_id,
RESOURCE_JOB_NOW
)
.is_err()
);
assert!(reopened.with_conn(|conn| { conn.execute("UPDATE backend_jobs SET input_disposition='available',request_json=?1 WHERE job_id='reclaim'", [serde_json::to_string(&original).unwrap()])?; Ok(()) }).is_err());
let erased = reopened
.reclaim_backend_job_input(
"resource-workspace",
"reclaim",
&retired.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
assert_eq!(erased.state, BackendJobState::Completed);
assert_eq!(erased.result, before.result);
}
#[tokio::test]
async fn backend_job_input_unknown_retention_and_legacy_invalidation_do_not_reinterpret_outcome()
{
use crate::backend_job::{JobInputDisposition, JobInputRetirementReason};
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
seed_resource_job_workspace(&store).await;
for (id, state) in [
("unfinished", None),
("unknown-input", Some(BackendJobAttemptState::Unknown)),
("failed-input", Some(BackendJobAttemptState::Failed)),
] {
let original = resource_job_request(id, id, "candidate");
let reservation = store
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap();
if let Some(state) = state {
store
.finish_backend_job_attempt(
"resource-workspace",
id,
&reservation.attempt.attempt_id,
state,
"fixture",
"known stored outcome",
RESOURCE_JOB_NOW,
)
.unwrap();
}
let before = store
.get_backend_job("resource-workspace", id)
.unwrap()
.unwrap();
assert!(
store
.reclaim_backend_job_input(
"resource-workspace",
id,
&before.input_digest,
JobInputRetirementReason::Completed
)
.is_err()
);
assert_eq!(before.load_executable_request().unwrap(), &original);
assert_eq!(
store
.get_backend_job("resource-workspace", id)
.unwrap()
.unwrap(),
before
);
let retired = store
.reclaim_backend_job_input(
"resource-workspace",
id,
&before.input_digest,
JobInputRetirementReason::LegacyInvalidation,
)
.unwrap();
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
assert_eq!(retired.state, before.state);
assert_eq!(retired.failure_category, before.failure_category);
assert_eq!(retired.completed_at, before.completed_at);
assert!(retired.load_executable_request().is_err());
assert!(
store
.reserve_backend_job_retry("resource-workspace", id, RESOURCE_JOB_NOW)
.is_err()
);
assert!(
store
.claim_backend_job_attempt_dispatch(
"resource-workspace",
id,
&reservation.attempt.attempt_id,
RESOURCE_JOB_NOW
)
.is_err()
);
assert_eq!(
store
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap()
.job,
retired
);
}
}
#[tokio::test]
async fn backend_job_input_available_loader_and_dispatch_refuse_missing_tampered_or_changed_intent()
{
for mutation in [
"json_set(request_json,'$.input',json('{\"tampered\":true}'))",
"json_remove(request_json,'$.input')",
"json_set(request_json,'$.instruction','changed')",
] {
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
seed_resource_job_workspace(&store).await;
let original = resource_job_request("tamper", "subject", "candidate");
let reservation = store
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap();
store
.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER backend_job_input_retirement")?;
conn.execute_batch(&format!(
"UPDATE backend_jobs SET request_json={mutation}"
))?;
Ok(())
})
.unwrap();
if let Ok(Some(job)) = store.get_backend_job("resource-workspace", "tamper") {
assert!(job.load_executable_request().is_err());
}
assert!(
store
.claim_backend_job_attempt_dispatch(
"resource-workspace",
"tamper",
&reservation.attempt.attempt_id,
RESOURCE_JOB_NOW
)
.is_err()
);
}
}
#[tokio::test]
async fn backend_job_input_migration_preserves_original_identity_and_rolls_back_failure() {
use crate::backend_job::{JobInputDisposition, JobInputRetirementReason};
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("migration.sqlite");
let store = SqliteWorkspaceStore::open_api_fixture(&path).unwrap();
seed_resource_job_workspace(&store).await;
let original = resource_job_request("upgrade", "subject", "candidate");
let before = store
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap();
store.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
}).unwrap();
drop(store);
assert!(SqliteWorkspaceStore::open_for_test(&path).is_err());
let conn = Connection::open(&path).unwrap();
assert_eq!(current_schema_version(&conn).unwrap(), 102);
assert!(!column_exists(&conn, "backend_jobs", "input_disposition").unwrap());
conn.execute_batch("DROP TRIGGER backend_job_input_retirement")
.unwrap();
drop(conn);
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
assert_eq!(
reopened
.get_backend_job("resource-workspace", "upgrade")
.unwrap()
.unwrap(),
before.job
);
let erased = reopened
.reclaim_backend_job_input(
"resource-workspace",
"upgrade",
&before.job.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
assert_eq!(erased.state, BackendJobState::Pending);
assert_eq!(
reopened
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap()
.job,
erased
);
}
#[tokio::test]
async fn backend_job_resource_reserve_converges_across_connections_without_changing_input() {
let dir = tempfile::tempdir().unwrap();
@@ -231,12 +231,10 @@ pub fn result_facts(
current_ticket: &ticket::Ticket,
) -> Result<TicketItemCheckFacts> {
let input = parse_input(&acceptance.job.request.input)?;
let result = validate_result(
&acceptance.job.request.input,
acceptance.job.result.as_ref().ok_or_else(|| {
Error::InvalidInput("completed Ticket item checker Job has no result".to_string())
})?,
)?;
let result_value = acceptance.job.result.as_ref().ok_or_else(|| {
Error::InvalidInput("completed Ticket item checker Job has no result".to_string())
})?;
let result = validate_result(&acceptance.job.request.input, result_value)?;
if input.content_digest != content_digest(current_ticket)
|| input.ticket_id != current_ticket.meta.id
{
@@ -246,21 +244,17 @@ pub fn result_facts(
return Ok(TicketItemCheckFacts::NoFindings);
}
Ok(TicketItemCheckFacts::Findings(render_result(acceptance)?))
Ok(TicketItemCheckFacts::Findings(render_result(
&acceptance.job.request.input,
result_value,
)?))
}
/// Stable rendering from the original checked snapshot. Used to pin pre-upgrade
/// unacknowledged requests without reinterpreting them against newer Ticket text.
pub(crate) fn render_result(acceptance: &BackendJobResultAcceptance) -> Result<String> {
let input = parse_input(&acceptance.job.request.input)?;
let result = validate_result(
&acceptance.job.request.input,
acceptance
.job
.result
.as_ref()
.ok_or_else(|| Error::InvalidInput("completed checker has no result".into()))?,
)?;
pub(crate) fn render_result(input: &Value, result: &Value) -> Result<String> {
let checked = parse_input(input)?;
let result = validate_result(input, result)?;
if result.findings.is_empty() {
return Err(Error::InvalidInput(
"clean checker result has no advisory body".into(),
@@ -268,7 +262,7 @@ pub(crate) fn render_result(acceptance: &BackendJobResultAcceptance) -> Result<S
}
let mut content = format!(
"[Ticket item checker advisory]\nTicket: {}\nContent digest: {}\n\nThis is a post-save advisory based only on the inspected Ticket text. It is not a new user request, approval gate, workflow blocker, or implementation instruction.\n",
input.ticket_resource_key, input.content_digest
checked.ticket_resource_key, checked.content_digest
);
for (index, finding) in result.findings.iter().enumerate() {
let quoted = serde_json::to_string(&finding.quote).map_err(|error| {
@@ -467,6 +461,8 @@ mod tests {
job: BackendJobRecord {
workspace_id: "checker-unit".into(),
intent_fingerprint: request.fingerprint().unwrap(),
input_digest: request.input_digest().unwrap(),
input_disposition: crate::backend_job::JobInputDisposition::Available,
request,
state: BackendJobState::Completed,
current_attempt: 1,