diff --git a/Cargo.lock b/Cargo.lock index fb65b50ba..a674c0ee9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3990,6 +3990,7 @@ dependencies = [ "protocol", "serde", "serde_json", + "session-store", "tokio", "workdir", ] @@ -4783,12 +4784,14 @@ version = "0.1.0" dependencies = [ "chrono", "feature-storage", + "job", "manifest", "memory", "rusqlite", "serde", "serde_json", "server-api", + "session-store", "sha2 0.11.0", "tempfile", "thiserror 2.0.18", diff --git a/crates/runtime-api/Cargo.toml b/crates/runtime-api/Cargo.toml index 4f250aad5..cc544d3e9 100644 --- a/crates/runtime-api/Cargo.toml +++ b/crates/runtime-api/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true api-macros = { workspace = true, features = ["reqwest", "axum"] } manifest.workspace = true protocol.workspace = true +session-store.workspace = true serde = { workspace = true, features = ["derive"] } workdir.workspace = true diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index 729243ba6..e0e3502ea 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -836,6 +836,34 @@ pub struct SessionMaterialPinResponse { pub pin_id: String, } +/// One body-bearing immutable slice. The byte budget includes slice metadata, +/// but not the enclosing availability enum or the caller's durable Job envelope. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SessionRecordingCaptureRequest { + pub workspace_id: String, + pub source: SessionPublicSource, + pub expected_session_id: String, + pub start: session_store::capture::RecordingPosition, + pub max_entries: usize, + pub max_serialized_bytes: usize, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "availability", rename_all = "snake_case")] +pub enum SessionRecordingCaptureAvailability { + NotMaterialized { + session_id: String, + }, + Captured { + slice: session_store::capture::RecordingCaptureSlice, + }, + Unavailable { + reason: SessionPublicUnavailableReason, + message: String, + }, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SessionMaterialCatalogRequest { @@ -1246,6 +1274,16 @@ pub trait RuntimeApi { #[body] request: SessionMaterialPinRequest, ) -> Result; + #[post( + "/v1/session-public/recording-capture", + status = 200, + error_status = 400 + )] + async fn session_recording_capture( + &self, + #[body] request: SessionRecordingCaptureRequest, + ) -> Result; + #[post( "/v1/session-public/material/catalog", status = 200, @@ -1730,6 +1768,13 @@ mod tests { Err(test_error(501)) } + async fn session_recording_capture( + &self, + _request: SessionRecordingCaptureRequest, + ) -> Result { + Err(test_error(501)) + } + async fn session_material_catalog( &self, _request: SessionMaterialCatalogRequest, diff --git a/crates/session-store/src/capture.rs b/crates/session-store/src/capture.rs index 07da4c1f7..b010e3381 100644 --- a/crates/session-store/src/capture.rs +++ b/crates/session-store/src/capture.rs @@ -13,6 +13,14 @@ //! observation share only this domain; they do not enable or modify each other's Features. //! The provider owns authorization to recapture a current Worker. A capture itself proves neither //! completion nor approval and grants no business-mutation authority. +//! +//! Recording Hosts use the common committed replay through the guarded APIs below. +//! [`recording_catalog_worker_guarded`] and [`recording_catalog_fs_guarded`] expose +//! descriptors only to an admission callback under the caller's existing publication/ +//! erasure guard and persisted active selector, without allocating a full body DTO. +//! [`capture_recording_slice_worker_guarded`] and [`capture_recording_slice_fs_guarded`] +//! separately construct bounded body-bearing inputs. Admission descriptors are not +//! sufficient Job inputs; saved recording DTOs are read/sliced without live storage. use std::sync::Arc; @@ -21,12 +29,14 @@ use crate::{ LoggedSessionHistoryOrigin as HistoryOrigin, }; mod body; +mod recording; use agen::{HistoryEntry, Item, Role}; pub use body::*; use protocol::{ SessionContentPart, SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntryData, }; +pub use recording::*; use serde::{Deserialize, Serialize}; const DEFAULT_SEARCH_LIMIT: usize = 20; @@ -50,7 +60,8 @@ impl SessionEntryRef { pub fn parse(value: &str) -> Option { let suffix = value.strip_prefix('E')?; if suffix.is_empty() - || suffix.len() > 64 + // Legacy Host boundary identities contain both Session and Segment UUIDs. + || suffix.len() > 256 || !suffix .bytes() .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') @@ -822,7 +833,7 @@ impl SessionCapture { } } -fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance { +pub(crate) fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance { match origin { HistoryOrigin::HumanInput { .. } | HistoryOrigin::LocalHumanInput => { SessionEntryProvenance::HumanInput diff --git a/crates/session-store/src/capture/body.rs b/crates/session-store/src/capture/body.rs index eb7768622..d1ca3dfc1 100644 --- a/crates/session-store/src/capture/body.rs +++ b/crates/session-store/src/capture/body.rs @@ -5,7 +5,7 @@ use super::*; use serde::de::Error as _; use sha2::{Digest, Sha256}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::io::Write; pub const CAPTURE_VERSION: u32 = 1; @@ -81,6 +81,17 @@ impl CaptureContentPart { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum CaptureBody { + /// Canonical recording bytes retain the historical Journal descriptor contract. + /// A nonzero offset or short text is an explicit fragment, never a full body. + /// History entries retain their original raw segment-history row, including + /// hidden rows in the count. A typed Host boundary has no history row: its + /// `source.entry_index` is the next-history gap at the original boundary. + CanonicalPublic { + descriptor: crate::public_material::PublicMaterialDescriptor, + byte_offset: u64, + text: String, + boundary: Option, + }, Message { role: CaptureRole, content: Vec, @@ -175,6 +186,7 @@ impl CaptureBody { pub(super) fn to_item(&self) -> Item { match self { + Self::CanonicalPublic { text, .. } => Item::system_message(text.clone()), Self::Message { role, content } => Item::Message { id: None, status: None, @@ -214,6 +226,7 @@ impl CaptureBody { fn fields(&self) -> Vec<(RawField, &str)> { match self { + Self::CanonicalPublic { text, .. } => vec![(RawField::CanonicalPublic, text)], Self::Message { content, .. } => content .iter() .enumerate() @@ -235,6 +248,16 @@ impl CaptureBody { fn metadata(&self, field: RawField) -> CaptureBodyMetadata { match self { + Self::CanonicalPublic { + descriptor, + byte_offset, + boundary, + .. + } => CaptureBodyMetadata::CanonicalPublic { + descriptor: descriptor.clone(), + byte_offset: *byte_offset, + boundary: *boundary, + }, Self::Message { role, content } => CaptureBodyMetadata::Message { role: *role, refusal: matches!(field, RawField::MessagePart(i) if matches!(content.get(i), Some(CaptureContentPart::Refusal { .. }))), @@ -264,6 +287,9 @@ impl CaptureBody { #[serde(deny_unknown_fields)] pub struct CaptureSource { pub segment_id: String, + /// Original raw segment-history position, not captured/public view order. + /// For `CanonicalPublic` Host boundaries only, this is the next-history gap; + /// several typed Host events may share that gap with the following history row. pub entry_index: u64, pub provenance: SessionEntryProvenance, /// Full original origin when supplied by the committed-history adapter. @@ -375,7 +401,7 @@ impl Write for ByteCounter { } } -fn json_size(value: &T, limit: usize) -> Result { +pub(super) fn json_size(value: &T, limit: usize) -> Result { let mut counter = ByteCounter { bytes: 0, limit }; serde_json::to_writer(&mut counter, value).map_err(|_| CaptureError::ResourceLimit)?; Ok(counter.bytes) @@ -403,6 +429,7 @@ impl CaptureDto { identity(&self.segment_id, limits)?; let mut ids = HashSet::new(); let mut sources = HashSet::new(); + let mut recording_source_floor = HashMap::<&str, u64>::new(); let mut previous = None; for entry in &self.entries { identity(&entry.entry_id, limits)?; @@ -419,7 +446,16 @@ impl CaptureDto { } previous = Some(entry.order); identity(&entry.source.segment_id, limits)?; - if !sources.insert((&entry.source.segment_id, entry.source.entry_index)) { + let host_boundary = matches!( + &entry.body, + CaptureBody::CanonicalPublic { + boundary: Some(_), + .. + } + ); + if !host_boundary + && !sources.insert((&entry.source.segment_id, entry.source.entry_index)) + { return Err(CaptureError::Invalid("duplicate source position")); } if !entry.policy.any() { @@ -443,6 +479,79 @@ impl CaptureDto { } } match &entry.body { + CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + boundary, + } => { + if descriptor.identity.entry_ref != format!("E{}", entry.entry_id) + || descriptor.identity.segment_id.to_string() != entry.source.segment_id + || descriptor.canonical_order != entry.order + || text.is_empty() + || descriptor.total_bytes == 0 + || byte_offset + .checked_add(text.len() as u64) + .is_none_or(|end| end > descriptor.total_bytes) + || descriptor.sha256.len() != 64 + || !descriptor + .sha256 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + || !entry.policy.recording_material + || entry.policy.memory_evidence + || entry.policy.observation_readable + { + return Err(CaptureError::Invalid("canonical descriptor")); + } + match (&descriptor.origin, &entry.source.history_origin, boundary) { + ( + crate::public_material::PublicMaterialOrigin::History { origin }, + Some(stored), + None, + ) if origin == stored => {} + ( + crate::public_material::PublicMaterialOrigin::HostRunBoundary, + None, + Some(_), + ) => {} + _ => return Err(CaptureError::Invalid("canonical origin")), + } + // A partial DTO cannot prove the hidden history extent, but can + // reject aliases and reversals of original per-Segment positions. + // Host events anchor a gap, not a fabricated history row. + let floor = recording_source_floor + .entry(&entry.source.segment_id) + .or_default(); + if entry.source.entry_index < *floor { + return Err(CaptureError::Invalid("recording source position order")); + } + *floor = if boundary.is_some() { + entry.source.entry_index + } else { + entry + .source + .entry_index + .checked_add(1) + .ok_or(CaptureError::Invalid("source position overflow"))? + }; + if *byte_offset == 0 && text.len() as u64 == descriptor.total_bytes { + let digest: String = Sha256::digest(text.as_bytes()) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + if digest != descriptor.sha256 { + return Err(CaptureError::Invalid("canonical digest")); + } + if let Some(boundary) = boundary { + let expected = + serde_json::json!({"kind":"run_boundary", "boundary":boundary}); + if serde_json::from_str::(text)? != expected { + return Err(CaptureError::Invalid("run boundary body")); + } + } + } + } CaptureBody::Message { content, .. } if content.is_empty() => { return Err(CaptureError::Invalid("empty message parts")); } @@ -493,6 +602,7 @@ pub struct ScopedSessionEntryRef { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum RawField { + CanonicalPublic, MessagePart(usize), PublicSystem, ToolArguments, @@ -515,6 +625,11 @@ pub struct RawRead { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum CaptureBodyMetadata { + CanonicalPublic { + descriptor: crate::public_material::PublicMaterialDescriptor, + byte_offset: u64, + boundary: Option, + }, Message { role: CaptureRole, refusal: bool, diff --git a/crates/session-store/src/capture/recording.rs b/crates/session-store/src/capture/recording.rs new file mode 100644 index 000000000..5a767da8c --- /dev/null +++ b/crates/session-store/src/capture/recording.rs @@ -0,0 +1,425 @@ +//! Host capture and pure canonical slicing. No Job state or live-reader fallback lives here. + +use super::*; +use crate::committed_replay::CommittedPublicReplay; +use crate::public_material::{ + PublicMaterialDescriptor, PublicMaterialError, PublicMaterialLimits, PublicMaterialUse, +}; +use crate::retention::SessionRetentionStore; +use crate::{SegmentId, SessionId}; +use std::path::Path; + +/// Continuation in canonical Journal order, not an observation/Memory view cursor. +/// A Host must reconcile the descriptor at this position with its saved checkpoint. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecordingPosition { + pub canonical_order: u64, + pub byte_offset: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecordingCaptureSlice { + pub capture: CaptureDto, + pub next: Option, +} + +/// Inspect recording admission descriptors without constructing a body-bearing DTO. +/// +/// The Host must authorize the source and hold the **same** Session publication/erasure +/// guard from before reading its persisted active pointer until this function returns +/// (including the callback). Pass that pointer's Segment, not a filename/mtime-derived +/// selector. This function verifies the guard and stored Session identities but does +/// not authorize the caller or establish that the supplied guard fences this root. +/// No lock is recursively acquired, no active pointer is reread, and no replay handle +/// or body-read capability is exposed to `inspect`. +/// +/// The common committed recording replay fixes per-file ends before decoding. It +/// retains only descriptors/locations, decoding one record at a time and streaming +/// canonical bodies into their digests. `replay_limits` bound record allocation, +/// aggregate storage extent, entries and Segments; there is no full-capture text/JSON +/// allocation or `CaptureLimits` ceiling. The callback receives only descriptors in +/// canonical order and may return a Host-specific mapping (including a `Result`). It +/// runs once on success and not at all on replay/identity/resource errors. +/// +/// This catalog is for Host admission, **not** a descriptor-only Job input. Construct +/// the Job's bounded body input with the guarded slice API at its own authorized +/// boundary; after saving it, execute only that immutable value. Neither descriptors +/// nor a completed callback authorize delayed adoption after erasure. +/// +/// ```no_run +/// # use std::path::Path; +/// # use session_store::{SessionId, SegmentId}; +/// # use session_store::retention::SessionRetentionGuard; +/// # use session_store::public_material::PublicMaterialError; +/// # fn admit(root: &Path, session: SessionId, active: SegmentId, +/// # guard: &SessionRetentionGuard) -> Result, PublicMaterialError> { +/// // `active` was resolved from persisted Host metadata while holding `guard`. +/// session_store::capture::recording_catalog_worker_guarded( +/// root, session, active, Default::default(), guard, +/// |descriptors| descriptors.iter().map(|d| d.total_bytes).collect(), +/// ) +/// # } +/// ``` +pub fn recording_catalog_worker_guarded( + session_root: &Path, + expected_session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + guard: &crate::retention::SessionRetentionGuard, + inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T, +) -> Result { + if guard.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + let replay = CommittedPublicReplay::open_worker_guarded( + session_root, + active_segment, + replay_limits, + None, + PublicMaterialUse::Recording, + )?; + if replay.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + Ok(inspect(&replay.into_recording_descriptors())) +} + +/// FsStore-layout counterpart of [`recording_catalog_worker_guarded`]. +/// +/// `sessions_root` is the directory containing per-Session directories. Invoke this +/// inside the Host's `with_recording_boundary` callback, passing **its existing guard +/// and persisted active Segment**. That same external retention control store must +/// fence publishers, pointer changes, archive moves and erasure. Do not reacquire the +/// guard or use a separately opened legacy public reader inside the boundary. +/// All descriptor-only, allocation and callback guarantees of the Worker variant +/// apply; this function does not construct an unbounded transient `CaptureDto`. +pub fn recording_catalog_fs_guarded( + sessions_root: &Path, + session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + guard: &crate::retention::SessionRetentionGuard, + inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T, +) -> Result { + if guard.session_id() != session { + return Err(PublicMaterialError::Inconsistent); + } + let replay = CommittedPublicReplay::open_fs_guarded( + sessions_root, + session, + active_segment, + replay_limits, + )?; + Ok(inspect(&replay.into_recording_descriptors())) +} + +/// Capture the selected original lineage, including typed Host run boundaries. +/// The caller must hold the Session publication/erasure guard while selecting the +/// active pointer and through this call. Runtime publishers use this same lock +/// for appends and Segment creation. No lock is recursively acquired here. +pub fn capture_recording_worker_guarded( + session_root: &Path, + expected_session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + capture_limits: CaptureLimits, + guard: &crate::retention::SessionRetentionGuard, +) -> Result { + if guard.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + let replay = CommittedPublicReplay::open_worker_guarded( + session_root, + active_segment, + replay_limits, + None, + PublicMaterialUse::Recording, + )?; + if replay.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + replay.into_recording_dto(capture_limits) +} + +/// Standalone convenience entry; Hosts resolving a mutable active pointer should +/// instead hold the fence while resolving it and call the guarded entry. +pub fn capture_recording_worker( + session_root: &Path, + expected_session: SessionId, + active_segment: SegmentId, + retention: &SessionRetentionStore, + replay_limits: PublicMaterialLimits, + capture_limits: CaptureLimits, +) -> Result { + let guard = retention + .lock_live(expected_session) + .map_err(|error| match error { + crate::StoreError::SessionErased(_) => PublicMaterialError::Erased, + _ => PublicMaterialError::Unavailable, + })?; + capture_recording_worker_guarded( + session_root, + expected_session, + active_segment, + replay_limits, + capture_limits, + &guard, + ) +} + +/// Bounded construction for one Job input: descriptor replay is bounded separately; +/// only at most the requested body budget is materialized from the fixed files. +pub fn capture_recording_slice_worker_guarded( + session_root: &Path, + expected_session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + start: RecordingPosition, + max_entries: usize, + max_serialized_bytes: usize, + guard: &crate::retention::SessionRetentionGuard, +) -> Result { + if guard.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + let replay = CommittedPublicReplay::open_worker_guarded( + session_root, + active_segment, + replay_limits, + None, + PublicMaterialUse::Recording, + )?; + if replay.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + replay.recording_slice(start, max_entries, max_serialized_bytes) +} + +/// The standalone FsStore layout uses the same committed replay and fixed values. +pub fn capture_recording_fs_guarded( + sessions_root: &Path, + session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + capture_limits: CaptureLimits, + guard: &crate::retention::SessionRetentionGuard, +) -> Result { + if guard.session_id() != session { + return Err(PublicMaterialError::Inconsistent); + } + CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)? + .into_recording_dto(capture_limits) +} + +/// Bounded standalone capture; the caller holds its FsStore retention fence. +pub fn capture_recording_slice_fs_guarded( + sessions_root: &Path, + session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + start: RecordingPosition, + max_entries: usize, + max_serialized_bytes: usize, + guard: &crate::retention::SessionRetentionGuard, +) -> Result { + if guard.session_id() != session { + return Err(PublicMaterialError::Inconsistent); + } + CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)? + .recording_slice(start, max_entries, max_serialized_bytes) +} + +impl CaptureDto { + /// Select the oldest canonical prefix fitting both budgets, including JSON + /// escaping, metadata and continuation. Only immutable owned bytes are read. + /// A later Job can recapture at `next`; never combine live pages into this value. + pub fn recording_slice( + &self, + start: RecordingPosition, + max_entries: usize, + max_serialized_bytes: usize, + ) -> Result { + if max_entries == 0 { + return Err(CaptureError::BudgetTooSmall); + } + if start.canonical_order > self.view_entry_count { + return Err(CaptureError::InvalidRange); + } + let mut result = RecordingCaptureSlice { + capture: CaptureDto { + version: self.version, + session_id: self.session_id.clone(), + segment_id: self.segment_id.clone(), + view_entry_count: self.view_entry_count, + entries: Vec::new(), + }, + next: (start.canonical_order < self.view_entry_count).then_some(start), + }; + if body::json_size(&result, max_serialized_bytes).is_err() { + return Err(CaptureError::BudgetTooSmall); + } + if start.canonical_order == self.view_entry_count { + if start.byte_offset != 0 { + return Err(CaptureError::InvalidRange); + } + result.capture.validate(CaptureLimits { + max_bytes: max_serialized_bytes, + max_text_bytes: max_serialized_bytes, + max_entries, + ..Default::default() + })?; + return Ok(result); + } + let first = self + .entries + .iter() + .position(|entry| entry.order == start.canonical_order) + .ok_or(CaptureError::InvalidRange)?; + let mut expected_order = start.canonical_order; + for entry in self.entries.iter().skip(first).take(max_entries) { + if entry.order != expected_order { + return Err(CaptureError::InvalidRange); + } + let CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + boundary, + } = &entry.body + else { + return Err(CaptureError::Invalid("not a recording capture")); + }; + if !entry.policy.recording_material { + return Err(CaptureError::PolicyDenied); + } + let offset = if entry.order == start.canonical_order { + start.byte_offset + } else { + *byte_offset + }; + let local = offset + .checked_sub(*byte_offset) + .and_then(|o| usize::try_from(o).ok()) + .ok_or(CaptureError::InvalidRange)?; + if local > text.len() + || !text.is_char_boundary(local) + || offset >= descriptor.total_bytes + { + return Err(CaptureError::InvalidRange); + } + let item = CaptureEntryDto { + entry_id: entry.entry_id.clone(), + order: entry.order, + source: entry.source.clone(), + policy: entry.policy, + body: CaptureBody::CanonicalPublic { + descriptor: descriptor.clone(), + byte_offset: offset, + text: String::new(), + boundary: *boundary, + }, + }; + result.capture.entries.push(item); + // At most budget bytes are considered before allocating a trial fragment. + let mut low = 0; + let mut high = text.len().saturating_sub(local).min(max_serialized_bytes); + let prior_next = result.next; + while low < high { + let mid = low + (high - low).div_ceil(2); + let mut end = local + mid; + while !text.is_char_boundary(end) { + end -= 1; + } + if let CaptureBody::CanonicalPublic { text: fragment, .. } = + &mut result.capture.entries.last_mut().unwrap().body + { + *fragment = text[local..end].to_owned(); + } + let next_offset = offset + (end - local) as u64; + result.next = if next_offset < descriptor.total_bytes { + Some(RecordingPosition { + canonical_order: entry.order, + byte_offset: next_offset, + }) + } else { + (entry.order + 1 < self.view_entry_count).then_some(RecordingPosition { + canonical_order: entry.order + 1, + byte_offset: 0, + }) + }; + if body::json_size(&result, max_serialized_bytes).is_ok() { + low = mid; + } else { + high = mid - 1; + } + } + let mut end = local + low; + while !text.is_char_boundary(end) { + end -= 1; + } + if end == local { + result.capture.entries.pop(); + result.next = prior_next; + if result.capture.entries.is_empty() { + return Err(CaptureError::BudgetTooSmall); + } + break; + } + if let CaptureBody::CanonicalPublic { text: fragment, .. } = + &mut result.capture.entries.last_mut().unwrap().body + { + *fragment = text[local..end].to_owned(); + } + let next_offset = offset + (end - local) as u64; + result.next = if next_offset < descriptor.total_bytes { + Some(RecordingPosition { + canonical_order: entry.order, + byte_offset: next_offset, + }) + } else { + (entry.order + 1 < self.view_entry_count).then_some(RecordingPosition { + canonical_order: entry.order + 1, + byte_offset: 0, + }) + }; + if next_offset < descriptor.total_bytes { + break; + } + expected_order = entry.order + 1; + } + body::json_size(&result, max_serialized_bytes).map_err(|_| CaptureError::BudgetTooSmall)?; + result.capture.validate(CaptureLimits { + max_bytes: max_serialized_bytes, + max_text_bytes: max_serialized_bytes, + max_entries, + ..Default::default() + })?; + Ok(result) + } + + /// Resolve canonical text already in the value. No original storage is consulted. + pub fn recording_text( + &self, + order: u64, + ) -> Result<(&crate::public_material::PublicMaterialDescriptor, u64, &str), CaptureError> { + let entry = self + .entries + .iter() + .find(|entry| entry.order == order) + .ok_or(CaptureError::UnknownReference)?; + if !entry.policy.recording_material { + return Err(CaptureError::PolicyDenied); + } + match &entry.body { + CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + .. + } => Ok((descriptor, *byte_offset, text)), + _ => Err(CaptureError::Invalid("not canonical recording text")), + } + } +} diff --git a/crates/session-store/src/committed_replay.rs b/crates/session-store/src/committed_replay.rs new file mode 100644 index 000000000..051096cca --- /dev/null +++ b/crates/session-store/src/committed_replay.rs @@ -0,0 +1,1346 @@ +//! Original public material from a saved, rollback-ineligible Session prefix. +//! +//! Unlike the Memory evidence index, this reader includes saved output from +//! interrupted runs and active runs with materialized AI activity. It does not +//! interpret domain-specific recording or completion policy. Callers authorize +//! the Session root and explicitly select its active Segment. +//! +//! Catalogs retain only descriptors and record locations, never original bodies. +//! Records are decoded one at a time under an explicit allocation ceiling, not +//! under a whole-run scan budget. Canonical bodies are streamed to SHA-256 and +//! bounded UTF-8 windows. A single JSONL record (including a Segment seed) must +//! fit `max_record_bytes`; exceeding that ceiling is an error, not a preview. + +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::{ + LogEntry, LoggedHistoryEntry, LoggedItem, LoggedRole, LoggedSessionHistoryOrigin, SegmentId, + SegmentOrigin, SessionId, WorkerSessionStore, +}; + +#[derive(Debug, thiserror::Error)] +pub enum PublicMaterialError { + #[error("public material storage: {0}")] + Storage(#[from] io::Error), + #[error("public material record: {0}")] + Record(#[from] serde_json::Error), + #[error("public material source is missing")] + Missing, + #[error("public material source was permanently erased")] + Erased, + #[error("public material source requires migration or is unavailable")] + Unavailable, + #[error("public material lineage or identity is inconsistent")] + Inconsistent, + #[error("public material resource ceiling exceeded")] + ResourceLimit, + #[error("public material changed since the catalog was read")] + Changed, + #[error("public material offset is not a UTF-8 boundary or is outside the body")] + InvalidOffset, +} + +#[derive(Debug, Clone, Copy)] +pub struct PublicMaterialLimits { + pub max_segments: usize, + /// Total history identities examined, including inherited and private items. + pub max_entries: usize, + /// Allocation ceiling for one persisted JSONL record, not an entire run. + pub max_record_bytes: u64, + /// Aggregate fixed storage extent, bounding replay metadata as well as body decoding. + pub max_storage_bytes: u64, +} + +impl Default for PublicMaterialLimits { + fn default() -> Self { + Self { + max_segments: 256, + max_entries: 1_000_000, + max_record_bytes: 64 * 1024 * 1024, + max_storage_bytes: 256 * 1024 * 1024, + } + } +} + +/// Original identity, never a body-based deduplication key. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicMaterialIdentity { + pub segment_id: SegmentId, + pub entry_ref: String, +} + +/// Public provenance distinguishes saved conversation from Host control facts. Run boundaries +/// are not model output, user intent, or evidence that the requested work succeeded. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum PublicMaterialOrigin { + History { origin: LoggedSessionHistoryOrigin }, + HostRunBoundary, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PublicRunOutcome { + Finished, + Paused, + Yielded, + LimitReached, +} + +/// Only whitelisted execution facts cross this boundary; raw error/provider messages do not. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum PublicRunBoundary { + Completed { + outcome: PublicRunOutcome, + interrupted: bool, + }, + Yielded, + Resumed, + Cancelled, + Errored, + Abandoned, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicMaterialDescriptor { + pub identity: PublicMaterialIdentity, + /// Zero-based position on this explicitly selected active lineage. Forks + /// may remove a suffix; callers must reconcile identities, not assume that + /// positions from a previous active lineage are interchangeable. + pub canonical_order: u64, + pub origin: PublicMaterialOrigin, + /// Lowercase hexadecimal SHA-256 of the complete canonical UTF-8 body. + pub sha256: String, + pub total_bytes: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicMaterialContent { + pub content: String, + pub next_offset: u64, + pub total_bytes: u64, + pub sha256: String, +} + +#[derive(Clone)] +struct Location { + segment: SegmentId, + offset: u64, + // Index within this physical record (e.g. a Segment seed), used only for I/O. + history_index: usize, +} + +struct Material { + descriptor: PublicMaterialDescriptor, + location: Location, + // Original segment-history row, or the next-history gap for a Host boundary. + // Never overwritten when a correction moves the physical body location. + source_history_position: u64, + is_tool_result: bool, + boundary: Option, +} + +struct SegmentHeader { + id: SegmentId, + parent: Option, + compact: bool, + // Compaction's exact source-history IDs disambiguate mid-turn boundaries. + source_ids: Option>, + file_len: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PublicMaterialUse { + Recording, + MemoryEvidence, +} + +/// A fixed material catalog. Opening a new catalog observes newly appended +/// records; reading a body through this catalog verifies its saved digest and +/// byte length again. Missing/corrected/deleted sources are explicit failures. +/// Evidence eligibility never publishes private model text or Feature payload: +/// System inputs also require an explicit public projection for every public use. +pub struct CommittedPublicReplay { + session_id: SessionId, + active_segment_id: SegmentId, + layout: Layout, + limits: PublicMaterialLimits, + materials: Vec, + purpose: PublicMaterialUse, + file_ends: HashMap, + retention: Option, +} + +enum Layout { + Worker(PathBuf), + Fs(PathBuf), +} + +impl Layout { + fn path(&self, segment: SegmentId) -> PathBuf { + match self { + Self::Worker(root) => root.join("segments").join(format!("{segment}.jsonl")), + Self::Fs(root) => root.join(format!("{segment}.jsonl")), + } + } +} + +impl CommittedPublicReplay { + pub fn open_worker( + session_root: &Path, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + ) -> Result { + Self::open_worker_guarded( + session_root, + active_segment_id, + limits, + None, + PublicMaterialUse::Recording, + ) + } + + /// Uses the Host's aggregate-external control root for the whole scan and every body read. + /// The same store must fence publishers, archive transfer, and physical deletion. + pub fn open_worker_with_retention( + session_root: &Path, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: crate::retention::SessionRetentionStore, + ) -> Result { + Self::open_worker_for_use( + session_root, + active_segment_id, + limits, + retention, + PublicMaterialUse::Recording, + ) + } + + /// Selects the evidence purpose independently of public display and observation. + pub fn open_worker_for_use( + session_root: &Path, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: crate::retention::SessionRetentionStore, + purpose: PublicMaterialUse, + ) -> Result { + Self::open_worker_guarded( + session_root, + active_segment_id, + limits, + Some(retention), + purpose, + ) + } + + pub(crate) fn open_worker_guarded( + session_root: &Path, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: Option, + purpose: PublicMaterialUse, + ) -> Result { + let store = WorkerSessionStore::open_read_only(session_root) + .map_err(|_| PublicMaterialError::Unavailable)?; + let session_id = store + .session_id() + .map_err(|_| PublicMaterialError::Unavailable)? + .ok_or(PublicMaterialError::Missing)?; + Self::open( + session_id, + active_segment_id, + Layout::Worker(session_root.into()), + limits, + retention, + purpose, + ) + } + + pub fn open_fs( + sessions_root: &Path, + session_id: SessionId, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + ) -> Result { + Self::open_fs_with_retention( + sessions_root, + session_id, + active_segment_id, + limits, + crate::retention::fs_retention(sessions_root), + ) + } + + /// Keeps the same erasure fence when a Host relocates originals to retained storage. + pub fn open_fs_with_retention( + sessions_root: &Path, + session_id: SessionId, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: crate::retention::SessionRetentionStore, + ) -> Result { + Self::open_fs_for_use( + sessions_root, + session_id, + active_segment_id, + limits, + retention, + PublicMaterialUse::Recording, + ) + } + + /// Uses retained originals with the independently selected evidence policy. + pub fn open_fs_for_use( + sessions_root: &Path, + session_id: SessionId, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: crate::retention::SessionRetentionStore, + purpose: PublicMaterialUse, + ) -> Result { + Self::open( + session_id, + active_segment_id, + Layout::Fs(sessions_root.join(session_id.to_string())), + limits, + Some(retention), + purpose, + ) + } + + pub(crate) fn open_fs_guarded( + sessions_root: &Path, + session: SessionId, + active: SegmentId, + limits: PublicMaterialLimits, + ) -> Result { + Self::open( + session, + active, + Layout::Fs(sessions_root.join(session.to_string())), + limits, + None, + PublicMaterialUse::Recording, + ) + } + + pub fn session_id(&self) -> SessionId { + self.session_id + } + pub fn active_segment_id(&self) -> SegmentId { + self.active_segment_id + } + pub fn descriptors(&self) -> impl ExactSizeIterator { + self.materials.iter().map(|value| &value.descriptor) + } + + /// Consume replay metadata for guarded recording admission, without reading or + /// collecting original bodies. The public capture facade exposes only these + /// descriptors to its callback, never this replay's body-read capability. + pub(crate) fn into_recording_descriptors(self) -> Vec { + self.materials + .into_iter() + .map(|value| value.descriptor) + .collect() + } + + fn open( + session_id: SessionId, + active_segment_id: SegmentId, + layout: Layout, + limits: PublicMaterialLimits, + retention: Option, + purpose: PublicMaterialUse, + ) -> Result { + let _guard = retention + .as_ref() + .map(|store| store.lock_live(session_id).map_err(retention_error)) + .transpose()?; + if limits.max_segments == 0 || limits.max_entries == 0 || limits.max_record_bytes == 0 { + return Err(PublicMaterialError::ResourceLimit); + } + // Freeze every file end before decoding even a Segment seed. Publishers + // hold the same Session publication lock; replay never refreshes this map. + let directory = match &layout { + Layout::Worker(root) => root.join("segments"), + Layout::Fs(root) => root.clone(), + }; + let mut file_ends = HashMap::new(); + let mut fixed_bytes = 0u64; + for entry in std::fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) != Some("jsonl") { + continue; + } + let Some(id) = path + .file_stem() + .and_then(|s| s.to_str()) + .and_then(|s| s.parse::().ok()) + else { + continue; + }; + if file_ends.len() >= limits.max_segments { + return Err(PublicMaterialError::ResourceLimit); + } + let reader = RecordReader::open(&path, limits.max_record_bytes)?; + // This also bounds retained descriptor/identity allocations during replay, + // not just the final capture. Oversized histories fail explicitly. + fixed_bytes = fixed_bytes + .checked_add(reader.end) + .ok_or(PublicMaterialError::ResourceLimit)?; + if fixed_bytes > limits.max_storage_bytes { + return Err(PublicMaterialError::ResourceLimit); + } + file_ends.insert(id, reader.end); + } + let mut chain = Vec::new(); + let mut visited = HashSet::new(); + let mut current = active_segment_id; + let mut header_entries = 0usize; + loop { + if !visited.insert(current) { + return Err(PublicMaterialError::Inconsistent); + } + if visited.len() > limits.max_segments { + return Err(PublicMaterialError::ResourceLimit); + } + let mut reader = RecordReader::open(&layout.path(current), limits.max_record_bytes)?; + let file_len = *file_ends + .get(¤t) + .ok_or(PublicMaterialError::Missing)?; + reader.end = file_len; + let Some(( + _, + LogEntry::AnnotatedSegmentStart { + session_id: stored, + forked_from, + compacted_from, + history, + .. + }, + )) = reader.next()? + else { + return Err(PublicMaterialError::Inconsistent); + }; + if stored != session_id || (forked_from.is_some() && compacted_from.is_some()) { + return Err(PublicMaterialError::Inconsistent); + } + header_entries = header_entries + .checked_add(history.len()) + .ok_or(PublicMaterialError::ResourceLimit)?; + if header_entries > limits.max_entries { + return Err(PublicMaterialError::ResourceLimit); + } + let compact = compacted_from.is_some(); + let source_ids: Option> = if compact { + let summaries: Vec<_> = history + .iter() + .filter(|entry| { + entry.metadata.input_metadata.as_ref().is_some_and(|input| { + input.usage().compaction == crate::InputCompactionUse::OpeningSummary + }) + }) + .collect(); + if summaries.len() != 1 { + return Err(PublicMaterialError::Inconsistent); + } + Some( + summaries[0] + .metadata + .derivation + .as_ref() + .ok_or(PublicMaterialError::Inconsistent)? + .sources + .iter() + .map(|id| id.0.clone()) + .collect(), + ) + } else { + None + }; + header_entries = header_entries + .checked_add(source_ids.as_ref().map_or(0, |ids| ids.len())) + .ok_or(PublicMaterialError::ResourceLimit)?; + if header_entries > limits.max_entries { + return Err(PublicMaterialError::ResourceLimit); + } + let parent = forked_from.or(compacted_from); + let next = parent.as_ref().map(|p| p.segment_id); + chain.push(SegmentHeader { + id: current, + parent, + compact, + source_ids, + file_len, + }); + match next { + Some(id) => current = id, + None => break, + } + } + chain.reverse(); + let mut materials = Vec::::new(); + let mut identities = HashMap::::new(); + let mut examined = 0usize; + let mut expected_fork_seed: Option> = None; + for (segment_index, header) in chain.iter().enumerate() { + let child = chain.get(segment_index + 1); + let mut reader = RecordReader::open(&layout.path(header.id), limits.max_record_bytes)?; + reader.end = header.file_len; + let mut history_ids = Vec::::new(); + let mut history_seen = HashSet::::new(); + let mut inherited_order = None; + let mut pending = Vec::::new(); + let mut in_run = false; + let mut safe = true; + let mut turn = 0usize; + let mut reached_cut = child.is_none(); + let mut compact_cut_turn_seen = false; + while let Some((offset, record)) = reader.next()? { + // Once a compaction's exact source history is reached, consume + // only non-history boundaries/corrections. A later history + // addition belongs to the parent's unadopted suffix. + if reached_cut + && child.is_some_and(|c| c.compact) + && matches!( + record, + LogEntry::AnnotatedUserInput { .. } + | LogEntry::AnnotatedAssistantItem { .. } + | LogEntry::AnnotatedToolResult { .. } + | LogEntry::AnnotatedSystemItem { .. } + ) + { + break; + } + let seed = matches!(record, LogEntry::AnnotatedSegmentStart { .. }); + if seed { + if let LogEntry::AnnotatedSegmentStart { + history, + session_id: stored, + .. + } = &record + { + if *stored != session_id { + return Err(PublicMaterialError::Inconsistent); + } + if let Some(expected) = expected_fork_seed.take() { + if history + .iter() + .map(|e| &e.metadata.entry_id.0) + .ne(expected.iter()) + { + return Err(PublicMaterialError::Inconsistent); + } + } + } + } + if offset != 0 && seed { + return Err(PublicMaterialError::Inconsistent); + } + match &record { + LogEntry::Invoke { .. } | LogEntry::RunResumed { .. } => { + if !pending.is_empty() { + return Err(PublicMaterialError::Inconsistent); + } + in_run = true; + safe = false; + } + LogEntry::RunCompleted { .. } + | LogEntry::RunYielded { .. } + | LogEntry::RunCancelled { .. } + | LogEntry::RunErrored { .. } + | LogEntry::PausedTurnAbandoned { .. } => { + safe = true; + in_run = false; + } + LogEntry::TurnEnd { turn_count, .. } => turn = *turn_count, + LogEntry::ActiveRunCheckpoint { + total_turn_count, .. + } => turn = *total_turn_count, + _ => {} + } + let system_history; + let entries: &[LoggedHistoryEntry] = match &record { + LogEntry::AnnotatedSegmentStart { history, .. } + | LogEntry::AnnotatedUserInput { history, .. } => history, + LogEntry::AnnotatedAssistantItem { entry, .. } + | LogEntry::AnnotatedToolResult { entry, .. } + | LogEntry::ToolResultCorrected { entry, .. } => std::slice::from_ref(entry), + LogEntry::AnnotatedSystemItem { entry, .. } => { + system_history = entry.to_history_entry(); + std::slice::from_ref(&system_history) + } + _ => &[], + }; + for (history_index, entry) in entries.iter().enumerate() { + charge(&mut examined, limits.max_entries)?; + let id = &entry.metadata.entry_id.0; + if id.is_empty() { + return Err(PublicMaterialError::Inconsistent); + } + let corrected = matches!(record, LogEntry::ToolResultCorrected { .. }); + if !corrected { + if !history_seen.insert(id.clone()) { + return Err(PublicMaterialError::Inconsistent); + } + history_ids.push(id.clone()); + } + if in_run && ai_materialized(&entry.item) { + safe = true; + } + let Some((sha256, total_bytes)) = canonical_metadata(entry, purpose)? else { + continue; + }; + let location = Location { + segment: header.id, + offset, + history_index, + }; + let descriptor = PublicMaterialDescriptor { + identity: PublicMaterialIdentity { + segment_id: header.id, + entry_ref: format!("E{id}"), + }, + canonical_order: 0, + origin: PublicMaterialOrigin::History { + origin: entry.metadata.origin.clone(), + }, + sha256, + total_bytes, + }; + if seed && header.parent.is_some() { + // Every inherited original must already exist on the + // selected ancestor prefix; compare identity + bytes. + let index = identities + .get(id) + .ok_or(PublicMaterialError::Inconsistent)?; + if inherited_order.is_some_and(|previous| *index <= previous) { + return Err(PublicMaterialError::Inconsistent); + } + inherited_order = Some(*index); + let original = &materials[*index].descriptor; + if original.sha256 != descriptor.sha256 + || original.total_bytes != descriptor.total_bytes + || original.origin != descriptor.origin + { + return Err(PublicMaterialError::Inconsistent); + } + } else if corrected { + let index = identities + .get(id) + .ok_or(PublicMaterialError::Inconsistent)?; + let original = &mut materials[*index]; + if !original.is_tool_result + || original.descriptor.origin != descriptor.origin + || !matches!(entry.item, LoggedItem::ToolResult { .. }) + { + return Err(PublicMaterialError::Inconsistent); + } + original.descriptor.sha256 = descriptor.sha256; + original.descriptor.total_bytes = descriptor.total_bytes; + original.location = location; + } else { + if identities.contains_key(id) + || pending.iter().any(|m| { + m.descriptor.identity.entry_ref == descriptor.identity.entry_ref + }) + { + return Err(PublicMaterialError::Inconsistent); + } + pending.push(Material { + descriptor, + location, + source_history_position: (history_ids.len() - 1) as u64, + is_tool_result: matches!(entry.item, LoggedItem::ToolResult { .. }), + boundary: None, + }); + } + } + if let Some(boundary) = run_boundary(&record) { + charge(&mut examined, limits.max_entries)?; + let reference = boundary_reference(&record, session_id, header.id, offset); + if identities.contains_key(&reference[1..]) + || pending + .iter() + .any(|m| m.descriptor.identity.entry_ref == reference) + { + return Err(PublicMaterialError::Inconsistent); + } + let mut sink = BodySink::new(0, 0); + serde_json::to_writer(&mut sink, &CanonicalBody::RunBoundary { boundary })?; + let (sha256, total_bytes, _) = sink.finish()?; + pending.push(Material { + descriptor: PublicMaterialDescriptor { + identity: PublicMaterialIdentity { + segment_id: header.id, + entry_ref: reference, + }, + canonical_order: 0, + origin: PublicMaterialOrigin::HostRunBoundary, + sha256, + total_bytes, + }, + location: Location { + segment: header.id, + offset, + history_index: 0, + }, + source_history_position: history_ids.len() as u64, + is_tool_result: false, + boundary: Some(boundary), + }); + } + if safe { + flush(&mut materials, &mut identities, &mut pending); + } + if let Some(child) = child { + if child.compact { + let sources = child + .source_ids + .as_ref() + .ok_or(PublicMaterialError::Inconsistent)?; + if &history_ids == sources { + // Do not stop at an earlier non-history record when + // compaction's source-history is empty. + if !sources.is_empty() || seed { + reached_cut = true; + if child + .parent + .as_ref() + .is_some_and(|p| p.at_turn_index == turn) + { + compact_cut_turn_seen = true; + } + } + } + } else if (seed && child.parent.as_ref().is_some_and(|p| p.at_turn_index == 0)) + || (turn + == child + .parent + .as_ref() + .ok_or(PublicMaterialError::Inconsistent)? + .at_turn_index + && matches!(record, LogEntry::TurnEnd { .. })) + { + reached_cut = true; + break; + } + } + } + if !reached_cut || (child.is_some_and(|c| c.compact) && !compact_cut_turn_seen) { + return Err(PublicMaterialError::Inconsistent); + } + if child.is_some_and(|c| !c.compact) { + expected_fork_seed = Some(history_ids); + } + // A compaction seed proves its selected input is durably retained; + // still require materialized AI or a terminal boundary for a run. + if !pending.is_empty() && child.is_some() { + return Err(PublicMaterialError::Inconsistent); + } + } + Ok(Self { + session_id, + active_segment_id, + layout, + limits, + materials, + purpose, + file_ends, + retention: retention.clone(), + }) + } + + /// Build a portable immutable value under the caller's publication/erasure fence. + /// Preflight the aggregate original-body allocation; never truncate an over-limit history. + pub(crate) fn into_recording_dto( + self, + limits: crate::capture::CaptureLimits, + ) -> Result { + use crate::capture::*; + let mut total = 0usize; + if self.materials.len() > limits.max_entries { + return Err(PublicMaterialError::ResourceLimit); + } + for material in &self.materials { + let len = usize::try_from(material.descriptor.total_bytes) + .map_err(|_| PublicMaterialError::ResourceLimit)?; + total = total + .checked_add(len) + .ok_or(PublicMaterialError::ResourceLimit)?; + if len > limits.max_text_bytes || total > limits.max_bytes { + return Err(PublicMaterialError::ResourceLimit); + } + } + let entries = self + .materials + .iter() + .map(|material| { + self.recording_entry( + material, + 0, + usize::try_from(material.descriptor.total_bytes).unwrap_or(usize::MAX), + ) + }) + .collect::, _>>()?; + let dto = CaptureDto { + version: CAPTURE_VERSION, + session_id: self.session_id.to_string(), + segment_id: self.active_segment_id.to_string(), + view_entry_count: self.materials.len() as u64, + entries, + }; + dto.validate(limits) + .map_err(|_| PublicMaterialError::ResourceLimit)?; + Ok(dto) + } + + fn recording_entry( + &self, + material: &Material, + offset: u64, + max_bytes: usize, + ) -> Result { + use crate::capture::*; + let d = &material.descriptor; + let content = self.read_content(d, offset, max_bytes.max(4))?; + let history_origin = match &d.origin { + PublicMaterialOrigin::History { origin } => Some(origin.clone()), + PublicMaterialOrigin::HostRunBoundary => None, + }; + Ok(CaptureEntryDto { + entry_id: d + .identity + .entry_ref + .strip_prefix('E') + .ok_or(PublicMaterialError::Inconsistent)? + .to_owned(), + order: d.canonical_order, + source: CaptureSource { + segment_id: d.identity.segment_id.to_string(), + entry_index: material.source_history_position, + provenance: history_origin + .as_ref() + .map(crate::capture::public_provenance) + .unwrap_or(protocol::SessionEntryProvenance::LegacyUnknown), + history_origin, + derived_from: Vec::new(), + }, + // Recording selection is independent of observation and Memory evidence. + policy: CaptureUsePolicy { + recording_material: true, + memory_evidence: false, + observation_readable: false, + }, + body: CaptureBody::CanonicalPublic { + descriptor: d.clone(), + byte_offset: offset, + text: content.content, + boundary: material.boundary, + }, + }) + } + + /// Capture only a bounded body prefix, after the same full-lineage committed replay. + /// A long Tool result never requires allocating its full canonical rendering. + pub(crate) fn recording_slice( + &self, + start: crate::capture::RecordingPosition, + max_entries: usize, + max_bytes: usize, + ) -> Result { + use crate::capture::*; + if max_entries == 0 || max_bytes < 4 { + return Err(PublicMaterialError::ResourceLimit); + } + let first = usize::try_from(start.canonical_order) + .map_err(|_| PublicMaterialError::InvalidOffset)?; + if first > self.materials.len() || (first == self.materials.len() && start.byte_offset != 0) + { + return Err(PublicMaterialError::InvalidOffset); + } + let mut entries = Vec::new(); + let mut remaining = max_bytes; + for material in self.materials.iter().skip(first).take(max_entries) { + if remaining < 4 { + break; + } + let offset = if material.descriptor.canonical_order == start.canonical_order { + start.byte_offset + } else { + 0 + }; + let entry = self.recording_entry(material, offset, remaining)?; + let CaptureBody::CanonicalPublic { text, .. } = &entry.body else { + unreachable!() + }; + remaining -= text.len(); + let partial = offset + (text.len() as u64) < material.descriptor.total_bytes; + entries.push(entry); + if partial { + break; + } + } + let dto = CaptureDto { + version: CAPTURE_VERSION, + session_id: self.session_id.to_string(), + segment_id: self.active_segment_id.to_string(), + view_entry_count: self.materials.len() as u64, + entries, + }; + dto.recording_slice(start, max_entries, max_bytes) + .map_err(|error| match error { + CaptureError::InvalidRange => PublicMaterialError::InvalidOffset, + _ => PublicMaterialError::ResourceLimit, + }) + } + + /// Read a canonical body window without allocating its full rendered text. + /// `max_content_bytes >= 4` guarantees progress for every UTF-8 scalar. + pub fn read_content( + &self, + descriptor: &PublicMaterialDescriptor, + offset: u64, + max_content_bytes: usize, + ) -> Result { + let _guard = self + .retention + .as_ref() + .map(|store| store.lock_live(self.session_id).map_err(retention_error)) + .transpose()?; + if max_content_bytes < 4 { + return Err(PublicMaterialError::ResourceLimit); + } + if offset > descriptor.total_bytes { + return Err(PublicMaterialError::InvalidOffset); + } + let material = self + .materials + .get( + usize::try_from(descriptor.canonical_order) + .map_err(|_| PublicMaterialError::Changed)?, + ) + .filter(|m| &m.descriptor == descriptor) + .ok_or(PublicMaterialError::Changed)?; + let mut reader = RecordReader::open( + &self.layout.path(material.location.segment), + self.limits.max_record_bytes, + )?; + reader.end = *self + .file_ends + .get(&material.location.segment) + .ok_or(PublicMaterialError::Missing)?; + reader.seek(material.location.offset)?; + let Some((_, record)) = reader.next()? else { + return Err(PublicMaterialError::Missing); + }; + let system_history; + let (body, origin) = if let Some(boundary) = run_boundary(&record) { + if boundary_reference( + &record, + self.session_id, + material.location.segment, + material.location.offset, + ) != descriptor.identity.entry_ref + { + return Err(PublicMaterialError::Changed); + } + ( + CanonicalBody::RunBoundary { boundary }, + PublicMaterialOrigin::HostRunBoundary, + ) + } else { + let entry = match &record { + LogEntry::AnnotatedSegmentStart { history, .. } + | LogEntry::AnnotatedUserInput { history, .. } => { + history.get(material.location.history_index) + } + LogEntry::AnnotatedAssistantItem { entry, .. } + | LogEntry::AnnotatedToolResult { entry, .. } + | LogEntry::ToolResultCorrected { entry, .. } => Some(entry), + LogEntry::AnnotatedSystemItem { entry, .. } => { + system_history = entry.to_history_entry(); + Some(&system_history) + } + _ => None, + } + .filter(|e| format!("E{}", e.metadata.entry_id.0) == descriptor.identity.entry_ref) + .ok_or(PublicMaterialError::Changed)?; + ( + canonical_body(entry, self.purpose).ok_or(PublicMaterialError::Changed)?, + PublicMaterialOrigin::History { + origin: entry.metadata.origin.clone(), + }, + ) + }; + let mut sink = BodySink::new(offset, max_content_bytes); + serde_json::to_writer(&mut sink, &body)?; + let (sha256, total_bytes, content) = sink.finish()?; + if sha256 != descriptor.sha256 + || total_bytes != descriptor.total_bytes + || origin != descriptor.origin + { + return Err(PublicMaterialError::Changed); + } + Ok(PublicMaterialContent { + next_offset: offset + content.len() as u64, + content, + total_bytes, + sha256, + }) + } +} + +fn retention_error(error: crate::StoreError) -> PublicMaterialError { + match error { + crate::StoreError::SessionErased(_) => PublicMaterialError::Erased, + crate::StoreError::Io(error) => PublicMaterialError::Storage(error), + crate::StoreError::Serde(error) => PublicMaterialError::Record(error), + _ => PublicMaterialError::Unavailable, + } +} + +fn charge(examined: &mut usize, max: usize) -> Result<(), PublicMaterialError> { + *examined = examined + .checked_add(1) + .ok_or(PublicMaterialError::ResourceLimit)?; + if *examined > max { + return Err(PublicMaterialError::ResourceLimit); + } + Ok(()) +} + +fn flush( + materials: &mut Vec, + identities: &mut HashMap, + pending: &mut Vec, +) { + for mut material in pending.drain(..) { + let index = materials.len(); + material.descriptor.canonical_order = index as u64; + identities.insert(material.descriptor.identity.entry_ref[1..].into(), index); + materials.push(material); + } +} + +fn ai_materialized(item: &LoggedItem) -> bool { + !matches!( + item, + LoggedItem::Message { + role: LoggedRole::User | LoggedRole::System, + .. + } + ) +} + +/// Versioned canonical JSON framing. This is original text, not a display +/// preview: both result summary and optional full content remain distinguishable. +#[derive(Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum CanonicalBody<'a> { + RunBoundary { + boundary: PublicRunBoundary, + }, + Message { + origin: &'a LoggedSessionHistoryOrigin, + role: LoggedRole, + content: std::borrow::Cow<'a, [crate::LoggedContentPart]>, + }, + ToolCall { + origin: &'a LoggedSessionHistoryOrigin, + call_id: &'a str, + name: &'a str, + arguments: &'a str, + }, + ToolResult { + origin: &'a LoggedSessionHistoryOrigin, + call_id: &'a str, + summary: &'a str, + content: &'a Option, + disposition: &'a agen::tool::ToolResultDisposition, + is_error: bool, + }, +} + +fn run_boundary(record: &LogEntry) -> Option { + Some(match record { + LogEntry::RunCompleted { + result, + interrupted, + .. + } => PublicRunBoundary::Completed { + outcome: match result { + agen::EngineResult::Finished => PublicRunOutcome::Finished, + agen::EngineResult::Paused => PublicRunOutcome::Paused, + agen::EngineResult::Yielded => PublicRunOutcome::Yielded, + agen::EngineResult::LimitReached => PublicRunOutcome::LimitReached, + }, + interrupted: *interrupted, + }, + LogEntry::RunYielded { .. } => PublicRunBoundary::Yielded, + LogEntry::RunResumed { .. } => PublicRunBoundary::Resumed, + LogEntry::RunCancelled { .. } => PublicRunBoundary::Cancelled, + LogEntry::RunErrored { .. } => PublicRunBoundary::Errored, + LogEntry::PausedTurnAbandoned { .. } => PublicRunBoundary::Abandoned, + _ => return None, + }) +} + +fn boundary_reference( + record: &LogEntry, + session: SessionId, + segment: SegmentId, + offset: u64, +) -> String { + let id = match record { + LogEntry::RunCompleted { entry_id, .. } + | LogEntry::PausedTurnAbandoned { entry_id, .. } + | LogEntry::RunYielded { entry_id, .. } + | LogEntry::RunResumed { entry_id, .. } + | LogEntry::RunCancelled { entry_id, .. } + | LogEntry::RunErrored { entry_id, .. } => entry_id.as_ref(), + _ => None, + }; + // Reuse the storage convention for legacy transitions, including the exact physical + // record position. New normal completions have no persisted transition ID either. + id.map(|id| format!("E{}", id.0)) + .unwrap_or_else(|| format!("El-run-{session}-{segment}-{offset}")) +} + +fn canonical_body( + entry: &LoggedHistoryEntry, + purpose: PublicMaterialUse, +) -> Option> { + if let Some(input) = &entry.metadata.input_metadata { + let usage = input.usage(); + let allowed = match purpose { + PublicMaterialUse::Recording => usage.recording_material, + PublicMaterialUse::MemoryEvidence => usage.memory_evidence, + }; + if allowed == crate::InputEvidenceUse::Exclude { + return None; + } + if let LoggedItem::Message { + role: LoggedRole::System, + .. + } = &entry.item + { + let public = input.public_item()?; + return Some(CanonicalBody::Message { + origin: &entry.metadata.origin, + role: LoggedRole::System, + content: std::borrow::Cow::Owned(vec![crate::LoggedContentPart::Text { + text: public.body.clone(), + }]), + }); + } + } + match &entry.item { + LoggedItem::Message { + role: LoggedRole::System, + .. + } + | LoggedItem::Reasoning { .. } + | LoggedItem::AssistantResponseBoundary { .. } => None, + LoggedItem::Message { role, content } => Some(CanonicalBody::Message { + origin: &entry.metadata.origin, + role: *role, + content: std::borrow::Cow::Borrowed(content), + }), + LoggedItem::ToolCall { + call_id, + name, + arguments, + .. + } => Some(CanonicalBody::ToolCall { + origin: &entry.metadata.origin, + call_id, + name, + arguments, + }), + LoggedItem::ToolResult { + call_id, + summary, + content, + disposition, + is_error, + .. + } => Some(CanonicalBody::ToolResult { + origin: &entry.metadata.origin, + call_id, + summary, + content, + disposition, + is_error: *is_error, + }), + } +} + +fn canonical_metadata( + entry: &LoggedHistoryEntry, + purpose: PublicMaterialUse, +) -> Result, PublicMaterialError> { + let Some(body) = canonical_body(entry, purpose) else { + return Ok(None); + }; + let mut sink = BodySink::new(0, 0); + serde_json::to_writer(&mut sink, &body)?; + Ok(Some((hex_digest(&sink.digest.finalize()), sink.total))) +} + +fn hex_digest(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +struct BodySink { + digest: Sha256, + total: u64, + offset: u64, + max: usize, + bytes: Vec, +} + +impl BodySink { + fn new(offset: u64, max: usize) -> Self { + Self { + digest: Sha256::new(), + total: 0, + offset, + max, + bytes: Vec::new(), + } + } + fn finish(mut self) -> Result<(String, u64, String), PublicMaterialError> { + // The final byte may land inside a UTF-8 scalar. Never include a + // partial scalar and never advance the caller over its missing bytes. + loop { + match std::str::from_utf8(&self.bytes) { + Ok(_) => break, + Err(e) if e.error_len().is_none() => { + self.bytes.truncate(e.valid_up_to()); + } + Err(_) => return Err(PublicMaterialError::InvalidOffset), + } + } + Ok(( + hex_digest(&self.digest.finalize()), + self.total, + String::from_utf8(self.bytes).map_err(|_| PublicMaterialError::InvalidOffset)?, + )) + } +} + +impl Write for BodySink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.digest.update(bytes); + let start = self + .offset + .saturating_sub(self.total) + .min(bytes.len() as u64) as usize; + let available = self.max.saturating_sub(self.bytes.len()); + let end = start.saturating_add(available).min(bytes.len()); + if end > start { + self.bytes.extend_from_slice(&bytes[start..end]); + } + self.total = self + .total + .checked_add(bytes.len() as u64) + .ok_or_else(|| io::Error::other("canonical body length overflow"))?; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +struct RecordReader { + reader: BufReader, + offset: u64, + end: u64, + max: u64, +} + +impl RecordReader { + fn open(path: &Path, max: u64) -> Result { + // Do not follow symlinks in a caller-selected storage root or a log. + let mut prefix = PathBuf::new(); + for component in path.components() { + prefix.push(component); + let metadata = std::fs::symlink_metadata(&prefix)?; + if metadata.file_type().is_symlink() { + return Err(PublicMaterialError::Unavailable); + } + } + let file = open_noatime(path)?; + if !file.metadata()?.is_file() { + return Err(PublicMaterialError::Unavailable); + } + let end = file.metadata()?.len(); + Ok(Self { + reader: BufReader::new(file), + offset: 0, + end, + max, + }) + } + fn seek(&mut self, offset: u64) -> Result<(), PublicMaterialError> { + self.reader.seek(SeekFrom::Start(offset))?; + self.offset = offset; + Ok(()) + } + fn next(&mut self) -> Result, PublicMaterialError> { + if self.offset >= self.end { + return Ok(None); + } + let mut bytes = Vec::new(); + let read = (&mut self.reader) + .take(self.max.saturating_add(1).min(self.end - self.offset)) + .read_until(b'\n', &mut bytes)? as u64; + if read > self.max { + return Err(PublicMaterialError::ResourceLimit); + } + if bytes.last() != Some(&b'\n') { + return Ok(None); + } // Uncommitted tail. + let offset = self.offset; + self.offset += read; + Ok(Some((offset, serde_json::from_slice(&bytes)?))) + } +} + +fn open_noatime(path: &Path) -> io::Result { + #[cfg(any(target_os = "linux", target_os = "android"))] + { + use std::os::unix::fs::OpenOptionsExt; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW) + .open(path) + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + { + let _ = path; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "no-atime reads unavailable", + )) + } +} + +#[cfg(test)] +#[path = "public_material/tests.rs"] +mod tests; + +/// Legacy descriptor-reader facade over the shared committed replay. +pub type SessionPublicMaterialReader = CommittedPublicReplay; diff --git a/crates/session-store/src/lib.rs b/crates/session-store/src/lib.rs index 15abc32dd..0f057d242 100644 --- a/crates/session-store/src/lib.rs +++ b/crates/session-store/src/lib.rs @@ -39,6 +39,7 @@ pub(crate) fn read_without_atime_bounded(path: &Path, max_bytes: u64) -> io::Res } pub mod capture; +mod committed_replay; pub mod event_trace; pub mod fs_store; pub mod history; diff --git a/crates/session-store/src/public_material.rs b/crates/session-store/src/public_material.rs index 150852174..7a0975148 100644 --- a/crates/session-store/src/public_material.rs +++ b/crates/session-store/src/public_material.rs @@ -1,1114 +1,8 @@ -//! Original public material from a saved, rollback-ineligible Session prefix. -//! -//! Unlike the Memory evidence index, this reader includes saved output from -//! interrupted runs and active runs with materialized AI activity. It does not -//! interpret domain-specific recording or completion policy. Callers authorize -//! the Session root and explicitly select its active Segment. -//! -//! Catalogs retain only descriptors and record locations, never original bodies. -//! Records are decoded one at a time under an explicit allocation ceiling, not -//! under a whole-run scan budget. Canonical bodies are streamed to SHA-256 and -//! bounded UTF-8 windows. A single JSONL record (including a Segment seed) must -//! fit `max_record_bytes`; exceeding that ceiling is an error, not a preview. +//! Legacy descriptor reads backed by the common committed public replay. +//! Recording capture uses the same replay, but freezes bodies into `capture::CaptureDto`. -use std::collections::{HashMap, HashSet}; -use std::fs::File; -use std::io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; - -use crate::{ - LogEntry, LoggedHistoryEntry, LoggedItem, LoggedRole, LoggedSessionHistoryOrigin, SegmentId, - SegmentOrigin, SessionId, WorkerSessionStore, +pub use crate::committed_replay::{ + PublicMaterialContent, PublicMaterialDescriptor, PublicMaterialError, PublicMaterialIdentity, + PublicMaterialLimits, PublicMaterialOrigin, PublicMaterialUse, PublicRunBoundary, + PublicRunOutcome, SessionPublicMaterialReader, }; - -#[derive(Debug, thiserror::Error)] -pub enum PublicMaterialError { - #[error("public material storage: {0}")] - Storage(#[from] io::Error), - #[error("public material record: {0}")] - Record(#[from] serde_json::Error), - #[error("public material source is missing")] - Missing, - #[error("public material source was permanently erased")] - Erased, - #[error("public material source requires migration or is unavailable")] - Unavailable, - #[error("public material lineage or identity is inconsistent")] - Inconsistent, - #[error("public material resource ceiling exceeded")] - ResourceLimit, - #[error("public material changed since the catalog was read")] - Changed, - #[error("public material offset is not a UTF-8 boundary or is outside the body")] - InvalidOffset, -} - -#[derive(Debug, Clone, Copy)] -pub struct PublicMaterialLimits { - pub max_segments: usize, - /// Total history identities examined, including inherited and private items. - pub max_entries: usize, - /// Allocation ceiling for one persisted JSONL record, not an entire run. - pub max_record_bytes: u64, -} - -impl Default for PublicMaterialLimits { - fn default() -> Self { - Self { - max_segments: 256, - max_entries: 1_000_000, - max_record_bytes: 64 * 1024 * 1024, - } - } -} - -/// Original identity, never a body-based deduplication key. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct PublicMaterialIdentity { - pub segment_id: SegmentId, - pub entry_ref: String, -} - -/// Public provenance distinguishes saved conversation from Host control facts. Run boundaries -/// are not model output, user intent, or evidence that the requested work succeeded. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum PublicMaterialOrigin { - History { origin: LoggedSessionHistoryOrigin }, - HostRunBoundary, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PublicRunOutcome { - Finished, - Paused, - Yielded, - LimitReached, -} - -/// Only whitelisted execution facts cross this boundary; raw error/provider messages do not. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum PublicRunBoundary { - Completed { - outcome: PublicRunOutcome, - interrupted: bool, - }, - Yielded, - Resumed, - Cancelled, - Errored, - Abandoned, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct PublicMaterialDescriptor { - pub identity: PublicMaterialIdentity, - /// Zero-based position on this explicitly selected active lineage. Forks - /// may remove a suffix; callers must reconcile identities, not assume that - /// positions from a previous active lineage are interchangeable. - pub canonical_order: u64, - pub origin: PublicMaterialOrigin, - /// Lowercase hexadecimal SHA-256 of the complete canonical UTF-8 body. - pub sha256: String, - pub total_bytes: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PublicMaterialContent { - pub content: String, - pub next_offset: u64, - pub total_bytes: u64, - pub sha256: String, -} - -#[derive(Clone)] -struct Location { - segment: SegmentId, - offset: u64, - history_index: usize, -} - -struct Material { - descriptor: PublicMaterialDescriptor, - location: Location, - is_tool_result: bool, -} - -struct SegmentHeader { - id: SegmentId, - parent: Option, - compact: bool, - // Compaction's exact source-history IDs disambiguate mid-turn boundaries. - source_ids: Option>, - file_len: u64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PublicMaterialUse { - Recording, - MemoryEvidence, -} - -/// A fixed material catalog. Opening a new catalog observes newly appended -/// records; reading a body through this catalog verifies its saved digest and -/// byte length again. Missing/corrected/deleted sources are explicit failures. -/// Evidence eligibility never publishes private model text or Feature payload: -/// System inputs also require an explicit public projection for every public use. -pub struct SessionPublicMaterialReader { - session_id: SessionId, - active_segment_id: SegmentId, - layout: Layout, - limits: PublicMaterialLimits, - materials: Vec, - purpose: PublicMaterialUse, - retention: Option, -} - -enum Layout { - Worker(PathBuf), - Fs(PathBuf), -} - -impl Layout { - fn path(&self, segment: SegmentId) -> PathBuf { - match self { - Self::Worker(root) => root.join("segments").join(format!("{segment}.jsonl")), - Self::Fs(root) => root.join(format!("{segment}.jsonl")), - } - } -} - -impl SessionPublicMaterialReader { - pub fn open_worker( - session_root: &Path, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - ) -> Result { - Self::open_worker_guarded( - session_root, - active_segment_id, - limits, - None, - PublicMaterialUse::Recording, - ) - } - - /// Uses the Host's aggregate-external control root for the whole scan and every body read. - /// The same store must fence publishers, archive transfer, and physical deletion. - pub fn open_worker_with_retention( - session_root: &Path, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: crate::retention::SessionRetentionStore, - ) -> Result { - Self::open_worker_for_use( - session_root, - active_segment_id, - limits, - retention, - PublicMaterialUse::Recording, - ) - } - - /// Selects the evidence purpose independently of public display and observation. - pub fn open_worker_for_use( - session_root: &Path, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: crate::retention::SessionRetentionStore, - purpose: PublicMaterialUse, - ) -> Result { - Self::open_worker_guarded( - session_root, - active_segment_id, - limits, - Some(retention), - purpose, - ) - } - - fn open_worker_guarded( - session_root: &Path, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: Option, - purpose: PublicMaterialUse, - ) -> Result { - let store = WorkerSessionStore::open_read_only(session_root) - .map_err(|_| PublicMaterialError::Unavailable)?; - let session_id = store - .session_id() - .map_err(|_| PublicMaterialError::Unavailable)? - .ok_or(PublicMaterialError::Missing)?; - Self::open( - session_id, - active_segment_id, - Layout::Worker(session_root.into()), - limits, - retention, - purpose, - ) - } - - pub fn open_fs( - sessions_root: &Path, - session_id: SessionId, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - ) -> Result { - Self::open_fs_with_retention( - sessions_root, - session_id, - active_segment_id, - limits, - crate::retention::fs_retention(sessions_root), - ) - } - - /// Keeps the same erasure fence when a Host relocates originals to retained storage. - pub fn open_fs_with_retention( - sessions_root: &Path, - session_id: SessionId, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: crate::retention::SessionRetentionStore, - ) -> Result { - Self::open_fs_for_use( - sessions_root, - session_id, - active_segment_id, - limits, - retention, - PublicMaterialUse::Recording, - ) - } - - /// Uses retained originals with the independently selected evidence policy. - pub fn open_fs_for_use( - sessions_root: &Path, - session_id: SessionId, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: crate::retention::SessionRetentionStore, - purpose: PublicMaterialUse, - ) -> Result { - Self::open( - session_id, - active_segment_id, - Layout::Fs(sessions_root.join(session_id.to_string())), - limits, - Some(retention), - purpose, - ) - } - - pub fn session_id(&self) -> SessionId { - self.session_id - } - pub fn active_segment_id(&self) -> SegmentId { - self.active_segment_id - } - pub fn descriptors(&self) -> impl ExactSizeIterator { - self.materials.iter().map(|value| &value.descriptor) - } - - fn open( - session_id: SessionId, - active_segment_id: SegmentId, - layout: Layout, - limits: PublicMaterialLimits, - retention: Option, - purpose: PublicMaterialUse, - ) -> Result { - let _guard = retention - .as_ref() - .map(|store| store.lock_live(session_id).map_err(retention_error)) - .transpose()?; - if limits.max_segments == 0 || limits.max_entries == 0 || limits.max_record_bytes == 0 { - return Err(PublicMaterialError::ResourceLimit); - } - let mut chain = Vec::new(); - let mut visited = HashSet::new(); - let mut current = active_segment_id; - let mut header_entries = 0usize; - loop { - if !visited.insert(current) { - return Err(PublicMaterialError::Inconsistent); - } - if visited.len() > limits.max_segments { - return Err(PublicMaterialError::ResourceLimit); - } - let mut reader = RecordReader::open(&layout.path(current), limits.max_record_bytes)?; - let file_len = reader.end; - let Some(( - _, - LogEntry::AnnotatedSegmentStart { - session_id: stored, - forked_from, - compacted_from, - history, - .. - }, - )) = reader.next()? - else { - return Err(PublicMaterialError::Inconsistent); - }; - if stored != session_id || (forked_from.is_some() && compacted_from.is_some()) { - return Err(PublicMaterialError::Inconsistent); - } - header_entries = header_entries - .checked_add(history.len()) - .ok_or(PublicMaterialError::ResourceLimit)?; - if header_entries > limits.max_entries { - return Err(PublicMaterialError::ResourceLimit); - } - let compact = compacted_from.is_some(); - let source_ids: Option> = if compact { - let summaries: Vec<_> = history - .iter() - .filter(|entry| { - entry.metadata.input_metadata.as_ref().is_some_and(|input| { - input.usage().compaction == crate::InputCompactionUse::OpeningSummary - }) - }) - .collect(); - if summaries.len() != 1 { - return Err(PublicMaterialError::Inconsistent); - } - Some( - summaries[0] - .metadata - .derivation - .as_ref() - .ok_or(PublicMaterialError::Inconsistent)? - .sources - .iter() - .map(|id| id.0.clone()) - .collect(), - ) - } else { - None - }; - header_entries = header_entries - .checked_add(source_ids.as_ref().map_or(0, |ids| ids.len())) - .ok_or(PublicMaterialError::ResourceLimit)?; - if header_entries > limits.max_entries { - return Err(PublicMaterialError::ResourceLimit); - } - let parent = forked_from.or(compacted_from); - let next = parent.as_ref().map(|p| p.segment_id); - chain.push(SegmentHeader { - id: current, - parent, - compact, - source_ids, - file_len, - }); - match next { - Some(id) => current = id, - None => break, - } - } - chain.reverse(); - let mut materials = Vec::::new(); - let mut identities = HashMap::::new(); - let mut examined = 0usize; - let mut expected_fork_seed: Option> = None; - for (segment_index, header) in chain.iter().enumerate() { - let child = chain.get(segment_index + 1); - let mut reader = RecordReader::open(&layout.path(header.id), limits.max_record_bytes)?; - reader.end = header.file_len; - let mut history_ids = Vec::::new(); - let mut history_seen = HashSet::::new(); - let mut inherited_order = None; - let mut pending = Vec::::new(); - let mut in_run = false; - let mut safe = true; - let mut turn = 0usize; - let mut reached_cut = child.is_none(); - let mut compact_cut_turn_seen = false; - while let Some((offset, record)) = reader.next()? { - // Once a compaction's exact source history is reached, consume - // only non-history boundaries/corrections. A later history - // addition belongs to the parent's unadopted suffix. - if reached_cut - && child.is_some_and(|c| c.compact) - && matches!( - record, - LogEntry::AnnotatedUserInput { .. } - | LogEntry::AnnotatedAssistantItem { .. } - | LogEntry::AnnotatedToolResult { .. } - | LogEntry::AnnotatedSystemItem { .. } - ) - { - break; - } - let seed = matches!(record, LogEntry::AnnotatedSegmentStart { .. }); - if seed { - if let LogEntry::AnnotatedSegmentStart { - history, - session_id: stored, - .. - } = &record - { - if *stored != session_id { - return Err(PublicMaterialError::Inconsistent); - } - if let Some(expected) = expected_fork_seed.take() { - if history - .iter() - .map(|e| &e.metadata.entry_id.0) - .ne(expected.iter()) - { - return Err(PublicMaterialError::Inconsistent); - } - } - } - } - if offset != 0 && seed { - return Err(PublicMaterialError::Inconsistent); - } - match &record { - LogEntry::Invoke { .. } | LogEntry::RunResumed { .. } => { - if !pending.is_empty() { - return Err(PublicMaterialError::Inconsistent); - } - in_run = true; - safe = false; - } - LogEntry::RunCompleted { .. } - | LogEntry::RunYielded { .. } - | LogEntry::RunCancelled { .. } - | LogEntry::RunErrored { .. } - | LogEntry::PausedTurnAbandoned { .. } => { - safe = true; - in_run = false; - } - LogEntry::TurnEnd { turn_count, .. } => turn = *turn_count, - LogEntry::ActiveRunCheckpoint { - total_turn_count, .. - } => turn = *total_turn_count, - _ => {} - } - let system_history; - let entries: &[LoggedHistoryEntry] = match &record { - LogEntry::AnnotatedSegmentStart { history, .. } - | LogEntry::AnnotatedUserInput { history, .. } => history, - LogEntry::AnnotatedAssistantItem { entry, .. } - | LogEntry::AnnotatedToolResult { entry, .. } - | LogEntry::ToolResultCorrected { entry, .. } => std::slice::from_ref(entry), - LogEntry::AnnotatedSystemItem { entry, .. } => { - system_history = entry.to_history_entry(); - std::slice::from_ref(&system_history) - } - _ => &[], - }; - for (history_index, entry) in entries.iter().enumerate() { - charge(&mut examined, limits.max_entries)?; - let id = &entry.metadata.entry_id.0; - if id.is_empty() { - return Err(PublicMaterialError::Inconsistent); - } - let corrected = matches!(record, LogEntry::ToolResultCorrected { .. }); - if !corrected { - if !history_seen.insert(id.clone()) { - return Err(PublicMaterialError::Inconsistent); - } - history_ids.push(id.clone()); - } - if in_run && ai_materialized(&entry.item) { - safe = true; - } - let Some((sha256, total_bytes)) = canonical_metadata(entry, purpose)? else { - continue; - }; - let location = Location { - segment: header.id, - offset, - history_index, - }; - let descriptor = PublicMaterialDescriptor { - identity: PublicMaterialIdentity { - segment_id: header.id, - entry_ref: format!("E{id}"), - }, - canonical_order: 0, - origin: PublicMaterialOrigin::History { - origin: entry.metadata.origin.clone(), - }, - sha256, - total_bytes, - }; - if seed && header.parent.is_some() { - // Every inherited original must already exist on the - // selected ancestor prefix; compare identity + bytes. - let index = identities - .get(id) - .ok_or(PublicMaterialError::Inconsistent)?; - if inherited_order.is_some_and(|previous| *index <= previous) { - return Err(PublicMaterialError::Inconsistent); - } - inherited_order = Some(*index); - let original = &materials[*index].descriptor; - if original.sha256 != descriptor.sha256 - || original.total_bytes != descriptor.total_bytes - || original.origin != descriptor.origin - { - return Err(PublicMaterialError::Inconsistent); - } - } else if corrected { - let index = identities - .get(id) - .ok_or(PublicMaterialError::Inconsistent)?; - let original = &mut materials[*index]; - if !original.is_tool_result - || original.descriptor.origin != descriptor.origin - || !matches!(entry.item, LoggedItem::ToolResult { .. }) - { - return Err(PublicMaterialError::Inconsistent); - } - original.descriptor.sha256 = descriptor.sha256; - original.descriptor.total_bytes = descriptor.total_bytes; - original.location = location; - } else { - if identities.contains_key(id) - || pending.iter().any(|m| { - m.descriptor.identity.entry_ref == descriptor.identity.entry_ref - }) - { - return Err(PublicMaterialError::Inconsistent); - } - pending.push(Material { - descriptor, - location, - is_tool_result: matches!(entry.item, LoggedItem::ToolResult { .. }), - }); - } - } - if let Some(boundary) = run_boundary(&record) { - charge(&mut examined, limits.max_entries)?; - let reference = boundary_reference(&record, session_id, header.id, offset); - if identities.contains_key(&reference[1..]) - || pending - .iter() - .any(|m| m.descriptor.identity.entry_ref == reference) - { - return Err(PublicMaterialError::Inconsistent); - } - let mut sink = BodySink::new(0, 0); - serde_json::to_writer(&mut sink, &CanonicalBody::RunBoundary { boundary })?; - let (sha256, total_bytes, _) = sink.finish()?; - pending.push(Material { - descriptor: PublicMaterialDescriptor { - identity: PublicMaterialIdentity { - segment_id: header.id, - entry_ref: reference, - }, - canonical_order: 0, - origin: PublicMaterialOrigin::HostRunBoundary, - sha256, - total_bytes, - }, - location: Location { - segment: header.id, - offset, - history_index: 0, - }, - is_tool_result: false, - }); - } - if safe { - flush(&mut materials, &mut identities, &mut pending); - } - if let Some(child) = child { - if child.compact { - let sources = child - .source_ids - .as_ref() - .ok_or(PublicMaterialError::Inconsistent)?; - if &history_ids == sources { - // Do not stop at an earlier non-history record when - // compaction's source-history is empty. - if !sources.is_empty() || seed { - reached_cut = true; - if child - .parent - .as_ref() - .is_some_and(|p| p.at_turn_index == turn) - { - compact_cut_turn_seen = true; - } - } - } - } else if (seed && child.parent.as_ref().is_some_and(|p| p.at_turn_index == 0)) - || (turn - == child - .parent - .as_ref() - .ok_or(PublicMaterialError::Inconsistent)? - .at_turn_index - && matches!(record, LogEntry::TurnEnd { .. })) - { - reached_cut = true; - break; - } - } - } - if !reached_cut || (child.is_some_and(|c| c.compact) && !compact_cut_turn_seen) { - return Err(PublicMaterialError::Inconsistent); - } - if child.is_some_and(|c| !c.compact) { - expected_fork_seed = Some(history_ids); - } - // A compaction seed proves its selected input is durably retained; - // still require materialized AI or a terminal boundary for a run. - if !pending.is_empty() && child.is_some() { - return Err(PublicMaterialError::Inconsistent); - } - } - Ok(Self { - session_id, - active_segment_id, - layout, - limits, - materials, - purpose, - retention: retention.clone(), - }) - } - - /// Read a canonical body window without allocating its full rendered text. - /// `max_content_bytes >= 4` guarantees progress for every UTF-8 scalar. - pub fn read_content( - &self, - descriptor: &PublicMaterialDescriptor, - offset: u64, - max_content_bytes: usize, - ) -> Result { - let _guard = self - .retention - .as_ref() - .map(|store| store.lock_live(self.session_id).map_err(retention_error)) - .transpose()?; - if max_content_bytes < 4 { - return Err(PublicMaterialError::ResourceLimit); - } - if offset > descriptor.total_bytes { - return Err(PublicMaterialError::InvalidOffset); - } - let material = self - .materials - .get( - usize::try_from(descriptor.canonical_order) - .map_err(|_| PublicMaterialError::Changed)?, - ) - .filter(|m| &m.descriptor == descriptor) - .ok_or(PublicMaterialError::Changed)?; - let mut reader = RecordReader::open( - &self.layout.path(material.location.segment), - self.limits.max_record_bytes, - )?; - reader.seek(material.location.offset)?; - let Some((_, record)) = reader.next()? else { - return Err(PublicMaterialError::Missing); - }; - let system_history; - let (body, origin) = if let Some(boundary) = run_boundary(&record) { - if boundary_reference( - &record, - self.session_id, - material.location.segment, - material.location.offset, - ) != descriptor.identity.entry_ref - { - return Err(PublicMaterialError::Changed); - } - ( - CanonicalBody::RunBoundary { boundary }, - PublicMaterialOrigin::HostRunBoundary, - ) - } else { - let entry = match &record { - LogEntry::AnnotatedSegmentStart { history, .. } - | LogEntry::AnnotatedUserInput { history, .. } => { - history.get(material.location.history_index) - } - LogEntry::AnnotatedAssistantItem { entry, .. } - | LogEntry::AnnotatedToolResult { entry, .. } - | LogEntry::ToolResultCorrected { entry, .. } => Some(entry), - LogEntry::AnnotatedSystemItem { entry, .. } => { - system_history = entry.to_history_entry(); - Some(&system_history) - } - _ => None, - } - .filter(|e| format!("E{}", e.metadata.entry_id.0) == descriptor.identity.entry_ref) - .ok_or(PublicMaterialError::Changed)?; - ( - canonical_body(entry, self.purpose).ok_or(PublicMaterialError::Changed)?, - PublicMaterialOrigin::History { - origin: entry.metadata.origin.clone(), - }, - ) - }; - let mut sink = BodySink::new(offset, max_content_bytes); - serde_json::to_writer(&mut sink, &body)?; - let (sha256, total_bytes, content) = sink.finish()?; - if sha256 != descriptor.sha256 - || total_bytes != descriptor.total_bytes - || origin != descriptor.origin - { - return Err(PublicMaterialError::Changed); - } - Ok(PublicMaterialContent { - next_offset: offset + content.len() as u64, - content, - total_bytes, - sha256, - }) - } -} - -fn retention_error(error: crate::StoreError) -> PublicMaterialError { - match error { - crate::StoreError::SessionErased(_) => PublicMaterialError::Erased, - crate::StoreError::Io(error) => PublicMaterialError::Storage(error), - crate::StoreError::Serde(error) => PublicMaterialError::Record(error), - _ => PublicMaterialError::Unavailable, - } -} - -fn charge(examined: &mut usize, max: usize) -> Result<(), PublicMaterialError> { - *examined = examined - .checked_add(1) - .ok_or(PublicMaterialError::ResourceLimit)?; - if *examined > max { - return Err(PublicMaterialError::ResourceLimit); - } - Ok(()) -} - -fn flush( - materials: &mut Vec, - identities: &mut HashMap, - pending: &mut Vec, -) { - for mut material in pending.drain(..) { - let index = materials.len(); - material.descriptor.canonical_order = index as u64; - identities.insert(material.descriptor.identity.entry_ref[1..].into(), index); - materials.push(material); - } -} - -fn ai_materialized(item: &LoggedItem) -> bool { - !matches!( - item, - LoggedItem::Message { - role: LoggedRole::User | LoggedRole::System, - .. - } - ) -} - -/// Versioned canonical JSON framing. This is original text, not a display -/// preview: both result summary and optional full content remain distinguishable. -#[derive(Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum CanonicalBody<'a> { - RunBoundary { - boundary: PublicRunBoundary, - }, - Message { - origin: &'a LoggedSessionHistoryOrigin, - role: LoggedRole, - content: std::borrow::Cow<'a, [crate::LoggedContentPart]>, - }, - ToolCall { - origin: &'a LoggedSessionHistoryOrigin, - call_id: &'a str, - name: &'a str, - arguments: &'a str, - }, - ToolResult { - origin: &'a LoggedSessionHistoryOrigin, - call_id: &'a str, - summary: &'a str, - content: &'a Option, - disposition: &'a agen::tool::ToolResultDisposition, - is_error: bool, - }, -} - -fn run_boundary(record: &LogEntry) -> Option { - Some(match record { - LogEntry::RunCompleted { - result, - interrupted, - .. - } => PublicRunBoundary::Completed { - outcome: match result { - agen::EngineResult::Finished => PublicRunOutcome::Finished, - agen::EngineResult::Paused => PublicRunOutcome::Paused, - agen::EngineResult::Yielded => PublicRunOutcome::Yielded, - agen::EngineResult::LimitReached => PublicRunOutcome::LimitReached, - }, - interrupted: *interrupted, - }, - LogEntry::RunYielded { .. } => PublicRunBoundary::Yielded, - LogEntry::RunResumed { .. } => PublicRunBoundary::Resumed, - LogEntry::RunCancelled { .. } => PublicRunBoundary::Cancelled, - LogEntry::RunErrored { .. } => PublicRunBoundary::Errored, - LogEntry::PausedTurnAbandoned { .. } => PublicRunBoundary::Abandoned, - _ => return None, - }) -} - -fn boundary_reference( - record: &LogEntry, - session: SessionId, - segment: SegmentId, - offset: u64, -) -> String { - let id = match record { - LogEntry::RunCompleted { entry_id, .. } - | LogEntry::PausedTurnAbandoned { entry_id, .. } - | LogEntry::RunYielded { entry_id, .. } - | LogEntry::RunResumed { entry_id, .. } - | LogEntry::RunCancelled { entry_id, .. } - | LogEntry::RunErrored { entry_id, .. } => entry_id.as_ref(), - _ => None, - }; - // Reuse the storage convention for legacy transitions, including the exact physical - // record position. New normal completions have no persisted transition ID either. - id.map(|id| format!("E{}", id.0)) - .unwrap_or_else(|| format!("El-run-{session}-{segment}-{offset}")) -} - -fn canonical_body( - entry: &LoggedHistoryEntry, - purpose: PublicMaterialUse, -) -> Option> { - if let Some(input) = &entry.metadata.input_metadata { - let usage = input.usage(); - let allowed = match purpose { - PublicMaterialUse::Recording => usage.recording_material, - PublicMaterialUse::MemoryEvidence => usage.memory_evidence, - }; - if allowed == crate::InputEvidenceUse::Exclude { - return None; - } - if let LoggedItem::Message { - role: LoggedRole::System, - .. - } = &entry.item - { - let public = input.public_item()?; - return Some(CanonicalBody::Message { - origin: &entry.metadata.origin, - role: LoggedRole::System, - content: std::borrow::Cow::Owned(vec![crate::LoggedContentPart::Text { - text: public.body.clone(), - }]), - }); - } - } - match &entry.item { - LoggedItem::Message { - role: LoggedRole::System, - .. - } - | LoggedItem::Reasoning { .. } - | LoggedItem::AssistantResponseBoundary { .. } => None, - LoggedItem::Message { role, content } => Some(CanonicalBody::Message { - origin: &entry.metadata.origin, - role: *role, - content: std::borrow::Cow::Borrowed(content), - }), - LoggedItem::ToolCall { - call_id, - name, - arguments, - .. - } => Some(CanonicalBody::ToolCall { - origin: &entry.metadata.origin, - call_id, - name, - arguments, - }), - LoggedItem::ToolResult { - call_id, - summary, - content, - disposition, - is_error, - .. - } => Some(CanonicalBody::ToolResult { - origin: &entry.metadata.origin, - call_id, - summary, - content, - disposition, - is_error: *is_error, - }), - } -} - -fn canonical_metadata( - entry: &LoggedHistoryEntry, - purpose: PublicMaterialUse, -) -> Result, PublicMaterialError> { - let Some(body) = canonical_body(entry, purpose) else { - return Ok(None); - }; - let mut sink = BodySink::new(0, 0); - serde_json::to_writer(&mut sink, &body)?; - Ok(Some((hex_digest(&sink.digest.finalize()), sink.total))) -} - -fn hex_digest(bytes: &[u8]) -> String { - bytes.iter().map(|b| format!("{b:02x}")).collect() -} - -struct BodySink { - digest: Sha256, - total: u64, - offset: u64, - max: usize, - bytes: Vec, -} - -impl BodySink { - fn new(offset: u64, max: usize) -> Self { - Self { - digest: Sha256::new(), - total: 0, - offset, - max, - bytes: Vec::new(), - } - } - fn finish(mut self) -> Result<(String, u64, String), PublicMaterialError> { - // The final byte may land inside a UTF-8 scalar. Never include a - // partial scalar and never advance the caller over its missing bytes. - loop { - match std::str::from_utf8(&self.bytes) { - Ok(_) => break, - Err(e) if e.error_len().is_none() => { - self.bytes.truncate(e.valid_up_to()); - } - Err(_) => return Err(PublicMaterialError::InvalidOffset), - } - } - Ok(( - hex_digest(&self.digest.finalize()), - self.total, - String::from_utf8(self.bytes).map_err(|_| PublicMaterialError::InvalidOffset)?, - )) - } -} - -impl Write for BodySink { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.digest.update(bytes); - let start = self - .offset - .saturating_sub(self.total) - .min(bytes.len() as u64) as usize; - let available = self.max.saturating_sub(self.bytes.len()); - let end = start.saturating_add(available).min(bytes.len()); - if end > start { - self.bytes.extend_from_slice(&bytes[start..end]); - } - self.total = self - .total - .checked_add(bytes.len() as u64) - .ok_or_else(|| io::Error::other("canonical body length overflow"))?; - Ok(bytes.len()) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -struct RecordReader { - reader: BufReader, - offset: u64, - end: u64, - max: u64, -} - -impl RecordReader { - fn open(path: &Path, max: u64) -> Result { - // Do not follow symlinks in a caller-selected storage root or a log. - let mut prefix = PathBuf::new(); - for component in path.components() { - prefix.push(component); - let metadata = std::fs::symlink_metadata(&prefix)?; - if metadata.file_type().is_symlink() { - return Err(PublicMaterialError::Unavailable); - } - } - let file = open_noatime(path)?; - if !file.metadata()?.is_file() { - return Err(PublicMaterialError::Unavailable); - } - let end = file.metadata()?.len(); - Ok(Self { - reader: BufReader::new(file), - offset: 0, - end, - max, - }) - } - fn seek(&mut self, offset: u64) -> Result<(), PublicMaterialError> { - self.reader.seek(SeekFrom::Start(offset))?; - self.offset = offset; - Ok(()) - } - fn next(&mut self) -> Result, PublicMaterialError> { - if self.offset >= self.end { - return Ok(None); - } - let mut bytes = Vec::new(); - let read = (&mut self.reader) - .take(self.max.saturating_add(1).min(self.end - self.offset)) - .read_until(b'\n', &mut bytes)? as u64; - if read > self.max { - return Err(PublicMaterialError::ResourceLimit); - } - if bytes.last() != Some(&b'\n') { - return Ok(None); - } // Uncommitted tail. - let offset = self.offset; - self.offset += read; - Ok(Some((offset, serde_json::from_slice(&bytes)?))) - } -} - -fn open_noatime(path: &Path) -> io::Result { - #[cfg(any(target_os = "linux", target_os = "android"))] - { - use std::os::unix::fs::OpenOptionsExt; - std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW) - .open(path) - } - #[cfg(not(any(target_os = "linux", target_os = "android")))] - { - let _ = path; - Err(io::Error::new( - io::ErrorKind::Unsupported, - "no-atime reads unavailable", - )) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/session-store/src/public_material/tests.rs b/crates/session-store/src/public_material/tests.rs index 230b81055..b7f4a8e7c 100644 --- a/crates/session-store/src/public_material/tests.rs +++ b/crates/session-store/src/public_material/tests.rs @@ -897,8 +897,28 @@ fn midturn_compaction_uses_exact_source_ids_and_preserves_discarded_originals() store .create_segment(session, compact, &[start, output("new", "new")]) .unwrap(); + let retention = crate::retention::fs_retention(temp.path()); + let guard = retention.lock_live(session).unwrap(); + let captured = crate::capture::capture_recording_fs_guarded( + temp.path(), + session, + compact, + Default::default(), + Default::default(), + &guard, + ) + .unwrap(); + drop(guard); + let saved_bytes = captured.encode(Default::default()).unwrap(); let reader = read(temp.path(), session, compact); assert_eq!(ids(&reader), ["Eu", "Ekept", "Enew"]); + for (entry, descriptor) in captured.entries.iter().zip(reader.descriptors()) { + assert_eq!(captured.recording_text(entry.order).unwrap().0, descriptor); + } + store + .append(session, compact, &output("after-snapshot", "next Job only")) + .unwrap(); + assert_eq!(captured.encode(Default::default()).unwrap(), saved_bytes); let d: Vec<_> = reader.descriptors().collect(); assert_eq!(d[0].identity.segment_id, root); assert_eq!(d[1].identity.segment_id, root); @@ -913,6 +933,16 @@ fn midturn_compaction_uses_exact_source_ids_and_preserves_discarded_originals() .is_err() ); assert!(reader.read_content(d[0], 0, 4096).is_err()); + assert_eq!( + captured + .recording_slice(Default::default(), 100, 8192) + .unwrap() + .capture + .entries + .len(), + 3 + ); + assert_eq!(captured.encode(Default::default()).unwrap(), saved_bytes); } #[test] @@ -1484,6 +1514,991 @@ fn generic_system_material_uses_explicit_evidence_purpose_not_producer_or_body() assert!(!body.contains("not-public")); assert!(!body.contains("private model text")); assert!(!body.contains("unknown:future")); + if purpose == PublicMaterialUse::Recording { + let retention = crate::retention::fs_retention(temp.path()); + let guard = retention.lock_live(session).unwrap(); + let capture = if worker_layout { + crate::capture::capture_recording_worker_guarded( + &root, + session, + fork, + Default::default(), + Default::default(), + &guard, + ) + } else { + crate::capture::capture_recording_fs_guarded( + temp.path(), + session, + fork, + Default::default(), + Default::default(), + &guard, + ) + } + .unwrap(); + assert_eq!(capture.entries.len(), 1); + let (saved, _, text) = capture.recording_text(0).unwrap(); + assert_eq!(saved, descriptor); + assert_eq!(text, body); + let view = + crate::capture::SessionCapture::try_from_dto(capture, Default::default()) + .unwrap(); + assert!(view.search(&Default::default()).is_empty()); + assert!(view.overview().is_empty()); + } + } + } +} + +#[test] +fn recording_capture_slices_canonical_utf8_with_metadata_and_legacy_descriptors() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("session"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let store = WorkerSessionStore::new_with_retention(&root, retention.clone()).unwrap(); + let mut records = vec![seed(session, vec![]), invoke(), input("u")]; + for i in 0..150 { + records.push(output(&format!("a-{i}"), "saved text")); + } + records.push(LogEntry::AnnotatedToolResult { + ts: 5, + entry: entry( + "tool", + LoggedItem::ToolResult { + call_id: "call".into(), + summary: "summary".into(), + content: Some("漢字🍎\n\"".repeat(6000)), + disposition: agen::tool::ToolResultDisposition::default(), + is_error: false, + attachments: vec![], + }, + ), + }); + records.push(cancelled()); + store.create_segment(session, segment, &records).unwrap(); + let guard = retention.lock_live(session).unwrap(); + let capture = capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + Default::default(), + &guard, + ) + .unwrap(); + drop(guard); + assert_eq!(capture.entries.len(), 153); + let reader = SessionPublicMaterialReader::open_worker_with_retention( + &root, + segment, + Default::default(), + retention.clone(), + ) + .unwrap(); + let descriptors: Vec<_> = reader.descriptors().cloned().collect(); + for (entry, descriptor) in capture.entries.iter().zip(&descriptors) { + let (saved, offset, text) = capture.recording_text(entry.order).unwrap(); + assert_eq!(saved, descriptor); + assert_eq!(offset, 0); + assert_eq!( + reader + .read_content(descriptor, 0, text.len().max(4)) + .unwrap() + .content, + text + ); + } + let bytes = capture.encode(Default::default()).unwrap(); + let restored = CaptureDto::decode(&bytes, Default::default()).unwrap(); + let mut position = RecordingPosition::default(); + let mut combined = vec![String::new(); descriptors.len()]; + let mut slices = 0; + loop { + let slice = restored.recording_slice(position, 29, 4096).unwrap(); + assert!(serde_json::to_vec(&slice).unwrap().len() <= 4096); + assert!(!slice.capture.entries.is_empty()); + slice.capture.validate(Default::default()).unwrap(); + for entry in &slice.capture.entries { + let (descriptor, offset, text) = slice.capture.recording_text(entry.order).unwrap(); + let body = &mut combined[entry.order as usize]; + assert_eq!(offset, body.len() as u64); + assert_eq!(descriptor, &descriptors[entry.order as usize]); + body.push_str(text); + } + slices += 1; + let Some(next) = slice.next else { + break; + }; + assert_ne!(next, position); + position = next; + } + assert!(slices > 10); + for (order, text) in combined.iter().enumerate() { + assert_eq!(text, capture.recording_text(order as u64).unwrap().2); + } + assert!(matches!( + capture.entries.last().unwrap().body, + CaptureBody::CanonicalPublic { + boundary: Some(PublicRunBoundary::Cancelled), + .. + } + )); + let mut corrupt = restored.clone(); + if let CaptureBody::CanonicalPublic { text, .. } = &mut corrupt.entries[0].body { + text.replace_range(0..1, "["); + } + assert!(corrupt.validate(Default::default()).is_err()); + let mut unknown: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + unknown["entries"][0]["body"]["descriptor"]["unexpected"] = true.into(); + assert!( + CaptureDto::decode(&serde_json::to_vec(&unknown).unwrap(), Default::default()).is_err() + ); + let mut missing = restored.clone(); + missing.entries.remove(1); + assert!(matches!( + missing.recording_slice(Default::default(), 3, 8192), + Err(CaptureError::InvalidRange) + )); + let tool_order = 151; + let tool_text = capture.recording_text(tool_order).unwrap().2; + let bad = tool_text.find('漢').unwrap() as u64 + 1; + assert!(matches!( + capture.recording_slice( + RecordingPosition { + canonical_order: tool_order, + byte_offset: bad + }, + 1, + 4096 + ), + Err(CaptureError::InvalidRange) + )); + assert!(matches!( + capture.recording_slice(Default::default(), 1, 10), + Err(CaptureError::BudgetTooSmall) + )); + // The full-value ceiling does not constrain bounded construction of a long Tool result. + let guard = retention.lock_live(session).unwrap(); + assert!(matches!( + capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + CaptureLimits { + max_bytes: 2048, + ..Default::default() + }, + &guard + ), + Err(PublicMaterialError::ResourceLimit) + )); + let bounded = capture_recording_slice_worker_guarded( + &root, + session, + segment, + Default::default(), + RecordingPosition { + canonical_order: tool_order, + byte_offset: 0, + }, + 1, + 4096, + &guard, + ) + .unwrap(); + assert!(serde_json::to_vec(&bounded).unwrap().len() <= 4096); + assert!(bounded.next.is_some()); +} + +#[test] +fn recording_capture_fence_blocks_publication_and_value_survives_later_append_and_erasure() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("session"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let store = WorkerSessionStore::new_with_retention(&root, retention.clone()).unwrap(); + store + .create_segment( + session, + segment, + &[ + seed(session, vec![]), + invoke(), + input("u"), + output("a", "saved"), + ], + ) + .unwrap(); + let guard = retention.lock_live(session).unwrap(); + let (started, receiver) = std::sync::mpsc::channel(); + let (done, completed) = std::sync::mpsc::channel(); + let writer = store.clone(); + let thread = std::thread::spawn(move || { + started.send(()).unwrap(); + writer + .append(session, segment, &output("later", "not in snapshot")) + .unwrap(); + done.send(()).unwrap(); + }); + receiver + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + let capture = capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + Default::default(), + &guard, + ) + .unwrap(); + assert!(completed.try_recv().is_err()); + let bytes = capture.encode(Default::default()).unwrap(); + drop(guard); + completed + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + thread.join().unwrap(); + assert_eq!(capture.encode(Default::default()).unwrap(), bytes); + assert_eq!(capture.entries.len(), 2); + retention + .delete( + session, + crate::retention::SessionDeletionIntent::ErasePermanently, + || { + std::fs::remove_dir_all(&root)?; + Ok(()) + }, + ) + .unwrap(); + assert!(matches!( + capture_recording_worker( + &root, + session, + segment, + &retention, + Default::default(), + Default::default() + ), + Err(PublicMaterialError::Erased) + )); + // Pure reads do not authorize adoption after erase; Host consumer revocation is separate. + assert_eq!( + capture + .recording_slice(Default::default(), 2, 8192) + .unwrap() + .capture + .entries + .len(), + 2 + ); +} + +#[test] +fn recording_capture_rejects_foreign_guard_and_storage_or_body_allocation_excess() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("session"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let store = WorkerSessionStore::new_with_retention(&root, retention.clone()).unwrap(); + store + .create_segment( + session, + segment, + &[seed( + session, + vec![message("original", LoggedRole::User, "text")], + )], + ) + .unwrap(); + let wrong_guard = retention.lock_live(crate::new_session_id()).unwrap(); + assert!(matches!( + capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + Default::default(), + &wrong_guard + ), + Err(PublicMaterialError::Inconsistent) + )); + let guard = retention.lock_live(session).unwrap(); + let limits = PublicMaterialLimits { + max_storage_bytes: 1, + ..Default::default() + }; + assert!(matches!( + capture_recording_worker_guarded( + &root, + session, + segment, + limits, + Default::default(), + &guard + ), + Err(PublicMaterialError::ResourceLimit) + )); + let limits = CaptureLimits { + max_text_bytes: 1, + ..Default::default() + }; + assert!(matches!( + capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + limits, + &guard + ), + Err(PublicMaterialError::ResourceLimit) + )); +} + +#[test] +fn fixed_replay_ends_do_not_refresh_for_appends_between_replay_and_body_snapshot() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("session"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let store = WorkerSessionStore::new_with_retention(&root, retention.clone()).unwrap(); + store + .create_segment( + session, + segment, + &[ + seed(session, vec![]), + invoke(), + input("u"), + output("a", "saved"), + ], + ) + .unwrap(); + let guard = retention.lock_live(session).unwrap(); + let replay = CommittedPublicReplay::open_worker_guarded( + &root, + segment, + Default::default(), + None, + PublicMaterialUse::Recording, + ) + .unwrap(); + let before = replay + .recording_slice(Default::default(), 100, 8192) + .unwrap(); + drop(guard); + // Production keeps its fence through construction. Releasing it deliberately + // here also verifies that the lower-level replay never refreshes file ends. + store + .append(session, segment, &output("later", "not in fixed files")) + .unwrap(); + let other = crate::new_segment_id(); + store + .create_segment( + session, + other, + &[ + seed(session, vec![]), + output("other", "different active segment"), + ], + ) + .unwrap(); + let after = replay + .recording_slice(RecordingPosition::default(), 100, 8192) + .unwrap(); + assert_eq!(after, before); + assert_eq!(after.capture.entries.len(), 2); +} + +#[test] +fn recording_source_coordinates_count_hidden_rows_and_preserve_originals_across_compaction() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let retention = crate::retention::fs_retention(temp.path()); + let store = FsStore::new_with_retention(temp.path(), retention.clone()).unwrap(); + let session = crate::new_session_id(); + let root = crate::new_segment_id(); + let compact = crate::new_segment_id(); + let hidden = + |id: &str| system_context(id, "private omitted row", crate::InputCompactionUse::Retain); + let root_user = message("root-user", LoggedRole::User, "first public row"); + let tool = entry( + "root-tool", + LoggedItem::ToolResult { + call_id: "call".into(), + summary: "original".into(), + content: Some("full original".into()), + disposition: Default::default(), + is_error: false, + attachments: vec![], + }, + ); + let mut corrected = tool.clone(); + if let LoggedItem::ToolResult { content, .. } = &mut corrected.item { + *content = Some("corrected full original".into()); + } + let source_ids = [ + "hidden-0", + "root-user", + "hidden-2", + "hidden-3", + "batch-user", + "reasoning", + "root-tool", + "root-answer", + ]; + store + .create_segment( + session, + root, + &[ + seed( + session, + vec![hidden("hidden-0"), root_user.clone(), hidden("hidden-2")], + ), + invoke(), + LogEntry::AnnotatedUserInput { + ts: 2, + segments: vec![], + extensions: vec![], + history: vec![ + hidden("hidden-3"), + message("batch-user", LoggedRole::User, "second public row"), + ], + }, + LogEntry::AnnotatedAssistantItem { + ts: 3, + entry: entry( + "reasoning", + LoggedItem::Reasoning { + text: "private reasoning".into(), + summary: vec![], + encrypted_content: None, + signature: None, + }, + ), + }, + LogEntry::AnnotatedToolResult { + ts: 4, + entry: tool.clone(), + }, + LogEntry::RunYielded { + ts: 6, + entry_id: Some(LoggedSessionHistoryEntryId("yield-gap".into())), + reason: protocol::RunYieldReason::Compaction, + active_run_turn_count: 1, + }, + LogEntry::RunResumed { + ts: 7, + entry_id: Some(LoggedSessionHistoryEntryId("resume-gap".into())), + source: protocol::RunResumeSource::Compaction, + active_run_turn_count: 1, + }, + output("root-answer", "public answer"), + LogEntry::TurnEnd { + ts: 8, + turn_count: 1, + }, + ], + ) + .unwrap(); + let mut summary = system_context( + "summary", + "not an original", + crate::InputCompactionUse::OpeningSummary, + ); + summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary; + summary.metadata.derivation = Some(crate::LoggedHistoryDerivation { + sources: source_ids + .iter() + .map(|id| LoggedSessionHistoryEntryId((*id).into())) + .collect(), + }); + let mut start = seed( + session, + vec![summary, root_user, tool, hidden("child-hidden-seed")], + ); + if let LogEntry::AnnotatedSegmentStart { compacted_from, .. } = &mut start { + *compacted_from = Some(SegmentOrigin { + segment_id: root, + at_turn_index: 1, + }); + } + store + .create_segment( + session, + compact, + &[ + start, + LogEntry::ToolResultCorrected { + ts: 9, + entry: corrected, + }, + output("child-answer", "new child original"), + LogEntry::AnnotatedUserInput { + ts: 10, + segments: vec![], + extensions: vec![], + history: vec![ + hidden("child-hidden-input"), + message("child-user", LoggedRole::User, "new child user"), + ], + }, + cancelled(), + ], + ) + .unwrap(); + let guard = retention.lock_live(session).unwrap(); + let capture = capture_recording_fs_guarded( + temp.path(), + session, + compact, + Default::default(), + Default::default(), + &guard, + ) + .unwrap(); + let catalog = recording_catalog_fs_guarded( + temp.path(), + session, + compact, + Default::default(), + &guard, + |descriptors| descriptors.to_vec(), + ) + .unwrap(); + assert_eq!( + catalog, + capture + .entries + .iter() + .map(|entry| capture.recording_text(entry.order).unwrap().0.clone()) + .collect::>() + ); + drop(guard); + let expected = [ + ("Eroot-user", root, 1), + ("Ebatch-user", root, 4), + ("Eroot-tool", root, 6), + ("Eyield-gap", root, 7), + ("Eresume-gap", root, 7), + ("Eroot-answer", root, 7), + ("Echild-answer", compact, 4), + ("Echild-user", compact, 6), + ]; + for ((reference, segment, index), entry) in expected.iter().zip(&capture.entries) { + assert_eq!(format!("E{}", entry.entry_id), *reference); + assert_eq!(entry.source.segment_id, segment.to_string()); + assert_eq!(entry.source.entry_index, *index); + } + assert_eq!(capture.entries.len(), expected.len() + 1); // final cancelled boundary at child gap 7 + assert_eq!(capture.entries.last().unwrap().source.entry_index, 7); + assert!( + capture + .recording_text(2) + .unwrap() + .2 + .contains("corrected full original") + ); + let reader = read(temp.path(), session, compact); + for (entry, descriptor) in capture.entries.iter().zip(reader.descriptors()) { + assert_eq!(capture.recording_text(entry.order).unwrap().0, descriptor); + } + let decoded = CaptureDto::decode( + &capture.encode(Default::default()).unwrap(), + Default::default(), + ) + .unwrap(); + assert_eq!(decoded, capture); + let slice = decoded + .recording_slice( + RecordingPosition { + canonical_order: 2, + byte_offset: 0, + }, + 5, + 16 * 1024, + ) + .unwrap(); + assert_eq!( + slice + .capture + .entries + .iter() + .map(|e| e.source.entry_index) + .collect::>(), + vec![6, 7, 7, 7, 4] + ); + slice.capture.validate(Default::default()).unwrap(); + let mut alias = capture.clone(); + alias.entries[1].source.entry_index = 1; + assert!(matches!( + alias.validate(Default::default()), + Err(CaptureError::Invalid("duplicate source position")) + )); + let mut reverse = capture.clone(); + reverse.entries[2].source.entry_index = 2; + assert!(matches!( + reverse.validate(Default::default()), + Err(CaptureError::Invalid("recording source position order")) + )); + let mut reversed_gap = capture; + reversed_gap.entries[3].source.entry_index = 5; + assert!(matches!( + reversed_gap.validate(Default::default()), + Err(CaptureError::Invalid("recording source position order")) + )); +} + +#[test] +fn recording_catalog_guarded_is_descriptor_only_beyond_body_capture_budget_in_both_layouts() { + use crate::capture::*; + for worker_layout in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("source"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let mut records = vec![seed( + session, + vec![ + system_context("hidden", "private", crate::InputCompactionUse::Retain), + message("u", LoggedRole::User, "request"), + ], + )]; + records.push(invoke()); + for i in 0..130 { + records.push(output(&format!("a-{i}"), &"漢字🍎\n\"".repeat(400))); + } + records.push(cancelled()); + if worker_layout { + WorkerSessionStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } else { + FsStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } + let guard = retention.lock_live(session).unwrap(); + let inspect = |descriptors: &[PublicMaterialDescriptor]| descriptors.to_vec(); + let catalog = if worker_layout { + recording_catalog_worker_guarded( + &root, + session, + segment, + Default::default(), + &guard, + inspect, + ) + } else { + recording_catalog_fs_guarded( + &root, + session, + segment, + Default::default(), + &guard, + inspect, + ) + } + .unwrap(); + assert_eq!(catalog.len(), 132); // user, 130 answers, typed Host boundary + assert_eq!(catalog[0].identity.entry_ref, "Eu"); + assert_eq!(catalog[131].origin, PublicMaterialOrigin::HostRunBoundary); + assert!(catalog.iter().map(|d| d.total_bytes).sum::() > 128 * 1024); + assert!( + catalog + .iter() + .enumerate() + .all(|(i, d)| d.canonical_order == i as u64) + ); + // This is an admission scan, not a full body DTO with a huge transient ceiling. + let small = CaptureLimits { + max_bytes: 8192, + ..Default::default() + }; + let full = if worker_layout { + capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + small, + &guard, + ) + } else { + capture_recording_fs_guarded(&root, session, segment, Default::default(), small, &guard) + }; + assert!(matches!(full, Err(PublicMaterialError::ResourceLimit))); + // Descriptor identities/order/digests are exactly those of the bounded body path. + for order in [0, 1, 130, 131] { + let start = RecordingPosition { + canonical_order: order, + byte_offset: 0, + }; + let slice = if worker_layout { + capture_recording_slice_worker_guarded( + &root, + session, + segment, + Default::default(), + start, + 1, + 4096, + &guard, + ) + } else { + capture_recording_slice_fs_guarded( + &root, + session, + segment, + Default::default(), + start, + 1, + 4096, + &guard, + ) + } + .unwrap(); + assert_eq!( + slice.capture.recording_text(order).unwrap().0, + &catalog[order as usize] + ); + assert!(serde_json::to_vec(&slice).unwrap().len() <= 4096); + } + } +} + +#[test] +fn recording_catalog_callback_keeps_same_external_guard_and_persisted_selector() { + use crate::capture::*; + use crate::{ + WorkerActiveSegmentRef, WorkerAggregateStore, WorkerMetadata, WorkerMetadataStore, + }; + for worker_layout in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("source"); + let control = temp.path().join("control"); + let retention = crate::retention::SessionRetentionStore::new(&control); + let session = crate::new_session_id(); + let selected = crate::new_segment_id(); + let unrelated_newer = crate::new_segment_id(); + for (segment, id) in [(selected, "selected"), (unrelated_newer, "not-active")] { + let records = [seed(session, vec![message(id, LoggedRole::User, id)])]; + if worker_layout { + WorkerSessionStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } else { + FsStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } + } + let metadata_root = temp.path().join("worker-metadata"); + let metadata = WorkerAggregateStore::new(&metadata_root, "test-worker").unwrap(); + let guard = retention.lock_live(session).unwrap(); + metadata + .write(&WorkerMetadata::new( + "test-worker", + Some(WorkerActiveSegmentRef::active_segment(session, selected)), + )) + .unwrap(); + // Simulate with_recording_boundary: persisted pointer resolution and descriptor + // mapping share exactly the same guard. Neither call selects the newest file. + let active = WorkerAggregateStore::open_read_only(&metadata_root, "test-worker") + .unwrap() + .read_read_only() + .unwrap() + .unwrap() + .active + .unwrap(); + assert_eq!(active.session_id, session); + let inspect = |descriptors: &[PublicMaterialDescriptor]| -> Result, &'static str> { + // Direct non-blocking check of the exact external fence, during callback. + // Publishers, active-pointer changes and erasure all use this same lock. + let probe = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(control.join(format!("{session}.lock"))) + .unwrap(); + assert!(!fs4::fs_std::FileExt::try_lock_exclusive(&probe).unwrap()); + assert_eq!(descriptors.len(), 1); + assert_eq!(descriptors[0].identity.segment_id, selected); + assert_eq!(descriptors[0].identity.entry_ref, "Eselected"); + Ok(descriptors.to_vec()) + }; + let catalog = if worker_layout { + recording_catalog_worker_guarded( + &root, + session, + active.segment_id.unwrap(), + Default::default(), + &guard, + inspect, + ) + } else { + recording_catalog_fs_guarded( + &root, + session, + active.segment_id.unwrap(), + Default::default(), + &guard, + inspect, + ) + } + .unwrap() + .unwrap(); + let callback_error = if worker_layout { + recording_catalog_worker_guarded( + &root, + session, + selected, + Default::default(), + &guard, + |_| Err::<(), _>("host-error"), + ) + } else { + recording_catalog_fs_guarded( + &root, + session, + selected, + Default::default(), + &guard, + |_| Err::<(), _>("host-error"), + ) + }; + assert_eq!(callback_error.unwrap(), Err("host-error")); + // The API borrows, not replaces/drops, the caller's guard. + let probe = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(control.join(format!("{session}.lock"))) + .unwrap(); + assert!(!fs4::fs_std::FileExt::try_lock_exclusive(&probe).unwrap()); + drop(guard); + assert!(fs4::fs_std::FileExt::try_lock_exclusive(&probe).unwrap()); + fs4::fs_std::FileExt::unlock(&probe).unwrap(); + retention + .delete( + session, + crate::retention::SessionDeletionIntent::ErasePermanently, + || { + std::fs::remove_dir_all(&root)?; + Ok(()) + }, + ) + .unwrap(); + assert!(matches!( + retention.lock_live(session), + Err(crate::StoreError::SessionErased(_)) + )); + // Descriptor results contain no source body and do not authorize post-erase reads/adoption. + assert_eq!(catalog[0].identity.entry_ref, "Eselected"); + } +} + +#[test] +fn recording_catalog_errors_do_not_invoke_callback_or_relax_resource_and_identity_checks() { + use crate::capture::*; + for worker_layout in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("source"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let records = [seed( + session, + vec![message("u", LoggedRole::User, "request")], + )]; + if worker_layout { + WorkerSessionStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } else { + FsStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } + let guard = retention.lock_live(session).unwrap(); + let other_session = crate::new_session_id(); + let other_guard = retention.lock_live(other_session).unwrap(); + let open = |session, segment, limits, guard| { + if worker_layout { + recording_catalog_worker_guarded(&root, session, segment, limits, guard, |_| { + panic!("callback must not run for rejected catalog") + }) + } else { + recording_catalog_fs_guarded(&root, session, segment, limits, guard, |_| { + panic!("callback must not run for rejected catalog") + }) + } + }; + assert!(matches!( + open(session, segment, Default::default(), &other_guard), + Err(PublicMaterialError::Inconsistent) + )); + for limits in [ + PublicMaterialLimits { + max_storage_bytes: 1, + ..Default::default() + }, + PublicMaterialLimits { + max_record_bytes: 1, + ..Default::default() + }, + PublicMaterialLimits { + max_entries: 0, + ..Default::default() + }, + PublicMaterialLimits { + max_segments: 0, + ..Default::default() + }, + ] { + assert!(matches!( + open(session, segment, limits, &guard), + Err(PublicMaterialError::ResourceLimit) + )); + } + assert!(open(session, crate::new_segment_id(), Default::default(), &guard).is_err()); + // Even a guard matching a claimed Session cannot bless another stored Session. + if worker_layout { + assert!(matches!( + open(other_session, segment, Default::default(), &other_guard), + Err(PublicMaterialError::Inconsistent) + )); + } else { + let impostor = root.join(other_session.to_string()); + std::fs::create_dir(&impostor).unwrap(); + std::fs::copy( + root.join(session.to_string()) + .join(format!("{segment}.jsonl")), + impostor.join(format!("{segment}.jsonl")), + ) + .unwrap(); + assert!(matches!( + open(other_session, segment, Default::default(), &other_guard), + Err(PublicMaterialError::Inconsistent) + )); } } } diff --git a/crates/session-store/src/retention.rs b/crates/session-store/src/retention.rs index 4acecbc96..28870c0eb 100644 --- a/crates/session-store/src/retention.rs +++ b/crates/session-store/src/retention.rs @@ -46,6 +46,13 @@ pub struct SessionRetentionStore { /// Do not recursively acquire the same Session lock inside a guarded callback. pub struct SessionRetentionGuard { _file: File, + session_id: SessionId, +} + +impl SessionRetentionGuard { + pub fn session_id(&self) -> SessionId { + self.session_id + } } impl SessionRetentionStore { @@ -71,7 +78,10 @@ impl SessionRetentionStore { } let file = options.open(self.root.join(format!("{session}.lock")))?; FileExt::lock_exclusive(&file)?; - Ok(SessionRetentionGuard { _file: file }) + Ok(SessionRetentionGuard { + _file: file, + session_id: session, + }) } fn read(&self, session: SessionId) -> Result { diff --git a/crates/standalone/src/job_store.rs b/crates/standalone/src/job_store.rs index 7721dc429..dd69c1568 100644 --- a/crates/standalone/src/job_store.rs +++ b/crates/standalone/src/job_store.rs @@ -25,7 +25,46 @@ use thiserror::Error; #[path = "job_store/legacy_migrations.rs"] mod legacy_migrations; -const SCHEMA_VERSION: i64 = 4; +const SCHEMA_VERSION: i64 = 5; + +/// Availability of the immutable input body, independent of execution outcome. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JobInputDisposition { + Available, + Reclaimed, + Erased, + Invalidated, +} + +impl JobInputDisposition { + pub fn as_str(self) -> &'static str { + match self { + Self::Available => "available", + Self::Reclaimed => "reclaimed", + Self::Erased => "erased", + Self::Invalidated => "invalidated", + } + } +} + +/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JobInputRetirementReason { + Completed, + ExplicitErasure, + LegacyInvalidation, +} + +impl JobInputRetirementReason { + pub fn disposition(self) -> JobInputDisposition { + match self { + Self::Completed => JobInputDisposition::Reclaimed, + Self::ExplicitErasure => JobInputDisposition::Erased, + Self::LegacyInvalidation => JobInputDisposition::Invalidated, + } + } +} #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -67,7 +106,11 @@ impl JobAttemptState { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct JobSnapshot { + /// Receipt metadata. After retirement `input` is null; use the verified loader to execute. pub request: JobRequest, + pub input_digest: String, + pub intent_fingerprint: String, + pub input_disposition: JobInputDisposition, /// Durable Host-issued domain delegation, not model input or Profile identity. pub domain_grant: Option, pub state: JobState, @@ -77,14 +120,27 @@ pub struct JobSnapshot { } impl JobSnapshot { + /// Load only the original available body, checking both content and immutable intent. + pub fn load_executable_request(&self) -> Result<&JobRequest, JobStoreError> { + if self.input_disposition != JobInputDisposition::Available { + return Err(JobStoreError::InvalidTransition("Job input is retired")); + } + self.request.validate()?; + if self.request.input_digest()? != self.input_digest + || self.request.fingerprint()? != self.intent_fingerprint + { + return Err(JobStoreError::Corrupt( + "immutable input or intent fingerprint mismatch".into(), + )); + } + Ok(&self.request) + } + /// Shared Feature-facing read contract; no Workspace or execution identity leaks. pub fn outcome(&self) -> job::JobOutcome { job::JobOutcome { job_id: self.request.job_id.clone(), - input_digest: self - .request - .input_digest() - .expect("validated immutable Job input"), + input_digest: self.input_digest.clone(), attempt_id: self.attempt.attempt_id.clone(), attempt: self.attempt.number, state: match self.state { @@ -204,9 +260,23 @@ impl JobStore { tx.execute_batch(RECONCILIATION_SCHEMA)?; tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; } - SCHEMA_VERSION => {} + 4 | SCHEMA_VERSION => {} other => return Err(JobStoreError::UnsupportedSchema(other)), } + if version < 5 { + tx.execute_batch(INPUT_RETENTION_SCHEMA)?; + let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?; + if has_legacy_archive { + tx.execute_batch("DROP TRIGGER legacy_job_archive_immutable; + CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive + BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END; + CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive + WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available') + OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE + BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;")?; + } + tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; + } // An accepted result is the only evidence of success. Reserved attempts // were never started and remain pending; cancelled attempts never resume. tx.execute_batch( @@ -249,10 +319,7 @@ impl JobStore { [&request.job_id], |row| row.get(0), )?; - if stored != fingerprint - || snapshot.request != request - || snapshot.domain_grant != grant - { + if stored != fingerprint || snapshot.domain_grant != grant { return Err(JobStoreError::IntentConflict(request.job_id)); } tx.commit()?; @@ -334,6 +401,7 @@ impl JobStore { .conn .transaction_with_behavior(TransactionBehavior::Immediate)?; let previous = require_snapshot(&tx, job_id)?; + previous.load_executable_request()?; if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed { return Err(JobStoreError::InvalidTransition( "retry requires known failure", @@ -354,6 +422,81 @@ impl JobStore { Ok(snapshot) } + /// Historical snapshots for one domain Session, including terminal and retired Jobs. + pub(crate) fn list_session_snapshots( + &self, + session_id: &str, + after: Option<&str>, + limit: usize, + ) -> Result, JobStoreError> { + let mut stmt = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_domain_grants g ON g.job_id=i.job_id WHERE json_extract(g.grant_json,'$.session_id')=?1 AND (?2 IS NULL OR i.job_id>?2) ORDER BY i.job_id LIMIT ?3")?; + let ids = stmt + .query_map( + params![session_id, after, limit.clamp(1, 100) as i64], + |r| r.get::<_, String>(0), + )? + .collect::>>()?; + ids.into_iter() + .map(|id| require_snapshot(&self.conn, &id)) + .collect() + } + + /// The Host must fence domain mutation and delivery before erasure/invalidation. + /// This atomically prevents dispatch/retry/result acceptance, without rewriting outcome. + pub(crate) fn reclaim_input( + &mut self, + job_id: &str, + expected_input_digest: &str, + reason: JobInputRetirementReason, + ) -> Result { + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let snapshot = require_snapshot(&tx, job_id)?; + if snapshot.input_digest != expected_input_digest { + return Err(JobStoreError::DigestMismatch); + } + let desired = reason.disposition(); + if snapshot.input_disposition == desired { + tx.commit()?; + reclaim_retired_input_storage(&self.conn)?; + return Ok(snapshot); + } + if reason == JobInputRetirementReason::Completed + && (snapshot.state != JobState::Completed + || snapshot.attempt.state != JobAttemptState::Completed + || snapshot.result.is_none()) + { + return Err(JobStoreError::InvalidTransition( + "reclaim requires known completed result", + )); + } + if snapshot.input_disposition != JobInputDisposition::Available + && reason != JobInputRetirementReason::ExplicitErasure + { + return Err(JobStoreError::InvalidTransition( + "retired input cannot be restored or reclassified", + )); + } + if reason == JobInputRetirementReason::Completed { + snapshot.load_executable_request()?; + } + let mut metadata = snapshot.request.clone(); + metadata.input = Value::Null; + tx.execute( + "UPDATE job_intents SET request_json=?2, input_disposition=?3 WHERE job_id=?1", + params![job_id, serde_json::to_string(&metadata)?, desired.as_str()], + )?; + let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?; + if has_legacy_archive { + tx.execute("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input',json('null')) WHERE job_id=?1", [job_id])?; + } + let snapshot = require_snapshot(&tx, job_id)?; + tx.commit()?; + reclaim_retired_input_storage(&self.conn)?; + Ok(snapshot) + } + pub(crate) fn get(&self, job_id: &str) -> Result { require_snapshot(&self.conn, job_id) } @@ -383,7 +526,7 @@ impl JobStore { let mut statement = self.conn.prepare( "SELECT i.job_id FROM job_intents i JOIN job_attempts a ON a.job_id = i.job_id AND a.number = i.current_attempt - WHERE i.state = 'pending' AND a.state = 'reserved' ORDER BY i.rowid", + WHERE i.state = 'pending' AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid", )?; let ids = statement .query_map([], |row| row.get::<_, String>(0))? @@ -406,6 +549,7 @@ impl JobStore { .transaction_with_behavior(TransactionBehavior::Immediate)?; let snapshot = require_snapshot(&tx, job_id)?; check_attempt(&snapshot, attempt_id)?; + snapshot.load_executable_request()?; if snapshot.state != JobState::Pending || snapshot.attempt.state != JobAttemptState::Reserved { @@ -447,7 +591,7 @@ impl JobStore { .transaction_with_behavior(TransactionBehavior::Immediate)?; let snapshot = require_snapshot(&tx, &bound.job_id)?; check_attempt(&snapshot, &bound.attempt_id)?; - if snapshot.request.input_digest()? != bound.input_digest { + if snapshot.input_digest != bound.input_digest { return Err(JobStoreError::DigestMismatch); } let (digest, encoded) = @@ -466,6 +610,7 @@ impl JobStore { tx.commit()?; return Ok(snapshot); } + snapshot.load_executable_request()?; if snapshot.state != JobState::Pending || snapshot.attempt.state != JobAttemptState::Dispatched { @@ -649,7 +794,7 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, let row = conn .query_row( "SELECT i.request_json, i.state, i.acknowledged, a.attempt_id, a.number, - a.state, a.failure, a.result_json FROM job_intents i JOIN job_attempts a + a.state, a.failure, a.result_json, a.input_digest, i.fingerprint, i.input_disposition FROM job_intents i JOIN job_attempts a ON a.job_id = i.job_id AND a.number = i.current_attempt WHERE i.job_id = ?1", [job_id], |row| { @@ -662,12 +807,27 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, row.get::<_, String>(5)?, row.get::<_, Option>(6)?, row.get::<_, Option>(7)?, + row.get::<_, String>(8)?, + row.get::<_, String>(9)?, + row.get::<_, String>(10)?, )) }, ) .optional()?; row.map( - |(request, state, acknowledged, attempt_id, number, attempt_state, failure, result)| { + |( + request, + state, + acknowledged, + attempt_id, + number, + attempt_state, + failure, + result, + input_digest, + intent_fingerprint, + disposition, + )| { let state = match state.as_str() { "pending" => JobState::Pending, "completed" => JobState::Completed, @@ -693,8 +853,17 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, ))); } }; + let request_value: Value = serde_json::from_str(&request)?; + if request_value.get("input").is_none() { + return Err(JobStoreError::Corrupt( + "stored Job input field is missing".into(), + )); + } Ok(JobSnapshot { - request: serde_json::from_str(&request)?, + request: serde_json::from_value(request_value)?, + input_digest, + intent_fingerprint, + input_disposition: serde_json::from_value(Value::String(disposition))?, domain_grant: conn .query_row( "SELECT grant_json FROM job_domain_grants WHERE job_id = ?1", @@ -719,6 +888,37 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, .transpose() } +// Retirement is already durable before this maintenance step. Failure must be +// retried with the same Job/digest/reason; it must never restore input. VACUUM +// removes historical free-page copies, including stores written before secure_delete. +fn reclaim_retired_input_storage(conn: &Connection) -> Result<(), JobStoreError> { + conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?; + let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?; + if busy != 0 { + return Err(JobStoreError::InvalidTransition( + "input retired; SQLite body cleanup blocked by a reader, retry retirement", + )); + } + Ok(()) +} + +const INPUT_RETENTION_SCHEMA: &str = " +ALTER TABLE job_intents ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available' + CHECK (input_disposition IN ('available','reclaimed','erased','invalidated')); +DROP TRIGGER job_intent_immutable; +CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF + job_id, fingerprint, serialization_key, max_concurrent ON job_intents +BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END; +CREATE TRIGGER job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON job_intents +WHEN ( + ((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated')) + OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased')) + AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') + AND json_type(NEW.request_json,'$.input') = 'null' +) IS NOT TRUE +BEGIN SELECT RAISE(ABORT, 'immutable Job input retirement'); END; +"; + const RECONCILIATION_SCHEMA: &str = " DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) @@ -885,7 +1085,11 @@ mod tests { Some("unconfirmed"), ) .unwrap(); - store.conn.execute_batch("DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap(); + store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable; + ALTER TABLE job_intents DROP COLUMN input_disposition; + CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents + BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END; + DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap(); drop(store); let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); assert_eq!(reopened.get("first").unwrap(), unknown); @@ -911,10 +1115,7 @@ mod tests { JobResultSubmission { job_id: snapshot.request.job_id.clone(), attempt_id: snapshot.attempt.attempt_id.clone(), - input_digest: snapshot - .request - .input_digest() - .expect("validated immutable Job input"), + input_digest: snapshot.input_digest.clone(), result, } } @@ -926,6 +1127,314 @@ mod tests { .unwrap() } + #[test] + fn input_reclaim_reopens_as_receipt_and_exact_replay_never_rehydrates() { + let (dir, mut store) = fixture(); + let original = request("retained"); + let grant = Some(json!({"session_id":"session-a", "consumer":"fixture"})); + let reserved = store + .reserve_granted(original.clone(), grant.clone()) + .unwrap(); + let running = store + .dispatch("retained", &reserved.attempt.attempt_id, 8) + .unwrap(); + let result = submission(&running, json!({"saved":true})); + let completed = store.accept(&result).unwrap(); + store.ack("retained").unwrap(); + let retired = store + .reclaim_input( + "retained", + &completed.input_digest, + JobInputRetirementReason::Completed, + ) + .unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed); + assert!(retired.request.input.is_null()); + assert!(retired.load_executable_request().is_err()); + assert_eq!( + retired.outcome().input_digest, + original.input_digest().unwrap() + ); + assert_eq!(retired.result, completed.result); + drop(store); + let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); + assert_eq!( + reopened + .reserve_granted(original.clone(), grant.clone()) + .unwrap(), + retired + ); + assert_eq!(reopened.accept(&result).unwrap(), retired); + assert_eq!( + reopened + .list_session_snapshots("session-a", None, 1) + .unwrap(), + vec![retired.clone()] + ); + assert!( + reopened + .list_session_snapshots("session-b", None, 100) + .unwrap() + .is_empty() + ); + assert!( + reopened + .list_session_snapshots("session-a", Some("retained"), 100) + .unwrap() + .is_empty() + ); + let mut changed = original.clone(); + changed.input = json!({"changed":true}); + assert!(matches!( + reopened.reserve_granted(changed, grant), + Err(JobStoreError::IntentConflict(_)) + )); + assert!(reopened.retry("retained").is_err()); + assert!( + reopened + .dispatch("retained", &retired.attempt.attempt_id, 8) + .is_err() + ); + assert!(reopened.conn.execute("UPDATE job_intents SET request_json=?1,input_disposition='available' WHERE job_id='retained'", [serde_json::to_string(&original).unwrap()]).is_err()); + let erased = reopened + .reclaim_input( + "retained", + &retired.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(erased.input_disposition, JobInputDisposition::Erased); + assert_eq!(erased.state, JobState::Completed); + assert!(erased.acknowledged); + assert_eq!( + reopened + .reserve_granted(original, erased.domain_grant.clone()) + .unwrap(), + erased + ); + } + + #[test] + fn unfinished_unknown_and_failed_inputs_are_kept_until_explicit_invalidation() { + let (_dir, mut store) = fixture(); + for (id, state) in [ + ("pending", None), + ("unknown", Some(JobAttemptState::Unknown)), + ("failed", Some(JobAttemptState::Failed)), + ] { + let original = request(id); + let snapshot = store.reserve(original.clone()).unwrap(); + if let Some(state) = state { + store.dispatch(id, &snapshot.attempt.attempt_id, 8).unwrap(); + store + .finish( + id, + &snapshot.attempt.attempt_id, + state, + Some("not complete"), + ) + .unwrap(); + } + let before = store.get(id).unwrap(); + assert!( + store + .reclaim_input( + id, + &before.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + assert_eq!(store.get(id).unwrap(), before); + assert_eq!(before.load_executable_request().unwrap(), &original); + assert!( + store + .reclaim_input(id, "wrong", JobInputRetirementReason::ExplicitErasure) + .is_err() + ); + let retired = store + .reclaim_input( + id, + &before.input_digest, + JobInputRetirementReason::LegacyInvalidation, + ) + .unwrap(); + assert_eq!(retired.state, before.state); + assert_eq!(retired.attempt, before.attempt); + assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated); + assert!(store.retry(id).is_err()); + assert!(store.dispatch(id, &before.attempt.attempt_id, 8).is_err()); + assert_eq!(store.reserve(original).unwrap(), retired); + } + } + + #[test] + fn erased_dispatched_input_refuses_delayed_result_without_rewriting_unknown() { + let (dir, mut store) = fixture(); + let running = dispatched(&mut store, request("erase-running")); + let result = submission(&running, json!({"late":true})); + let erased = store + .reclaim_input( + "erase-running", + &running.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert!(store.accept(&result).is_err()); + assert_eq!(erased.state, JobState::Pending); + drop(store); + let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); + let recovered = reopened.get("erase-running").unwrap(); + assert_eq!(recovered.state, JobState::Unknown); + assert_eq!(recovered.input_disposition, JobInputDisposition::Erased); + assert_eq!(recovered.input_digest, running.input_digest); + assert!(reopened.accept(&result).is_err()); + assert!(reopened.retry("erase-running").is_err()); + } + + #[test] + fn available_input_loader_refuses_tamper_missing_body_and_envelope_changes() { + for mutation in [ + "json_set(request_json,'$.input',json('{\"tampered\":true}'))", + "json_remove(request_json,'$.input')", + "json_set(request_json,'$.instruction','changed')", + ] { + let (_dir, mut store) = fixture(); + store.reserve(request("tamper")).unwrap(); + // Simulate external corruption bypassing SQLite's normal write guard. + store + .conn + .execute_batch("DROP TRIGGER job_input_retirement") + .unwrap(); + store + .conn + .execute_batch(&format!("UPDATE job_intents SET request_json={mutation}")) + .unwrap(); + match store.get("tamper") { + Ok(snapshot) => { + assert!(snapshot.load_executable_request().is_err()); + assert!( + store + .dispatch("tamper", &snapshot.attempt.attempt_id, 8) + .is_err() + ); + } + Err(JobStoreError::Json(_) | JobStoreError::Corrupt(_)) => {} + other => panic!("unexpected corrupt input read: {other:?}"), + } + } + } + + #[test] + fn input_retirement_cleanup_failure_is_durable_and_exact_retry_finishes_without_rehydration() { + let (dir, mut store) = fixture(); + let mut original = request("physical-erasure"); + let private_body = "private-input-retirement-original-bytes".repeat(1000); + original.input = json!({"body":private_body}); + let reserved = store.reserve(original.clone()).unwrap(); + // A reader retaining the old SQLite snapshot prevents truncation of the old WAL. + let reader = Connection::open(dir.path().join("jobs.sqlite3")).unwrap(); + reader + .execute_batch("BEGIN; SELECT request_json FROM job_intents;") + .unwrap(); + store.conn.busy_timeout(std::time::Duration::ZERO).unwrap(); + assert!( + store + .reclaim_input( + "physical-erasure", + &reserved.input_digest, + JobInputRetirementReason::ExplicitErasure + ) + .is_err() + ); + let retired = store.get("physical-erasure").unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Erased); + assert!(retired.load_executable_request().is_err()); + reader.execute_batch("ROLLBACK").unwrap(); + drop(reader); + let replay = store + .reclaim_input( + "physical-erasure", + &reserved.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(replay, retired); + assert_eq!(store.reserve(original).unwrap(), retired); + let needle = b"private-input-retirement-original-bytes"; + for entry in std::fs::read_dir(dir.path()).unwrap() { + let path = entry.unwrap().path(); + if path.is_file() { + let bytes = std::fs::read(&path).unwrap(); + assert!( + !bytes.windows(needle.len()).any(|bytes| bytes == needle), + "retired body remained in {}", + path.display() + ); + } + } + } + + #[test] + fn missing_original_null_input_is_not_treated_as_available_null() { + let (_dir, mut store) = fixture(); + let mut original = request("null-original"); + original.input = Value::Null; + let reserved = store.reserve(original).unwrap(); + assert!(reserved.load_executable_request().is_ok()); + store.conn.execute_batch("DROP TRIGGER job_input_retirement; UPDATE job_intents SET request_json=json_remove(request_json,'$.input')").unwrap(); + assert!(store.get("null-original").is_err()); + } + + #[test] + fn input_retirement_migration_preserves_receipts_and_rolls_back_failed_ddl() { + let (dir, mut store) = fixture(); + let original = store.reserve(request("upgrade")).unwrap(); + store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable; + ALTER TABLE job_intents DROP COLUMN input_disposition; + CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents + BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END; + PRAGMA user_version=4; + CREATE TRIGGER job_input_retirement BEFORE UPDATE ON job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;").unwrap(); + drop(store); + let path = dir.path().join("jobs.sqlite3"); + assert!(JobStore::open(&path).is_err()); + let conn = Connection::open(&path).unwrap(); + let version: i64 = conn + .pragma_query_value(None, "user_version", |r| r.get(0)) + .unwrap(); + assert_eq!(version, 4); + let present: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info('job_intents') WHERE name='input_disposition')",[],|r|r.get(0)).unwrap(); + assert!(!present); + conn.execute_batch("DROP TRIGGER job_input_retirement") + .unwrap(); + drop(conn); + let mut reopened = JobStore::open(&path).unwrap(); + assert_eq!(reopened.get("upgrade").unwrap(), original); + assert_eq!( + reopened + .get("upgrade") + .unwrap() + .load_executable_request() + .unwrap(), + &request("upgrade") + ); + let retired = reopened + .reclaim_input( + "upgrade", + &original.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(retired.state, JobState::Pending); + assert!( + reopened + .conn + .execute_batch("UPDATE job_intents SET input_disposition='available'") + .is_err() + ); + } + #[test] fn database_configuration_schema_and_identity_constraints() { let (_dir, mut store) = fixture(); diff --git a/crates/standalone/src/job_store/legacy_migrations.rs b/crates/standalone/src/job_store/legacy_migrations.rs index ec05286cf..f1b8cb539 100644 --- a/crates/standalone/src/job_store/legacy_migrations.rs +++ b/crates/standalone/src/job_store/legacy_migrations.rs @@ -143,6 +143,42 @@ mod tests { (original, receipt, grant) } + #[test] + fn legacy_archive_input_erasure_preserves_fingerprints_and_attempt_receipts() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + seed(&path, 2); + let mut store = JobStore::open(&path).unwrap(); + let before: (String, String) = store + .conn + .query_row( + "SELECT fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + let snapshot = store.get("done").unwrap(); + store + .reclaim_input( + "done", + &snapshot.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + let (body, fingerprint, attempts): (String,String,String) = store.conn.query_row("SELECT request_json,fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap(); + assert!(serde_json::from_str::(&body).unwrap()["input"].is_null()); + assert_eq!((fingerprint, attempts), before); + assert!(store.conn.execute_batch("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input','restored') WHERE job_id='done'").is_err()); + drop(store); + let mut reopened = JobStore::open(&path).unwrap(); + let receipt = reopened + .reserve_granted(request("done"), snapshot.domain_grant) + .unwrap(); + assert_eq!(receipt.state, JobState::Completed); + assert_eq!(receipt.input_disposition, JobInputDisposition::Erased); + assert!(receipt.load_executable_request().is_err()); + } + #[test] fn legacy_versions_preserve_intent_receipts_attempts_grants_and_replay() { for version in [1, 2] { diff --git a/crates/standalone/src/jobs.rs b/crates/standalone/src/jobs.rs index 890eed041..bc375e005 100644 --- a/crates/standalone/src/jobs.rs +++ b/crates/standalone/src/jobs.rs @@ -26,7 +26,10 @@ use job::{JobRequest, JobResultSubmission}; use tokio::sync::watch; use crate::job_store::JobStore; -pub use crate::job_store::{JobAttempt, JobAttemptState, JobSnapshot, JobState, JobStoreError}; +pub use crate::job_store::{ + JobAttempt, JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, + JobState, JobStoreError, +}; #[derive(Debug, thiserror::Error)] pub enum JobServiceError { @@ -40,6 +43,8 @@ pub enum JobServiceError { Capacity, #[error("standalone Job cleanup failed: {0}")] Cleanup(String), + #[error("standalone Job shutdown is incomplete; execution resources are still owned")] + ShutdownIncomplete, } pub(crate) trait JobAttemptFence: Send + Sync { @@ -173,6 +178,36 @@ impl StandaloneJobs { Ok(self.with_store(|store| store.reserve(request))?) } + /// Enumerate historical domain snapshots for a Session, including retired input receipts. + pub fn list_session_snapshots( + &self, + session_id: &str, + after: Option<&str>, + limit: usize, + ) -> Result, JobServiceError> { + self.with_store(|store| store.list_session_snapshots(session_id, after, limit)) + } + + /// Retire body only. The caller fences domain effects/delivered snapshots before erasure. + /// Normal reclamation also requires that execution cleanup has finished. + pub fn reclaim_input( + &self, + job_id: &str, + expected_input_digest: &str, + reason: JobInputRetirementReason, + ) -> Result { + let executions = self.executions.lock().expect("Job executions poisoned"); + if reason == JobInputRetirementReason::Completed + && executions + .jobs + .get(job_id) + .is_some_and(|execution| !execution.task.is_finished()) + { + return Err(JobServiceError::Capacity); + } + self.with_store(|store| store.reclaim_input(job_id, expected_input_digest, reason)) + } + pub fn get(&self, job_id: &str) -> Result { Ok(self.with_store(|store| store.get(job_id))?) } @@ -274,6 +309,12 @@ impl StandaloneJobs { job_id: &str, client: Option>, ) -> Result<(), JobServiceError> { + // Preparation owns copies of the immutable input and domain adapter too. Exclude + // owner shutdown before creating them, not only before publishing the task. + let mut executions = self.executions.lock().expect("Job executions poisoned"); + if executions.closed { + return Err(JobServiceError::Closed); + } let client = client.or_else(|| { self.model_client .lock() @@ -282,16 +323,14 @@ impl StandaloneJobs { .map(|c| c.clone_boxed()) }); let snapshot = self.get(job_id)?; + snapshot.load_executable_request()?; let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds)); let deadline = tokio::time::Instant::now() + timeout; let fence: Arc = Arc::new(BoundAttemptFence { store: self.store.clone(), job_id: job_id.into(), attempt_id: snapshot.attempt.attempt_id.clone(), - input_digest: snapshot - .request - .input_digest() - .map_err(JobStoreError::from)?, + input_digest: snapshot.input_digest.clone(), deadline, }); let grant = match ( @@ -311,10 +350,6 @@ impl StandaloneJobs { let prepared = worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant) .map_err(|error| JobServiceError::Configuration(error.to_string()))?; - let mut executions = self.executions.lock().expect("Job executions poisoned"); - if executions.closed { - return Err(JobServiceError::Closed); - } executions.jobs.retain(|_, run| { !(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(())))) }); @@ -367,10 +402,7 @@ impl StandaloneJobs { store: store.clone(), job_id: id.clone(), attempt_id: attempt_id.clone(), - input_digest: snapshot - .request - .input_digest() - .map_err(JobStoreError::from)?, + input_digest: snapshot.input_digest.clone(), deadline, }); let (cancel, mut cancellation) = watch::channel(false); @@ -446,23 +478,25 @@ impl StandaloneJobs { .clone() .expect("completion"); result.map_err(JobServiceError::Cleanup)?; - let task = { - let mut executions = self.executions.lock().expect("Job executions poisoned"); - if executions - .jobs - .get(job_id) - .is_some_and(|run| run.attempt_id == attempt) - { - executions.jobs.remove(job_id) - } else { - None - } - }; - if let Some(execution) = task { - execution - .task - .await - .map_err(|error| JobServiceError::Cleanup(error.to_string()))??; + // Completion is published immediately before the task returns. Do not remove + // its ownership record until the runtime confirms that return (and future drop). + while self + .executions + .lock() + .expect("Job executions poisoned") + .jobs + .get(job_id) + .is_some_and(|run| run.attempt_id == attempt && !run.task.is_finished()) + { + tokio::task::yield_now().await; + } + let mut executions = self.executions.lock().expect("Job executions poisoned"); + if executions + .jobs + .get(job_id) + .is_some_and(|run| run.attempt_id == attempt) + { + executions.jobs.remove(job_id); } } self.get(job_id) @@ -476,33 +510,63 @@ impl StandaloneJobs { } } - pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> { + /// Initiate owner shutdown without blocking a synchronous caller. Success confirms all + /// local executions have returned, including request/history and Feature cleanup. An + /// incomplete/error result retains the service resources and must retain the Host lease. + pub(crate) fn try_shutdown(&self) -> Result<(), JobServiceError> { self.close(); - let tasks = { - let mut executions = self.executions.lock().expect("Job executions poisoned"); - std::mem::take(&mut executions.jobs) - }; - let mut error = None; - for execution in tasks.into_values() { - match execution.task.await { - Ok(Ok(())) => {} - Ok(Err(failure)) => { - error = Some(failure); - } - Err(failure) => { - error = Some(JobServiceError::Cleanup(failure.to_string())); + let mut executions = self.executions.lock().expect("Job executions poisoned"); + for execution in executions.jobs.values() { + if !execution.task.is_finished() { + return Err(JobServiceError::ShutdownIncomplete); + } + match execution.complete.borrow().as_ref() { + Some(Ok(())) => {} + Some(Err(error)) => return Err(JobServiceError::Cleanup(error.clone())), + None => { + return Err(JobServiceError::Cleanup( + "Internal Worker ended without cleanup confirmation".into(), + )); } } } - // Close SQLite before the Host releases its lease. Cloned handles may - // outlive the Host but cannot mutate or retain the live connection. + executions.jobs.clear(); + // Retained external handles share these slots; they cannot retain a live store, + // provider or model client after successful owner shutdown. self.store.lock().expect("Job store poisoned").take(); self.domain.lock().expect("Job domain poisoned").take(); self.model_client .lock() .expect("Job model transport poisoned") .take(); - error.map_or(Ok(()), Err) + Ok(()) + } + + pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> { + self.close(); + let completions: Vec<_> = self + .executions + .lock() + .expect("Job executions poisoned") + .jobs + .values() + .map(|execution| execution.complete.clone()) + .collect(); + // Never take execution ownership out of the shared service across an await: a + // cancelled shutdown future or a concurrent close must still see the barrier. + for mut completion in completions { + completion + .wait_for(Option::is_some) + .await + .map_err(|error| JobServiceError::Cleanup(error.to_string()))?; + } + loop { + match self.try_shutdown() { + // Only the task's final return follows its completion notification. + Err(JobServiceError::ShutdownIncomplete) => tokio::task::yield_now().await, + result => return result, + } + } } } @@ -574,7 +638,8 @@ impl JobAttemptFence for BoundAttemptFence { if snapshot.state != JobState::Pending || snapshot.attempt.state != JobAttemptState::Dispatched || snapshot.attempt.attempt_id != self.attempt_id - || snapshot.request.input_digest().map_err(|e| e.to_string())? != self.input_digest + || snapshot.input_digest != self.input_digest + || snapshot.input_disposition != JobInputDisposition::Available { return Err("Job domain grant is not bound to the current live attempt".into()); } diff --git a/crates/standalone/src/jobs/tests.rs b/crates/standalone/src/jobs/tests.rs index cc5d00f7b..06325c054 100644 --- a/crates/standalone/src/jobs/tests.rs +++ b/crates/standalone/src/jobs/tests.rs @@ -468,3 +468,104 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han assert!(matches!(operation, Err(JobServiceError::Closed))); } } + +#[tokio::test] +async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() { + // Actual generic runner + SQLite; only the provider stream is scripted/blocked. + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("jobs.sqlite3"); + let jobs = StandaloneJobs::open(&path, temp.path().into()).unwrap(); + let reserved = jobs.request(request("cancel-shutdown-wait")).unwrap(); + let model = DomainTestClient { + stall: true, + ..Default::default() + }; + jobs.start_with_model_client(&reserved.request.job_id, model.clone()) + .await + .unwrap(); + await_model(&model).await; + { + let shutdown = jobs.shutdown(); + tokio::pin!(shutdown); + assert!(futures::poll!(shutdown.as_mut()).is_pending()); + // Drop the caller's waiter while cancellation/cleanup is still pending. + } + assert!(matches!( + jobs.try_shutdown(), + Err(JobServiceError::ShutdownIncomplete) + )); + assert!(jobs.store.lock().unwrap().is_some()); + assert!(matches!( + jobs.request(request("after-shutdown")), + Err(JobServiceError::Closed) + )); + tokio::time::timeout(Duration::from_secs(5), jobs.shutdown()) + .await + .unwrap() + .unwrap(); + assert!(jobs.store.lock().unwrap().is_none()); + let reopened = StandaloneJobs::open(&path, temp.path().into()).unwrap(); + let unknown = reopened.get(&reserved.request.job_id).unwrap(); + assert_eq!(unknown.attempt.state, JobAttemptState::Unknown); + assert_eq!(unknown.input_disposition, JobInputDisposition::Available); + assert_eq!(unknown.request, reserved.request); + assert_eq!(unknown.input_digest, reserved.input_digest); + assert!(reopened.retry(&unknown.request.job_id).is_err()); + reopened.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn input_retirement_service_fences_live_attempt_and_preserves_session_receipts() { + // Production service + SQLite + attempt/result fences, without a model or Worker fixture. + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("jobs.sqlite3"); + let jobs = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap(); + let original = request("input-retirement"); + let grant = json!({"session_id":"session-a"}); + let reserved = jobs + .request_granted(original.clone(), grant.clone()) + .unwrap(); + jobs.with_store(|store| store.dispatch("input-retirement", &reserved.attempt.attempt_id, 8)) + .unwrap(); + let fence = BoundAttemptFence { + store: jobs.store.clone(), + job_id: "input-retirement".into(), + attempt_id: reserved.attempt.attempt_id.clone(), + input_digest: reserved.input_digest.clone(), + deadline: tokio::time::Instant::now() + Duration::from_secs(10), + }; + assert!(fence.ensure_live().is_ok()); + assert!( + jobs.reclaim_input( + "input-retirement", + &reserved.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + let erased = jobs + .reclaim_input( + "input-retirement", + &reserved.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert!(fence.ensure_live().is_err()); + assert!( + sink(&jobs, "input-retirement", &reserved.attempt.attempt_id) + .submit(result("input-retirement", &reserved.attempt.attempt_id)) + .is_err() + ); + assert!(jobs.start("input-retirement").await.is_err()); + assert_eq!( + jobs.list_session_snapshots("session-a", None, 100).unwrap(), + vec![erased.clone()] + ); + assert!( + jobs.list_session_snapshots("session-b", None, 100) + .unwrap() + .is_empty() + ); + assert_eq!(jobs.request_granted(original, grant).unwrap(), erased); + jobs.shutdown().await.unwrap(); +} diff --git a/crates/standalone/src/subjektiv.rs b/crates/standalone/src/subjektiv.rs index a5cea4379..353538072 100644 --- a/crates/standalone/src/subjektiv.rs +++ b/crates/standalone/src/subjektiv.rs @@ -54,6 +54,8 @@ pub enum SubjectError { Journal(#[from] subjektiv::journal::JournalError), #[error(transparent)] Storage(#[from] feature_storage::FeatureStorageError), + #[error("local Subject Job shutdown: {0}")] + JobShutdown(#[from] crate::jobs::JobServiceError), #[error("local Subject storage metadata: {0}")] Json(#[from] serde_json::Error), #[error("local Subject storage permission or identity is invalid: {0}")] @@ -338,8 +340,16 @@ impl SubjectConnection { } } pub fn close(&mut self) -> Result<(), SubjectError> { - self.host.jobs.lock().expect("Subject Jobs poisoned").take(); self.host.live.store(false, Ordering::Release); + { + let mut jobs = self.host.jobs.lock().expect("Subject Jobs poisoned"); + if let Some(service) = jobs.as_ref() { + // Signal cancellation, but retain the handle and exclusive lease until + // the generic owner confirms local request/history/resource teardown. + service.try_shutdown()?; + } + jobs.take(); + } self.host.catalog.manager.shutdown()?; if let Some(file) = &self.lease { file.unlock()?; diff --git a/crates/standalone/src/subjektiv/journal.rs b/crates/standalone/src/subjektiv/journal.rs index 5b2c880a4..85bbe26f1 100644 --- a/crates/standalone/src/subjektiv/journal.rs +++ b/crates/standalone/src/subjektiv/journal.rs @@ -5,9 +5,12 @@ //! recovery authority; shutdown does not delete either original Sessions or pending requests. use super::*; -use crate::jobs::{JobAttemptState, JobSnapshot, JobState}; +use crate::jobs::{ + JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, JobState, +}; +use session_store::capture::{CaptureDto, CaptureLimits, RecordingPosition}; use subjektiv::api::JobAttemptBinding; -use subjektiv::journal::recording::JournalMaterial; +use subjektiv::journal::recording::JournalInput; use subjektiv::journal::transport::*; use subjektiv::journal::*; use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature}; @@ -16,7 +19,6 @@ use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeat mod tests; pub(super) const PURPOSE: &str = "subjektiv_journal"; -const INPUT_BUDGET: usize = 48 * 1024; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -27,6 +29,7 @@ struct LocalJournalGrant { session_id: String, journal_id: String, ranges: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] origins: Vec, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -35,11 +38,6 @@ struct SourceOrigin { pointer: JournalSourcePointer, origin: session_store::public_material::PublicMaterialOrigin, } -impl LocalJournalGrant { - fn input(&self) -> serde_json::Value { - serde_json::json!({"selected_ranges": self.ranges, "origins": self.origins}) - } -} impl StandaloneSubjects { /// Explicit operator erasure, separate from ordinary Worker removal. Requires the @@ -84,32 +82,29 @@ impl StandaloneSubjects { self.state_root.clone(), ) .map_err(|error| SubjectError::InvalidScope(error.to_string()))?; - for snapshot in jobs - .pending() - .map_err(|e| SubjectError::InvalidScope(e.to_string()))? - .into_iter() - .chain( - jobs.unknown_granted() - .map_err(|e| SubjectError::InvalidScope(e.to_string()))?, - ) - { - if snapshot.domain_grant.as_ref().is_some_and(|grant| { - grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id) - }) { + let mut after = None; + loop { + let page = jobs + .list_session_snapshots(session_id, after.as_deref(), 100) + .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; + if page.is_empty() { + break; + } + for snapshot in &page { jobs.cancel(&snapshot.request.job_id) .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; + jobs.reclaim_input( + &snapshot.request.job_id, + &snapshot.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; + if snapshot.state == JobState::Completed && !snapshot.acknowledged { + jobs.acknowledge(&snapshot.request.job_id) + .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; + } } - } - for snapshot in jobs - .unacknowledged_results() - .map_err(|e| SubjectError::InvalidScope(e.to_string()))? - { - if snapshot.domain_grant.as_ref().is_some_and(|grant| { - grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id) - }) { - jobs.acknowledge(&snapshot.request.job_id) - .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; - } + after = page.last().map(|job| job.request.job_id.clone()); } session_store::FsStore::new_with_retention( path, @@ -590,17 +585,28 @@ impl LocalSubjectHost { self.ensure_live()?; let jobs = self.jobs()?; for saved in jobs.unacknowledged_results().map_err(service_error)? { - if self.journal_grant(&saved).is_ok() { + if self.journal_grant_scope(&saved).is_ok() { self.complete_journal_job(&jobs, &saved)?; } } for pending in jobs.pending().map_err(service_error)? { - if self.journal_grant(&pending).is_ok() { + if pending.request.purpose != PURPOSE { + continue; + } + self.journal_grant_scope(&pending)?; + if pending.request.input.get("capture").is_some() { + self.journal_grant(&pending)?; return Ok(Some((jobs, pending.request.job_id))); } + // Old descriptor jobs never acquire new execution authority. Their immutable + // identity/input is retired, not upgraded in place; saved checkpoint wins. + let replacement = self.replace_legacy_journal_job(&jobs, &pending)?; + if let Some(id) = replacement { + return Ok(Some((jobs, id))); + } } for unknown in jobs.unknown_granted().map_err(service_error)? { - let Ok(grant) = self.journal_grant(&unknown) else { + let Ok(grant) = self.journal_grant_scope(&unknown) else { continue; }; let context = authority(self, &grant.session_id)?; @@ -685,32 +691,96 @@ impl LocalSubjectHost { } } } - let materials = self.select_journal_materials(&context, &journal.id)?; - if materials.is_empty() { + let mut retained = None; + // Saved body snapshots always win over descriptor migration. In particular, + // a failed continuation must not recapture its original text even if an + // invalidated legacy grant still bounds a larger unprocessed suffix. + for legacy_pass in [false, true] { + let mut after_job = None; + loop { + let page = jobs + .list_session_snapshots( + &journal.source_session_id, + after_job.as_deref(), + 100, + ) + .map_err(service_error)?; + if page.is_empty() { + break; + } + for saved in &page { + if saved.request.purpose != PURPOSE { + continue; + } + let legacy = saved.input_disposition + == JobInputDisposition::Invalidated + || (saved.input_disposition == JobInputDisposition::Available + && saved.request.input.get("capture").is_none()); + if legacy != legacy_pass + || (!legacy + && saved.input_disposition != JobInputDisposition::Available) + { + continue; + } + let saved_grant = self.journal_grant_scope(saved)?; + if saved_grant.journal_id != journal.id { + continue; + } + if legacy { + if let Some(id) = self.replace_legacy_journal_job(&jobs, saved)? { + return Ok(Some((jobs, id))); + } + continue; + } + self.journal_grant(saved)?; + let original = JournalInput::decode(&saved.request.input) + .map_err(journal_error)?; + let continuation = self.build_journal_input( + &context, + &journal.id, + Some(&original.capture), + )?; + if !continuation.selected_ranges.is_empty() { + retained = Some(continuation); + break; + } + if saved.state == JobState::Completed && saved.acknowledged { + // Recover the acknowledgement-before-reclamation gap without + // turning a retained receipt into an executable empty input. + jobs.reclaim_input( + &saved.request.job_id, + &saved.input_digest, + JobInputRetirementReason::Completed, + ) + .map_err(service_error)?; + } + } + if retained.is_some() { + break; + } + after_job = page.last().map(|job| job.request.job_id.clone()); + } + if retained.is_some() { + break; + } + } + let input = match retained { + Some(input) => input, + None => self.build_journal_input(&context, &journal.id, None)?, + }; + + if input.selected_ranges.is_empty() { continue; } - let ranges = materials - .into_iter() - .map(|material| material.range) - .collect::>(); - let origins = self.journal_origins(&ranges)?; let grant = LocalJournalGrant { writer_binding_id: Some(self.writer.binding_id.clone()), binding: self.binding.clone(), session_id: journal.source_session_id, journal_id: journal.id.clone(), - ranges, - origins, - }; - let request = job::JobRequest { - job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()), - purpose: PURPOSE.into(), input_ref: format!("subjektiv://{}/journal/{}", self.binding.subject_id, journal.id), - input: grant.input(), - instruction: "ReadJournalState first. Input descriptors are the frozen canonical public ranges authorized for this Job, not already read material. Read every selected source byte with bounded ReadJournalText before recording; then read initial/current position and open text in bounded pages. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(), - profile: "builtin:subjektiv-journal-recording".into(), - serialization_key: Some(format!("subjektiv-journal:{}", journal.id)), - limits: job::JobLimits { timeout_seconds: 600, ..Default::default() }, + ranges: input.selected_ranges.clone(), + origins: vec![], }; + let request = self.journal_request(&journal.id, &input)?; let saved = jobs .request_granted( request, @@ -727,6 +797,138 @@ impl LocalSubjectHost { } } + fn replace_legacy_journal_job( + &self, + jobs: &crate::jobs::StandaloneJobs, + old: &JobSnapshot, + ) -> Result, SubjektivHostError> { + let grant = self.journal_grant_scope(old)?; + if old.input_disposition == JobInputDisposition::Available { + old.load_executable_request() + .map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?; + if old.request.input + != serde_json::json!({"selected_ranges":grant.ranges,"origins":grant.origins}) + { + return Err(SubjektivHostError::InvalidArgument( + "invalid legacy Journal descriptor input".into(), + )); + } + } else if old.input_disposition != JobInputDisposition::Invalidated { + return Err(SubjektivHostError::InvalidArgument( + "legacy Journal input is unavailable".into(), + )); + } + let context = authority(self, &grant.session_id)?; + if let Some(active) = self + .catalog + .store + .active_journal_generation(&context, &grant.journal_id) + .map_err(journal_error)? + { + if active.job.job_id != old.request.job_id { + return Err(SubjektivHostError::InvalidArgument( + "another generation owns legacy migration".into(), + )); + } + self.catalog + .store + .close_journal_generation(&context, &active.id, &active.job) + .map_err(journal_error)?; + } + jobs.cancel(&old.request.job_id).map_err(service_error)?; + let current = self + .catalog + .store + .read_journal(&context, &grant.journal_id) + .map_err(journal_error)?; + let end = grant + .ranges + .last() + .ok_or_else(|| SubjektivHostError::InvalidArgument("empty legacy selection".into()))?; + let end_position = if end.bytes.end == end.source.total_bytes { + (end.source.order + 1, 0) + } else { + (end.source.order, end.bytes.end) + }; + let replacement = if (current.process_order, current.process_byte) >= end_position { + None + } else { + let mut capture = self.recording_capture(&grant.session_id)?; + capture + .entries + .retain(|entry| grant.ranges.iter().any(|r| r.source.order == entry.order)); + for entry in &mut capture.entries { + let range = grant + .ranges + .iter() + .find(|r| r.source.order == entry.order) + .unwrap(); + let session_store::capture::CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + .. + } = &mut entry.body + else { + unreachable!() + }; + if source_descriptor(&grant.session_id, descriptor) != range.source + || !grant.origins.iter().any(|origin| { + origin.pointer == range.source.pointer && origin.origin == descriptor.origin + }) + || !text.is_char_boundary(range.bytes.start as usize) + || !text.is_char_boundary(range.bytes.end as usize) + { + return Err(SubjektivHostError::InvalidArgument( + "legacy original source identity/range changed".into(), + )); + } + *text = text[range.bytes.start as usize..range.bytes.end as usize].to_owned(); + *byte_offset = range.bytes.start; + } + let input = self.build_journal_input(&context, &grant.journal_id, Some(&capture))?; + let request = self.journal_request(&grant.journal_id, &input)?; + let next_grant = LocalJournalGrant { + ranges: input.selected_ranges.clone(), + origins: vec![], + ..grant + }; + Some( + jobs.request_granted( + request, + serde_json::to_value(next_grant) + .map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?, + ) + .map_err(service_error)? + .request + .job_id, + ) + }; + jobs.reclaim_input( + &old.request.job_id, + &old.input_digest, + JobInputRetirementReason::LegacyInvalidation, + ) + .map_err(service_error)?; + Ok(replacement) + } + + fn journal_request( + &self, + journal_id: &str, + input: &JournalInput, + ) -> Result { + Ok(job::JobRequest { + job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()), purpose: PURPOSE.into(), + input_ref: format!("subjektiv://{}/journal/{journal_id}", self.binding.subject_id), + input: serde_json::to_value(input).map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?, + instruction: "Input contains the frozen canonical public source bodies, provenance and byte ranges. Use only this saved input, never re-fetch Source. ReadJournalState first and read initial/current Journal position/open text in bounded pages if needed. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(), + profile: "builtin:subjektiv-journal-recording".into(), + serialization_key: Some(format!("subjektiv-journal:{journal_id}")), + limits: job::JobLimits { timeout_seconds: 600, ..Default::default() }, + }) + } + fn pin_journal_session(&self, session: &str) -> Result<(), SubjektivHostError> { let path = crate::subjektiv_sessions::authorized_sessions_path( &self.catalog.state_root, @@ -869,11 +1071,16 @@ impl LocalSubjectHost { Ok(()) } - fn material_reader( + fn with_recording_boundary( &self, session: &str, - ) -> Result - { + read: impl FnOnce( + &Path, + session_store::SessionId, + session_store::SegmentId, + &session_store::retention::SessionRetentionGuard, + ) -> Result, + ) -> Result { let sessions = crate::subjektiv_sessions::authorized_sessions_path( &self.catalog.state_root, &self.binding.scope_id, @@ -906,6 +1113,11 @@ impl LocalSubjectHost { .ok_or_else(|| SubjektivHostError::InvalidArgument("invalid Session storage".into()))? .join("worker") .join(&record.storage_key); + let session_id = session + .parse() + .map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?; + let retention = crate::store::session_retention(&self.catalog.state_root); + let guard = retention.lock_live(session_id).map_err(source_error)?; // This is the persisted active selector, not the newest filename/mtime or a model claim. let metadata = session_store::WorkerAggregateStore::open_read_only(metadata_root, &record.storage_key) @@ -933,154 +1145,203 @@ impl LocalSubjectHost { "no attested active lineage for the retained Session".into(), ) })?; - session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention( - &sessions, - session - .parse() - .map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?, - active, - session_store::public_material::PublicMaterialLimits { - max_segments: 256, - max_entries: 100_000, - max_record_bytes: 64 * 1024 * 1024, - }, - crate::store::session_retention(&self.catalog.state_root), - ) - .map_err(|error| SubjektivHostError::Unavailable(error.to_string())) + read(&sessions, session_id, active, &guard) + } + + fn recording_capture(&self, session: &str) -> Result { + self.with_recording_boundary(session, |sessions, session_id, active, guard| { + session_store::capture::capture_recording_fs_guarded( + sessions, + session_id, + active, + session_store::public_material::PublicMaterialLimits::default(), + CaptureLimits { + max_bytes: 128 * 1024 * 1024, + max_entries: 100_000, + max_text_bytes: 64 * 1024 * 1024, + ..Default::default() + }, + guard, + ) + .map_err(|e| SubjektivHostError::Unavailable(e.to_string())) + }) } fn journal_sources( &self, session: &str, ) -> Result, SubjektivHostError> { - let reader = self.material_reader(session)?; - Ok(reader - .descriptors() - .map(|descriptor| JournalSourceRange { - source: source_descriptor(session, descriptor), - bytes: JournalByteRange { - start: 0, - end: descriptor.total_bytes, + self.with_recording_boundary(session, |sessions, session_id, active, guard| { + session_store::capture::recording_catalog_fs_guarded( + sessions, + session_id, + active, + session_store::public_material::PublicMaterialLimits::default(), + guard, + |descriptors| { + descriptors + .iter() + .map(|descriptor| JournalSourceRange { + source: source_descriptor(session, descriptor), + bytes: JournalByteRange { + start: 0, + end: descriptor.total_bytes, + }, + }) + .collect() }, - }) - .collect()) - } - - fn source_reader( - &self, - source: &JournalSource, - ) -> Result - { - let sessions = crate::subjektiv_sessions::authorized_sessions_path( - &self.catalog.state_root, - &self.binding.scope_id, - &self.binding.subject_id, - &self.catalog.store, - &source.session_id, - )?; - // Frozen, already admitted original pointers do not follow the Worker's later active - // Session/Segment. This reads that original lineage and still checks descriptor identity, - // canonical order and digest below; the shared service enforces the frozen byte range. - session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention( - &sessions, - source - .session_id - .parse() - .map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?, - source.pointer.segment_id, - session_store::public_material::PublicMaterialLimits { - max_segments: 256, - max_entries: 100_000, - max_record_bytes: 64 * 1024 * 1024, - }, - crate::store::session_retention(&self.catalog.state_root), - ) - .map_err(|error| SubjektivHostError::Unavailable(error.to_string())) - } - - fn journal_origins( - &self, - ranges: &[JournalSourceRange], - ) -> Result, SubjektivHostError> { - if ranges.is_empty() { - return Err(SubjektivHostError::InvalidArgument( - "empty material selection".into(), - )); - } - ranges - .iter() - .map(|range| { - let reader = self.source_reader(&range.source)?; - reader - .descriptors() - .find(|descriptor| { - source_descriptor(&range.source.session_id, descriptor) == range.source - }) - .map(|descriptor| SourceOrigin { - pointer: range.source.pointer.clone(), - origin: descriptor.origin.clone(), - }) - .ok_or_else(|| { - SubjektivHostError::InvalidArgument( - "source provenance/identity changed".into(), - ) - }) - }) - .collect() - } - - fn journal_source_window( - &self, - source: &JournalSource, - offset: u64, - max_bytes: usize, - ) -> Result { - let reader = self.source_reader(source)?; - let descriptor = reader.descriptors().find(|descriptor| source_descriptor(&source.session_id, descriptor) == *source) - .ok_or_else(|| SubjektivHostError::InvalidArgument("canonical source identity/order/digest changed or is outside the authorized Session".into()))?; - reader - .read_content(descriptor, offset, max_bytes) - .map(|window| window.content) - .map_err(|error| SubjektivHostError::Unavailable(error.to_string())) - } - - fn journal_source_material( - &self, - source: &JournalSource, - start: u64, - limit: usize, - ) -> JournalResult { - let text = self - .journal_source_window(source, start, limit.max(4)) - .map_err(|error| JournalError::Conflict(error.to_string()))?; - Ok(JournalMaterial { - range: JournalSourceRange { - source: source.clone(), - bytes: JournalByteRange { - start, - end: start + text.len() as u64, - }, - }, - text, + ) + .map_err(|e| SubjektivHostError::Unavailable(e.to_string())) }) } - fn select_journal_materials( + fn build_journal_input( &self, context: &JournalHostContext, id: &str, - ) -> Result, SubjektivHostError> { - subjektiv::journal::recording::select_materials_bounded( - &self.catalog.store, - context, - id, - INPUT_BUDGET, - |source, start, limit| self.journal_source_material(source, start, limit), - ) + saved: Option<&CaptureDto>, + ) -> Result { + let session = self + .catalog + .store + .read_journal(context, id) + .map_err(journal_error)? + .source_session_id; + JournalInput::build(&self.catalog.store, context, id, |order, offset, budget| { + let capture = match saved { + Some(capture) => capture.clone(), + None => { + let capture = self + .with_recording_boundary(&session, |sessions, session_id, active, guard| { + session_store::capture::capture_recording_slice_fs_guarded( + sessions, + session_id, + active, + session_store::public_material::PublicMaterialLimits::default(), + RecordingPosition { + canonical_order: order, + byte_offset: offset, + }, + MAX_JOURNAL_RANGES, + budget, + guard, + ) + .map(|slice| slice.capture) + .map_err(|e| SubjektivHostError::Unavailable(e.to_string())) + }) + .map_err(|e| JournalError::Conflict(e.to_string()))?; + // Admission and capture are separate saves. An append between them + // may be in replay but has not yet been accepted for this Journal. + return self.accepted_capture_prefix(context, id, capture); + } + }; + let end = capture + .entries + .last() + .map(|entry| { + capture + .recording_text(entry.order) + .map(|(descriptor, start, text)| { + if start + text.len() as u64 == descriptor.total_bytes { + RecordingPosition { + canonical_order: entry.order + 1, + byte_offset: 0, + } + } else { + RecordingPosition { + canonical_order: entry.order, + byte_offset: start + text.len() as u64, + } + } + }) + }) + .transpose() + .map_err(|e| JournalError::Conflict(e.to_string()))?; + if end.is_none_or(|end| (order, offset) >= (end.canonical_order, end.byte_offset)) { + return Ok(CaptureDto { + entries: vec![], + ..capture + }); + } + capture + .recording_slice( + RecordingPosition { + canonical_order: order, + byte_offset: offset, + }, + MAX_JOURNAL_RANGES, + budget, + ) + .map(|slice| slice.capture) + .map_err(|e| JournalError::Conflict(e.to_string())) + }) .map_err(journal_error) } - fn journal_grant( + fn accepted_capture_prefix( + &self, + context: &JournalHostContext, + id: &str, + mut capture: CaptureDto, + ) -> JournalResult { + let mut selected = Vec::new(); + for mut entry in capture.entries { + let status = match self + .catalog + .store + .read_journal_source(context, id, entry.order) + { + Ok(status) => status, + Err(JournalError::NotFound(_)) => break, + Err(error) => return Err(error), + }; + let session_store::capture::CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + .. + } = &mut entry.body + else { + return Err(JournalError::InvalidInput("non-recording capture".into())); + }; + if source_descriptor(&capture.session_id, descriptor) != status.source { + return Err(JournalError::Conflict( + "captured source differs from admitted identity".into(), + )); + } + let Some(accepted) = status + .accepted + .iter() + .find(|range| range.start <= *byte_offset && *byte_offset < range.end) + else { + break; + }; + let end = status + .unavailable + .iter() + .filter(|range| range.end > *byte_offset) + .fold(accepted.end, |end, range| { + end.min(range.start.max(*byte_offset)) + }); + let mut length = text.len().min(end.saturating_sub(*byte_offset) as usize); + while !text.is_char_boundary(length) { + length -= 1; + } + if length == 0 { + break; + } + text.truncate(length); + let partial = *byte_offset + (text.len() as u64) < descriptor.total_bytes; + selected.push(entry); + if partial { + break; + } + } + capture.entries = selected; + Ok(capture) + } + + fn journal_grant_scope( &self, snapshot: &JobSnapshot, ) -> Result { @@ -1097,7 +1358,6 @@ impl LocalSubjectHost { || snapshot.request.purpose != PURPOSE || snapshot.request.serialization_key.as_deref() != Some(&format!("subjektiv-journal:{}", grant.journal_id)) - || snapshot.request.input != grant.input() { return Err(SubjektivHostError::InvalidArgument( "Job input/grant does not match this Journal capability".into(), @@ -1135,12 +1395,33 @@ impl LocalSubjectHost { Ok(grant) } + fn journal_grant( + &self, + snapshot: &JobSnapshot, + ) -> Result { + snapshot + .load_executable_request() + .map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?; + let grant = self.journal_grant_scope(snapshot)?; + let input = JournalInput::decode(&snapshot.request.input).map_err(journal_error)?; + if input.subject_id != self.binding.subject_id + || input.journal_id != grant.journal_id + || input.capture.session_id != grant.session_id + || input.selected_ranges != grant.ranges + { + return Err(SubjektivHostError::InvalidArgument( + "saved Journal body identity/ranges differ from grant".into(), + )); + } + Ok(grant) + } + fn complete_journal_job( &self, jobs: &crate::jobs::StandaloneJobs, snapshot: &JobSnapshot, ) -> Result<(), SubjektivHostError> { - let grant = self.journal_grant(snapshot)?; + let grant = self.journal_grant_scope(snapshot)?; let context = authority(self, &grant.session_id)?; let completion: JournalJobCompletion = serde_json::from_value(snapshot.result.clone().ok_or_else(|| { @@ -1174,6 +1455,12 @@ impl LocalSubjectHost { } jobs.acknowledge(&snapshot.request.job_id) .map_err(service_error)?; + jobs.reclaim_input( + &snapshot.request.job_id, + &snapshot.input_digest, + JobInputRetirementReason::Completed, + ) + .map_err(service_error)?; if let Err(error) = self.prune_retained_workers() { // Pruning is independent of a successful checkpoint/result. Keep the originals // and continue draining other saved prefixes; a retention error is not model failure. @@ -1192,47 +1479,18 @@ pub(super) fn grant( .journal_grant(snapshot) .map_err(|error| error.to_string())?; let context = authority(&host, &grant.session_id).map_err(|error| error.to_string())?; - // Re-fetch selected identities before opening mutation authority. Only descriptors are - // retained in Job storage; the model must fetch actual source bytes with bounded tools. - for range in &grant.ranges { - let length = usize::try_from( - range - .bytes - .end - .checked_sub(range.bytes.start) - .ok_or("invalid source range")?, - ) - .map_err(|_| "source range overflow")?; - if length > INPUT_BUDGET { - return Err("source range exceeds frozen input budget".into()); - } - let text = host - .journal_source_window(&range.source, range.bytes.start, length.max(4)) - .map_err(|error| error.to_string())?; - if text.len() != length { - return Err("frozen Journal source byte range changed".into()); - } - } - if host - .journal_origins(&grant.ranges) - .map_err(|error| error.to_string())? - != grant.origins - { - return Err( - "frozen Journal provenance does not match the authorized original Session".into(), - ); - } + let input = JournalInput::decode(&snapshot.request.input).map_err(|e| e.to_string())?; + let input_digest = snapshot.input_digest.clone(); let generation = host .catalog .store - .prepare_journal_generation( + .prepare_journal_input( &context, - &grant.journal_id, + &input, JobAttemptBinding { job_id: snapshot.request.job_id.clone(), attempt_id: snapshot.attempt.attempt_id.clone(), }, - grant.ranges.clone(), ) .map_err(|error| error.to_string())?; let job_context = JournalJobContext::validated( @@ -1242,12 +1500,17 @@ pub(super) fn grant( generation.job.clone(), ) .map_err(|error| error.to_string())?; + input + .validate_generation(&generation) + .map_err(|e| e.to_string())?; let adapter = Arc::new(LocalJournalHost { host, grant, context, job_context, + #[cfg(test)] generation, + input_digest, fence, }); Ok(worker::job::JobFeatureGrant { @@ -1263,7 +1526,9 @@ struct LocalJournalHost { grant: LocalJournalGrant, context: JournalHostContext, job_context: JournalJobContext, + #[cfg(test)] generation: JournalGeneration, + input_digest: String, fence: Arc, } impl LocalJournalHost { @@ -1296,18 +1561,20 @@ impl JournalJobHost for LocalJournalHost { request: JournalJobRequest, ) -> Result { let context = self.live()?; - subjektiv::journal::service::execute_bounded( + // The generic runner has supplied this exact immutable input before it can + // invoke this bound tool. Preparation alone is not evidence of delivery. + subjektiv::journal::service::attest_input_delivery( &self.host.catalog.store, &context, + &self.input_digest, + &self.grant.ranges, + ) + .map_err(journal_error)?; + subjektiv::journal::service::execute_snapshot( + &self.host.catalog.store, + &context, + &self.input_digest, request, - |source, start, limit| { - if source.session_id != self.generation.initial_journal.source_session_id { - return Err(JournalError::PermissionDenied( - "foreign original Session".into(), - )); - } - self.host.journal_source_material(source, start, limit) - }, ) .map_err(journal_error) } diff --git a/crates/standalone/src/subjektiv/journal/tests.rs b/crates/standalone/src/subjektiv/journal/tests.rs index c488b4879..0e4d76f5f 100644 --- a/crates/standalone/src/subjektiv/journal/tests.rs +++ b/crates/standalone/src/subjektiv/journal/tests.rs @@ -1,6 +1,6 @@ //! Integration of standalone attribution, original FsStore material, shared Journal storage //! and the local Job adapter. Only the outer model/execution lease is replaced in these cases. -//! The generic Job runner is exercised separately by the standalone Jobs integration suite. +//! The scripted model also exercises saved direct input through the actual generic Job runner. use super::*; use crate::store::{StaleLeasePolicy, StandaloneWorkerAllocation, StandaloneWorkerStore}; @@ -144,8 +144,8 @@ impl Fixture { .unwrap() .contains(sentinel) ); - // Include SQLite/WAL/journal files: immutable request/grant storage must never hold - // original text, even in freed pages left behind by explicit source erasure. + // Include SQLite/WAL/journal files: erasure must scrub retired original bodies, + // including any freed pages or journal left by the immutable request store. for entry in fs::read_dir(self.connection.jobs_path.parent().unwrap()).unwrap() { let entry = entry.unwrap(); if entry.file_type().unwrap().is_file() { @@ -194,9 +194,9 @@ impl Fixture { context, job_context, generation, + input_digest: snapshot.input_digest.clone(), fence: fence.clone(), }; - adapter.read_all_sources(); (adapter, fence, snapshot) } } @@ -220,37 +220,6 @@ fn message(id: &str, text: &str) -> session_store::LoggedHistoryEntry { } } impl LocalJournalHost { - fn read_all_sources(&self) { - for range in &self.generation.selected_ranges { - let mut offset = range.bytes.start as usize; - let mut content_token = None; - loop { - let response = self - .execute(JournalJobRequest::ReadText { - read: JournalTextRead { - target: JournalTextTarget::Source { - pointer: range.source.pointer.clone(), - }, - offset, - limit: MAX_JOURNAL_READ_BYTES, - content_token: content_token.clone(), - }, - }) - .unwrap(); - let JournalJobResponse::Text(text) = response else { - panic!() - }; - match text.next_byte { - Some(next) => { - assert!(next > offset); - offset = next; - content_token = Some(text.content_token); - } - None => break, - } - } - } - } fn state(&self) -> Result { match self.execute(JournalJobRequest::ReadState)? { JournalJobResponse::State(state) => Ok(state), @@ -362,8 +331,8 @@ fn duplicate_admission_reserves_one_job_and_excludes_private_prompt() { assert_eq!(jobs.pending().unwrap().len(), 1); let input = jobs.get(&id).unwrap().request.input.to_string(); assert!( - !input.contains("short request"), - "Job storage retains descriptors, never a second copy of original text" + input.contains("short request"), + "Job storage supplies the canonical body directly" ); assert!(!input.contains("PRIVATE PROMPT")); let foreign = SubjektivHostContext { @@ -498,14 +467,16 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() { &adapter.generation.job, ) .unwrap(); - let materials = fixture + let next = fixture .host() - .select_journal_materials(&adapter.context, &adapter.grant.journal_id) + .build_journal_input(&adapter.context, &adapter.grant.journal_id, None) .unwrap(); - assert_eq!(materials[0].range.source.pointer.entry_ref, "Elast"); + assert_eq!(next.selected_ranges[0].source.pointer.entry_ref, "Elast"); assert!( - materials[0] - .text + next.capture + .recording_text(next.selected_ranges[0].source.order) + .unwrap() + .2 .contains("last input while generation runs") ); } @@ -547,7 +518,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() { let (adapter, _, _) = fixture.adapter(); let range = &adapter.grant.ranges[0]; assert!(range.bytes.end < range.source.total_bytes); - assert!(range.bytes.end - range.bytes.start < INPUT_BUDGET as u64); + assert!(range.bytes.end - range.bytes.start < job::MAX_JOB_INPUT_BYTES as u64); let snapshot = fixture .host() .jobs() @@ -557,7 +528,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() { .pop() .unwrap(); assert!(serde_json::to_vec(&snapshot.request.input).unwrap().len() < 128 * 1024); - assert!(!snapshot.request.input.to_string().contains("漢字")); + assert!(snapshot.request.input.to_string().contains("漢字")); assert!(adapter.finish().is_err()); } @@ -628,43 +599,35 @@ impl JournalScript { ) } else if let Some(state) = result("state") { let state = &state["value"]; - for (index, range) in state["selected_ranges"] - .as_array() - .unwrap() + let input: serde_json::Value = request + .items .iter() - .enumerate() - { - let mut offset = range["bytes"]["start"].as_u64().unwrap(); - let mut token = Value::Null; - loop { - let id = format!("source-{index}-{offset}"); - if let Some(page) = result(&id) { - let page = &page["value"]; - let key = format!("{}:{id}", state["generation_id"].as_str().unwrap()); - let text = page["text"].as_str().unwrap().to_owned(); - if let Some(previous) = - self.source_reads.lock().unwrap().insert(key, text.clone()) - { - assert_eq!( - previous, text, - "frozen source read changed within a generation" - ); - } - if let Some(next) = page["next_byte"].as_u64() { - assert!(next > offset); - offset = next; - token = page["content_token"].clone(); - } else { - break; - } - } else { - let input = json!({"target":{"target":"source","pointer":range["source"]["pointer"]},"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token}); - return vec![ - Event::tool_use_start(0, &id, "ReadJournalText"), - Event::tool_input_delta(0, input.to_string()), - Event::tool_use_stop(0), - ]; - } + .filter_map(Item::as_text) + .find_map(|text| { + text.lines() + .find_map(|line| line.strip_prefix("input_json: ")) + }) + .map(|json| serde_json::from_str(json).unwrap()) + .expect("direct saved Journal input"); + let input = JournalInput::decode(&input).unwrap(); + assert_eq!( + serde_json::to_value(&input.selected_ranges).unwrap(), + state["selected_ranges"] + ); + for entry in &input.capture.entries { + let (_, offset, text) = input.capture.recording_text(entry.order).unwrap(); + let key = format!( + "{}:{}:{offset}", + state["generation_id"].as_str().unwrap(), + entry.order + ); + if let Some(previous) = self + .source_reads + .lock() + .unwrap() + .insert(key, text.to_owned()) + { + assert_eq!(previous, text, "saved source changed within generation"); } } let (name, action) = if state["current_open"].is_null() { @@ -1166,6 +1129,11 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c .items; assert_eq!(entries.len(), 1); let saved_entry = entries[0].id.clone(); + let original_input = jobs.get(&id).unwrap().request.input; + fixture.append( + "newer-than-snapshot", + "NEW INPUT MUST NOT MIX INTO RESUMED SNAPSHOT", + ); jobs.shutdown().await.unwrap(); fixture.connection.close().unwrap(); let mut reopened = StandaloneSubjects::open_existing(&state) @@ -1176,8 +1144,29 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c .unwrap(); reopened.host.bind_jobs(jobs.clone()).unwrap(); assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown); - // Domain reconciliation inspects the now-fenced prior attempt and preserves its receipts. + // Deny every original-source read while preparing and running the replacement. + let source_path = fixture + .sessions + .root_dir() + .join(&fixture.context.session_id); + let denied = fixture + ._temp + .path() + .join("temporarily-unavailable-original"); + fs::rename(&source_path, &denied).unwrap(); let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap(); + let replacement_input = jobs.get(&replacement).unwrap().request.input; + assert_eq!(replacement_input["capture"], original_input["capture"]); + assert!( + !replacement_input + .to_string() + .contains("NEW INPUT MUST NOT MIX") + ); + assert_eq!(jobs.get(&id).unwrap().request.input, original_input); + assert_eq!( + jobs.get(&id).unwrap().input_disposition, + JobInputDisposition::Available + ); assert_ne!( replacement, id, "unknown external execution is not blindly replayed" @@ -1205,8 +1194,26 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c .items; assert_eq!(entries.len(), 1); assert_eq!(entries[0].id, saved_entry); + fs::rename(&denied, &source_path).unwrap(); + let journal = reopened + .host + .catalog + .store + .read_journal(&context, &journal_id) + .unwrap(); + let original = JournalInput::decode(&original_input).unwrap(); + let end = original.selected_ranges.last().unwrap(); + assert_eq!( + (journal.process_order, journal.process_byte), + if end.bytes.end == end.source.total_bytes { + (end.source.order + 1, 0) + } else { + (end.source.order, end.bytes.end) + } + ); + assert!(reopened.host.admit_journal(&fixture.context).unwrap()); assert!( - reopened + !reopened .host .session_fully_reflected(&fixture.context.session_id) .unwrap() @@ -1302,9 +1309,9 @@ fn missing_original_material_blocks_worker_removal_without_erasing_state() { } #[test] -fn frozen_original_remains_readable_after_worker_switches_its_active_session() { +fn frozen_input_survives_active_session_change_and_source_tool_remains_refused() { let fixture = Fixture::new("old Session original"); - let (adapter, _, _) = fixture.adapter(); + let (adapter, _, snapshot) = fixture.adapter(); let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap(); let worker = fixture._allocation.as_ref().unwrap().worker_id(); let record = workers.load(worker).unwrap(); @@ -1318,22 +1325,30 @@ fn frozen_original_remains_readable_after_worker_switches_its_active_session() { )), )) .unwrap(); - let response = adapter - .execute(JournalJobRequest::ReadText { - read: JournalTextRead { - target: JournalTextTarget::Source { - pointer: adapter.generation.selected_ranges[0].source.pointer.clone(), - }, - offset: 0, - limit: MAX_JOURNAL_READ_BYTES, - content_token: None, - }, - }) - .unwrap(); - let JournalJobResponse::Text(page) = response else { - panic!() - }; - assert!(page.text.contains("old Session original")); + assert!( + adapter + .execute(JournalJobRequest::ReadText { + read: JournalTextRead { + target: JournalTextTarget::Source { + pointer: adapter.generation.selected_ranges[0].source.pointer.clone() + }, + offset: 0, + limit: MAX_JOURNAL_READ_BYTES, + content_token: None + } + }) + .is_err(), + "original Source lookup is explicitly unavailable" + ); + let input = JournalInput::decode(&snapshot.request.input).unwrap(); + assert!( + input + .capture + .recording_text(0) + .unwrap() + .2 + .contains("old Session original") + ); assert!( fixture .host() @@ -1512,14 +1527,18 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins() const SENTINEL: &str = "original-plaintext-sentinel-must-never-enter-immutable-job-storage"; let mut fixture = Fixture::new(SENTINEL); let (_, _, snapshot) = fixture.adapter(); - fixture.assert_job_storage_excludes(&snapshot, SENTINEL); + assert!(snapshot.request.input.to_string().contains(SENTINEL)); let state = fixture.host().catalog.state_root.clone(); let binding = fixture.connection.binding.clone(); let session: session_store::SessionId = fixture.context.session_id.parse().unwrap(); - let old_reader = fixture - .host() - .material_reader(&fixture.context.session_id) - .unwrap(); + let old_reader = SessionPublicMaterialReader::open_fs_with_retention( + fixture.sessions.root_dir(), + session, + fixture.segment, + PublicMaterialLimits::default(), + crate::store::session_retention(&state), + ) + .unwrap(); let descriptor = old_reader.descriptors().next().unwrap().clone(); let (worker, workers) = fixture.stop_worker(); assert!(workers.delete(worker).is_err()); @@ -1759,14 +1778,12 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job( let snapshot = jobs.get(&next).unwrap(); let grant = fixture.host().journal_grant(&snapshot).unwrap(); assert_eq!(grant.ranges.len(), 1); - assert_eq!( - grant.origins[0].origin, - PublicMaterialOrigin::HostRunBoundary - ); - let content = fixture - .host() - .journal_source_window(&grant.ranges[0].source, 0, 4096) + let input = JournalInput::decode(&snapshot.request.input).unwrap(); + let (descriptor, _, content) = input + .capture + .recording_text(grant.ranges[0].source.order) .unwrap(); + assert_eq!(descriptor.origin, PublicMaterialOrigin::HostRunBoundary); let value: serde_json::Value = serde_json::from_str(&content).unwrap(); assert_eq!(value["kind"], "run_boundary"); assert_eq!(value["boundary"]["kind"], "errored"); @@ -1791,3 +1808,621 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job( assert!(fixture.host().next_journal_job().unwrap().is_none()); jobs.shutdown().await.unwrap(); } + +/// Original FsStore -> bounded common capture -> persisted Job -> actual generic runner -> +/// sequential Journal tools. Only the LLM transport is scripted. Source files are deliberately +/// unavailable after persistence, and input bodies are inspected in the provider request. +#[tokio::test] +async fn saved_body_jobs_process_over_a_hundred_entries_and_utf8_without_live_source_reads() { + let fixture = Fixture::new("oldest original"); + for index in 0..115 { + fixture.append( + &format!("entry-{index}"), + &format!("public original {index}"), + ); + } + fixture.append("long-utf8", &"漢字🙂\n\"".repeat(24_000)); + fixture.host().admit_journal(&fixture.context).unwrap(); + let expected = fixture + .host() + .recording_capture(&fixture.context.session_id) + .unwrap(); + let source_path = fixture + .sessions + .root_dir() + .join(&fixture.context.session_id); + let denied = fixture._temp.path().join("denied-original-source"); + let mut seen: std::collections::BTreeMap = Default::default(); + let mut jobs_processed = 0; + loop { + let Some((jobs, id)) = fixture.host().next_journal_job().unwrap() else { + break; + }; + let saved = jobs.get(&id).unwrap(); + let input = JournalInput::decode(&saved.request.input).unwrap(); + assert!( + serde_json::to_vec(&saved.request.input).unwrap().len() <= job::MAX_JOB_INPUT_BYTES + ); + assert!(input.capture.entries.len() <= MAX_JOURNAL_RANGES); + for entry in &input.capture.entries { + let (_, offset, text) = input.capture.recording_text(entry.order).unwrap(); + let previous = seen.entry(entry.order).or_default(); + assert_eq!( + previous.len() as u64, + offset, + "no gap, preview or duplicate prefix" + ); + previous.push_str(text); + } + // Deny all catalogs/reads, including grant preparation. The model still receives + // exactly the persisted body, and the Host must not follow the active pointer. + fs::rename(&source_path, &denied).unwrap(); + let model = JournalScript::default(); + jobs.start_with_model_client(&id, model.clone()) + .await + .unwrap(); + let result = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id)) + .await + .unwrap() + .unwrap(); + assert_eq!(result.state, JobState::Completed); + assert!( + !model.source_reads.lock().unwrap().is_empty(), + "provider saw saved source bodies" + ); + fixture.host().complete_journal_job(&jobs, &result).unwrap(); + assert_eq!( + jobs.get(&id).unwrap().input_disposition, + JobInputDisposition::Reclaimed + ); + fs::rename(&denied, &source_path).unwrap(); + jobs_processed += 1; + assert!(jobs_processed < 20, "bounded prefix drain must progress"); + } + assert!( + jobs_processed > 2, + "entry count and long text both require continuation" + ); + assert_eq!(seen.len(), expected.entries.len()); + for entry in &expected.entries { + assert_eq!( + seen[&entry.order], + expected.recording_text(entry.order).unwrap().2 + ); + } + fixture.host().jobs().unwrap().shutdown().await.unwrap(); +} + +#[tokio::test] +async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful_records() { + let fixture = Fixture::new("original descriptor-only input"); + let (adapter, _, modern) = fixture.adapter(); + let jobs = fixture.host().jobs().unwrap(); + fixture + .host() + .catalog + .store + .close_journal_generation( + &adapter.context, + &adapter.generation.id, + &adapter.generation.job, + ) + .unwrap(); + jobs.cancel(&modern.request.job_id).unwrap(); + let input = JournalInput::decode(&modern.request.input).unwrap(); + let origins: Vec<_> = input + .capture + .entries + .iter() + .map(|entry| { + let (descriptor, _, _) = input.capture.recording_text(entry.order).unwrap(); + SourceOrigin { + pointer: source_descriptor(&fixture.context.session_id, descriptor).pointer, + origin: descriptor.origin.clone(), + } + }) + .collect(); + let mut old_grant = adapter.grant.clone(); + old_grant.origins = origins; + old_grant.writer_binding_id = None; + let mut request = modern.request.clone(); + request.job_id = format!("legacy-descriptor:{}", uuid::Uuid::now_v7()); + request.input = + serde_json::json!({"selected_ranges":old_grant.ranges,"origins":old_grant.origins}); + let old = jobs + .request_granted(request, serde_json::to_value(&old_grant).unwrap()) + .unwrap(); + assert!( + fixture.host().journal_grant(&old).is_err(), + "descriptor jobs must not execute as snapshots" + ); + let generation = fixture + .host() + .catalog + .store + .prepare_journal_generation( + &adapter.context, + &old_grant.journal_id, + JobAttemptBinding { + job_id: old.request.job_id.clone(), + attempt_id: old.attempt.attempt_id.clone(), + }, + old_grant.ranges.clone(), + ) + .unwrap(); + let context = JournalJobContext::validated( + &fixture.host().catalog.store, + &adapter.context, + &generation.id, + generation.job.clone(), + ) + .unwrap(); + let saved = fixture + .host() + .catalog + .store + .execute_journal( + &context, + JournalOperation { + request_id: "legacy-record-success-response-lost".into(), + expected_state: generation.initial_journal.state_token.clone(), + action: JournalAction::Create { + title: "already saved".into(), + body: "legacy successful operation".into(), + ongoing: "unfinished".into(), + state: JournalEntryState::Open, + source_end: None, + correction_refs: vec![], + }, + covered_ranges: generation.selected_ranges.clone(), + }, + ) + .unwrap(); + fixture.append( + "new-not-selected", + "LATEST SOURCE MUST WAIT UNTIL OLD INPUT FINISHES", + ); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (_, next) = fixture.host().next_journal_job().unwrap().unwrap(); + assert_ne!(next, old.request.job_id); + let replacement = jobs.get(&next).unwrap(); + let replacement_input = JournalInput::decode(&replacement.request.input).unwrap(); + assert!( + !replacement + .request + .input + .to_string() + .contains("LATEST SOURCE MUST WAIT") + ); + assert_eq!(replacement_input.selected_ranges, old_grant.ranges); + let retired = jobs.get(&old.request.job_id).unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated); + assert_eq!(retired.input_digest, old.input_digest); + assert_eq!(retired.intent_fingerprint, old.intent_fingerprint); + assert!(retired.request.input.is_null()); + assert!( + fixture + .host() + .catalog + .store + .execute_journal( + &context, + JournalOperation { + request_id: "legacy-delayed".into(), + expected_state: saved.state_token, + action: JournalAction::ConfirmUnchanged, + covered_ranges: old_grant.ranges, + } + ) + .is_err() + ); + jobs.start_with_model_client(&next, JournalScript::default()) + .await + .unwrap(); + let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&next)) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.state, JobState::Completed); + fixture + .host() + .complete_journal_job(&jobs, &completed) + .unwrap(); + let entries = fixture + .host() + .catalog + .store + .list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32) + .unwrap(); + assert_eq!(entries.items.len(), 1); + assert_eq!(entries.items[0].id, saved.entry_id.unwrap()); + jobs.shutdown().await.unwrap(); +} + +#[test] +fn explicit_erase_retires_every_historical_body_across_job_history_pages() { + const SENTINEL: &str = "historical-input-erasure-sentinel-781"; + let mut fixture = Fixture::new("ordinary original"); + let (adapter, _, first) = fixture.adapter(); + let jobs = fixture.host().jobs().unwrap(); + for index in 0..105 { + let mut request = first.request.clone(); + request.job_id = format!("historical-{index:03}"); + request.purpose = "historical-other-domain-job".into(); + request.serialization_key = None; + request.input = serde_json::json!({"body":SENTINEL, "index":index}); + let saved = jobs + .request_granted( + request, + serde_json::json!({"session_id":fixture.context.session_id}), + ) + .unwrap(); + jobs.cancel(&saved.request.job_id).unwrap(); + } + let state = fixture.host().catalog.state_root.clone(); + let subject = fixture.connection.binding.subject_id.clone(); + fixture.connection.close().unwrap(); + StandaloneSubjects::open_existing(&state) + .unwrap() + .erase_session_completely(&subject, &fixture.context.session_id, "erase-all-history") + .unwrap(); + let jobs = crate::jobs::StandaloneJobs::open( + &fixture.connection.jobs_path, + fixture._temp.path().into(), + ) + .unwrap(); + let mut after = None; + let mut count = 0; + loop { + let page = jobs + .list_session_snapshots(&fixture.context.session_id, after.as_deref(), 100) + .unwrap(); + if page.is_empty() { + break; + } + for saved in &page { + assert_eq!(saved.input_disposition, JobInputDisposition::Erased); + assert!(saved.request.input.is_null()); + assert!(saved.load_executable_request().is_err()); + } + count += page.len(); + after = page.last().map(|job| job.request.job_id.clone()); + } + assert_eq!(count, 106); + assert!( + adapter.execute(JournalJobRequest::ReadState).is_err(), + "prepared local adapter is invalidated" + ); + fixture.assert_job_storage_excludes(&jobs.get(&first.request.job_id).unwrap(), SENTINEL); +} + +#[test] +fn saved_snapshot_missing_tampered_foreign_or_changed_initial_state_is_not_recaptured() { + let fixture = Fixture::new("saved immutable original"); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap(); + let saved = jobs.get(&id).unwrap(); + for changed in [serde_json::Value::Null, serde_json::json!({"capture":{}})] { + let mut malformed = saved.clone(); + malformed.request.input = changed; + assert!(fixture.host().journal_grant(&malformed).is_err()); + } + let original = JournalInput::decode(&saved.request.input).unwrap(); + for field in ["subject_id", "journal_id", "initial_state_token"] { + let mut changed = saved.request.clone(); + changed.job_id = format!("changed-{field}"); + changed.serialization_key = None; + changed.input[field] = serde_json::json!("foreign-or-altered"); + // Even a separately reserved immutable Job may not adopt the original capability. + let changed = jobs + .request_granted(changed, saved.domain_grant.clone().unwrap()) + .unwrap(); + assert!( + grant( + fixture.host().clone(), + &changed, + Arc::new(TestFence(AtomicBool::new(true))) + ) + .is_err() + ); + } + let mut generation = fixture + .host() + .catalog + .store + .prepare_journal_generation( + &authority(fixture.host(), &fixture.context.session_id).unwrap(), + &original.journal_id, + JobAttemptBinding { + job_id: id, + attempt_id: saved.attempt.attempt_id.clone(), + }, + original.selected_ranges.clone(), + ) + .unwrap(); + generation.initial_journal.state_token = "changed-after-snapshot".into(); + assert!(original.validate_generation(&generation).is_err()); + assert_eq!( + saved.request.input, + jobs.get(&saved.request.job_id).unwrap().request.input + ); +} + +#[tokio::test] +async fn append_between_admission_and_body_capture_does_not_publish_unaccepted_input() { + let fixture = Fixture::new("accepted oldest prefix"); + fixture.host().admit_journal(&fixture.context).unwrap(); + fixture.append("not-yet-admitted", "newly saved but not accepted"); + let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap(); + let input = jobs.get(&id).unwrap().request.input; + assert!(input.to_string().contains("accepted oldest prefix")); + assert!(!input.to_string().contains("newly saved but not accepted")); + jobs.start_with_model_client(&id, JournalScript::default()) + .await + .unwrap(); + let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id)) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.state, JobState::Completed); + fixture + .host() + .complete_journal_job(&jobs, &completed) + .unwrap(); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (_, next) = fixture.host().next_journal_job().unwrap().unwrap(); + let remaining = jobs.get(&next).unwrap().request.input; + assert!( + remaining + .to_string() + .contains("newly saved but not accepted") + ); + assert!(!remaining.to_string().contains("accepted oldest prefix")); + jobs.shutdown().await.unwrap(); +} + +/// Generic Job preparation must not attest delivery. The saved body is supplied by the +/// runner before a bound Journal tool may execute; no model-provided digest is trusted. +#[tokio::test] +async fn actual_bound_tool_attests_delivery_but_prepared_blocked_job_does_not() { + let fixture = Fixture::new("body delivered through generic runner"); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap(); + let saved = jobs.get(&id).unwrap(); + let model = JournalScript { + gate_first: true, + pause_after_record: true, + ..Default::default() + }; + jobs.start_with_model_client(&id, model.clone()) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified()) + .await + .unwrap(); + let host_context = authority(fixture.host(), &fixture.context.session_id).unwrap(); + let journal_id = fixture + .host() + .catalog + .store + .journal_for_session(&host_context) + .unwrap() + .unwrap(); + let generation = fixture + .host() + .catalog + .store + .active_journal_generation(&host_context, &journal_id) + .unwrap() + .unwrap(); + let context = JournalJobContext::validated( + &fixture.host().catalog.store, + &host_context, + &generation.id, + generation.job.clone(), + ) + .unwrap(); + let undelivered = subjektiv::journal::service::execute_snapshot( + &fixture.host().catalog.store, + &context, + &saved.input_digest, + JournalJobRequest::ReadState, + ) + .unwrap_err(); + assert!( + undelivered.to_string().contains("delivery"), + "prepared grant must not satisfy delivery gate: {undelivered}" + ); + let before = fixture + .host() + .catalog + .store + .read_journal(&host_context, &journal_id) + .unwrap(); + assert_eq!((before.process_order, before.process_byte), (0, 0)); + model.release.notify_one(); + tokio::time::timeout( + std::time::Duration::from_secs(20), + model.record_saved.notified(), + ) + .await + .unwrap(); + assert!( + subjektiv::journal::service::execute_snapshot( + &fixture.host().catalog.store, + &context, + &saved.input_digest, + JournalJobRequest::ReadState + ) + .is_ok(), + "actual bound tool execution attests exact saved input" + ); + model.release.notify_one(); + let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id)) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.state, JobState::Completed); + fixture + .host() + .complete_journal_job(&jobs, &completed) + .unwrap(); + jobs.shutdown().await.unwrap(); +} + +#[derive(Clone, Default)] +struct BlockedJournalRequest { + entered: Arc, + dropped: Arc, + request_dropped: Arc, +} +struct OwnedBlockedJournalRequest { + // A local request copy, including the saved input, must be dropped before erasure + // can acquire the lease. This does not claim remote/provider bytes are retractable. + _request: agen::llm_client::Request, + model: BlockedJournalRequest, +} +impl Drop for OwnedBlockedJournalRequest { + fn drop(&mut self) { + self.model.request_dropped.store(true, Ordering::Release); + self.model.dropped.notify_one(); + } +} +#[async_trait::async_trait] +impl agen::llm_client::LlmClient for BlockedJournalRequest { + async fn stream( + &self, + request: agen::llm_client::Request, + ) -> Result< + std::pin::Pin< + Box< + dyn futures::Stream< + Item = Result< + agen::llm_client::event::Event, + agen::llm_client::error::ClientError, + >, + > + Send, + >, + >, + agen::llm_client::error::ClientError, + > { + let owned = OwnedBlockedJournalRequest { + _request: request, + model: self.clone(), + }; + assert!( + owned + ._request + .items + .iter() + .filter_map(agen::llm_client::types::Item::as_text) + .any(|text| text.contains("local blocked source body")) + ); + self.entered.notify_one(); + std::future::pending::<()>().await; + drop(owned); + unreachable!() + } + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } +} + +/// Real generic Jobs and a retained public handle, with only the provider transport blocked. +/// Connection close must initiate cancellation and preserve the exclusive lease until local +/// request/history ownership is gone (or return an explicit incomplete barrier). +#[tokio::test] +async fn close_cancels_retained_local_job_before_erase_can_acquire_exclusive_lease() { + let mut fixture = Fixture::new("local blocked source body"); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap(); + let retained = jobs.clone(); + let model = BlockedJournalRequest::default(); + retained + .start_with_model_client(&id, model.clone()) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified()) + .await + .unwrap(); + let state = fixture.host().catalog.state_root.clone(); + let subject = fixture.connection.binding.subject_id.clone(); + let close_error = fixture.connection.close().unwrap_err(); + assert!(matches!( + close_error, + SubjectError::JobShutdown(crate::jobs::JobServiceError::ShutdownIncomplete) + )); + assert!(fixture.connection.lease.is_some()); + assert!(fixture.host().jobs.lock().unwrap().is_some()); + assert!(matches!( + retained.start(&id).await, + Err(crate::jobs::JobServiceError::Closed) + )); + let locally_dropped_before_erase = model.request_dropped.load(Ordering::Acquire); + let early_erasure = StandaloneSubjects::open_existing(&state) + .unwrap() + .erase_session_completely( + &subject, + &fixture.context.session_id, + "erase-blocked-local-run", + ); + let early_lease_was_retained = matches!(early_erasure, Err(SubjectError::Active(_))); + let teardown_from_close = model.request_dropped.load(Ordering::Acquire) + || tokio::time::timeout(std::time::Duration::from_secs(5), model.dropped.notified()) + .await + .is_ok(); + // Always tear down before asserting the observed violation, so a failed regression + // cannot leak the blocked model future into another test. + let _ = retained.shutdown().await; + assert!(model.request_dropped.load(Ordering::Acquire)); + fixture.connection.close().unwrap(); + assert!(fixture.connection.lease.is_none()); + assert!(fixture.host().jobs.lock().unwrap().is_none()); + // Cancellation is not completion/erasure. Reopening retains the same unknown + // attempt's immutable body until the explicit erasure transaction retires it. + let reopened = crate::jobs::StandaloneJobs::open( + &fixture.connection.jobs_path, + fixture._temp.path().into(), + ) + .unwrap(); + let unknown = reopened.get(&id).unwrap(); + assert_eq!(unknown.attempt.state, crate::jobs::JobAttemptState::Unknown); + assert_eq!(unknown.input_disposition, JobInputDisposition::Available); + assert!( + unknown + .request + .input + .to_string() + .contains("local blocked source body") + ); + reopened.shutdown().await.unwrap(); + StandaloneSubjects::open_existing(&state) + .unwrap() + .erase_session_completely( + &subject, + &fixture.context.session_id, + "erase-blocked-local-run", + ) + .unwrap(); + let erased_store = crate::jobs::StandaloneJobs::open( + &fixture.connection.jobs_path, + fixture._temp.path().into(), + ) + .unwrap(); + let erased = erased_store.get(&id).unwrap(); + assert_eq!(erased.input_disposition, JobInputDisposition::Erased); + fixture.assert_job_storage_excludes(&erased, "local blocked source body"); + erased_store.shutdown().await.unwrap(); + assert!( + locally_dropped_before_erase || early_lease_was_retained, + "close released exclusive lease while a retained generic Job still owned local input/history" + ); + assert!( + teardown_from_close, + "connection close did not initiate cancellation of the externally retained local Job" + ); + assert!( + retained.get(&id).is_err(), + "retained handles are closed after teardown" + ); +} diff --git a/crates/standalone/tests/host.rs b/crates/standalone/tests/host.rs index 4670410eb..64882761f 100644 --- a/crates/standalone/tests/host.rs +++ b/crates/standalone/tests/host.rs @@ -615,10 +615,14 @@ async fn headless_unix_client_uses_production_history_lease_and_local_authority( use tokio::io::AsyncWriteExt; let temp = tempfile::tempdir()?; - // Only the transport endpoint needs a short private pathname. Persistence remains the - // same production store exercised by the in-process Host tests above. - std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o700))?; - let endpoint_path = temp.path().join("operator.sock"); + // Shorten only the owned endpoint's components, preserving the harness TMPDIR + // even when it is checkout-local. No process-global cwd or external /tmp fallback. + let socket_dir = tempfile::Builder::new() + .prefix("") + .rand_bytes(4) + .tempdir()?; + std::fs::set_permissions(socket_dir.path(), std::fs::Permissions::from_mode(0o700))?; + let endpoint_path = socket_dir.path().join("s"); let endpoint = standalone::UnixEndpoint::bind(&endpoint_path)?; let state_dir = temp.path().join("state"); let launch = StandaloneLaunchConfig::new( diff --git a/crates/standalone/tests/subjektiv.rs b/crates/standalone/tests/subjektiv.rs index 0e587d22b..28edfb5be 100644 --- a/crates/standalone/tests/subjektiv.rs +++ b/crates/standalone/tests/subjektiv.rs @@ -479,38 +479,28 @@ fn journal_response(request: &Request) -> Vec { }), ); } - for (index, range) in state["selected_ranges"] - .as_array() - .unwrap() + let input: Value = request + .items .iter() - .enumerate() - { - let mut offset = range["bytes"]["start"].as_u64().unwrap(); - let mut token = Value::Null; - loop { - let id = format!("journal-source-{index}-{offset}"); - match result(&id) { - Some(page) => match page["value"]["next_byte"].as_u64() { - Some(next) => { - assert!(next > offset); - offset = next; - token = page["value"]["content_token"].clone(); - } - None => break, - }, - None => { - return tool( - &id, - "ReadJournalText", - json!({ - "target":{"target":"source","pointer":range["source"]["pointer"]}, - "offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token - }), - ); - } - } - } - } + .filter_map(Item::as_text) + .find_map(|text| { + text.lines() + .find_map(|line| line.strip_prefix("input_json: ")) + }) + .map(|json| serde_json::from_str(json).unwrap()) + .expect("saved direct Journal input"); + let input = subjektiv::journal::recording::JournalInput::decode(&input).unwrap(); + assert_eq!( + serde_json::to_value(&input.selected_ranges).unwrap(), + state["selected_ranges"] + ); + assert!( + input + .capture + .entries + .iter() + .all(|entry| input.capture.recording_text(entry.order).is_ok()) + ); let (name, action) = if state["current_open"].is_null() { ( "CreateJournalEntry", diff --git a/crates/subjektiv/Cargo.toml b/crates/subjektiv/Cargo.toml index 9480041cf..fc1c00ee2 100644 --- a/crates/subjektiv/Cargo.toml +++ b/crates/subjektiv/Cargo.toml @@ -15,6 +15,8 @@ serde = { workspace = true, features = ["derive"] } serde_json.workspace = true sha2.workspace = true server-api.workspace = true +session-store.workspace = true +job.workspace = true thiserror.workspace = true uuid = { workspace = true, features = ["v7"] } diff --git a/crates/subjektiv/src/journal/recording.rs b/crates/subjektiv/src/journal/recording.rs index aaf81a29e..63947b90d 100644 --- a/crates/subjektiv/src/journal/recording.rs +++ b/crates/subjektiv/src/journal/recording.rs @@ -5,8 +5,221 @@ //! original canonical identity while streaming and return only the requested bounded fragment. use super::*; +use session_store::capture::{CaptureDto, CaptureLimits}; use sha2::{Digest, Sha256}; +/// A Job owns only this bounded original-body prefix. The initial content condition binds +/// the separate generation snapshot (whose potentially large Journal text remains paged). +/// Neither possessing these bytes nor delivering them is a processing checkpoint. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JournalInput { + pub version: u32, + pub subject_id: String, + pub journal_id: String, + pub initial_state_token: String, + pub initial_open_id: Option, + pub previous_fixed_end: Option, + pub selected_ranges: Vec, + pub capture: CaptureDto, +} + +impl JournalInput { + /// Capture callback runs once for this immutable Job, never once per page. Fitting uses + /// only that captured value and measures the complete serialized input, including the + /// initial Journal condition and ranges duplicated beside the capture's descriptors. + pub fn build( + store: &SubjektivStore, + host: &JournalHostContext, + journal_id: &str, + capture_once: impl FnOnce(u64, u64, usize) -> JournalResult, + ) -> JournalResult { + let (journal, open, previous_fixed_end) = store.database.try_with_connection(|tx| { + super::store::check_host(store, tx, host, true)?; + Ok::<_, JournalError>(( + super::store::journal(tx, host, journal_id, true)?, + super::store::open(tx, journal_id)?, + super::store::previous_fixed(tx, journal_id)?.map(|p| p.0), + )) + })?; + let mut input = Self { + version: 1, + subject_id: journal.subject_id.clone(), + journal_id: journal_id.into(), + initial_state_token: journal.state_token, + initial_open_id: open.map(|e| e.id), + previous_fixed_end, + selected_ranges: Vec::new(), + capture: CaptureDto { + version: 1, + session_id: journal.source_session_id, + segment_id: "pending".into(), + view_entry_count: 0, + entries: Vec::new(), + }, + }; + let envelope = serde_json::to_vec(&input)?.len(); + // Capture once at a fixed storage boundary. Any adjustment for the Journal envelope + // below operates on this value only, never by making another live capture request. + let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope); + input.capture = capture_once(journal.process_order, journal.process_byte, budget)?; + input.selected_ranges = capture_ranges(&input.capture)?; + if input.selected_ranges.is_empty() { + return Ok(input); + } + input.fit()?; + let value = serde_json::to_value(&input)?; + Self::decode(&value) + } + + fn fit(&mut self) -> JournalResult<()> { + let original = self.capture.clone(); + let first = self + .selected_ranges + .first() + .ok_or_else(|| invalid("empty recording selection"))?; + let start = session_store::capture::RecordingPosition { + canonical_order: first.source.order, + byte_offset: first.bytes.start, + }; + let mut budget = serde_json::to_vec(&original)?.len(); + loop { + let bytes = serde_json::to_vec(self)?.len(); + if bytes <= job::MAX_JOB_INPUT_BYTES { + return Ok(()); + } + budget = budget + .checked_sub(bytes - job::MAX_JOB_INPUT_BYTES) + .ok_or_else(|| invalid("Journal envelope cannot fit source bytes"))?; + self.capture = original + .recording_slice(start, MAX_JOURNAL_RANGES, budget) + .map_err(|e| invalid(&e.to_string()))? + .capture; + self.selected_ranges = capture_ranges(&self.capture)?; + if self.selected_ranges.is_empty() { + return Err(invalid("Journal envelope cannot fit one source scalar")); + } + } + } + + pub fn decode(value: &serde_json::Value) -> JournalResult { + if serde_json::to_vec(value)?.len() > job::MAX_JOB_INPUT_BYTES { + return Err(invalid( + "Journal input exceeds whole serialized Job input limit", + )); + } + let input: Self = serde_json::from_value(value.clone())?; + input + .capture + .validate(CaptureLimits { + max_bytes: job::MAX_JOB_INPUT_BYTES, + max_entries: MAX_JOURNAL_RANGES, + max_text_bytes: job::MAX_JOB_INPUT_BYTES, + ..CaptureLimits::default() + }) + .map_err(|e| invalid(&e.to_string()))?; + if input.version != 1 + || input.subject_id.is_empty() + || input.journal_id.is_empty() + || input.initial_state_token.is_empty() + || input.selected_ranges.is_empty() + || input.selected_ranges.len() > MAX_JOURNAL_RANGES + || input + .selected_ranges + .iter() + .any(|r| r.source.session_id != input.capture.session_id) + { + return Err(invalid("invalid Journal input identity or selection")); + } + if capture_ranges(&input.capture)? != input.selected_ranges { + return Err(conflict( + "Journal body snapshot does not match selected canonical ranges", + )); + } + Ok(input) + } + + pub fn validate_generation(&self, generation: &JournalGeneration) -> JournalResult<()> { + if generation.journal_id != self.journal_id + || generation.initial_journal.subject_id != self.subject_id + || generation.initial_journal.source_session_id != self.capture.session_id + || generation.initial_journal.state_token != self.initial_state_token + || generation.initial_open.as_ref().map(|e| &e.id) != self.initial_open_id.as_ref() + || generation.previous_fixed_end != self.previous_fixed_end + || generation.selected_ranges != self.selected_ranges + { + return Err(conflict( + "saved Journal input differs from its initial generation snapshot", + )); + } + Ok(()) + } +} + +/// Converts the shared recording projection into Journal's coverage language without +/// interpreting model text or constructing a second capture representation. +pub fn capture_ranges(capture: &CaptureDto) -> JournalResult> { + use session_store::capture::CaptureBody; + let mut ranges = Vec::new(); + for entry in &capture.entries { + let CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + .. + } = &entry.body + else { + return Err(invalid( + "Journal requires canonical original recording capture", + )); + }; + if !entry.policy.recording_material + || text.is_empty() + || descriptor.identity.segment_id.to_string() != entry.source.segment_id + || descriptor.identity.entry_ref != format!("E{}", entry.entry_id) + || descriptor.canonical_order != entry.order + || byte_offset + .checked_add(text.len() as u64) + .is_none_or(|end| end > descriptor.total_bytes) + { + return Err(conflict( + "recording body identity, policy or range mismatch", + )); + } + let source = JournalSource { + session_id: capture.session_id.clone(), + pointer: JournalSourcePointer { + segment_id: descriptor.identity.segment_id, + entry_ref: descriptor.identity.entry_ref.clone(), + }, + order: descriptor.canonical_order, + content_sha256: descriptor.sha256.clone(), + total_bytes: descriptor.total_bytes, + }; + if *byte_offset == 0 && text.len() as u64 == descriptor.total_bytes { + verify_canonical_text(&source, text)?; + } + ranges.push(JournalSourceRange { + source, + bytes: JournalByteRange { + start: *byte_offset, + end: *byte_offset + text.len() as u64, + }, + }); + } + for pair in ranges.windows(2) { + if pair[0].source.order.checked_add(1) != Some(pair[1].source.order) + || pair[0].bytes.end != pair[0].source.total_bytes + || pair[1].bytes.start != 0 + { + return Err(conflict( + "recording capture is not an oldest contiguous prefix", + )); + } + } + Ok(ranges) +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct JournalMaterial { diff --git a/crates/subjektiv/src/journal/service.rs b/crates/subjektiv/src/journal/service.rs index 085d20904..1829a0e18 100644 --- a/crates/subjektiv/src/journal/service.rs +++ b/crates/subjektiv/src/journal/service.rs @@ -66,8 +66,8 @@ fn text_page( }) } -/// Adapter for Host captures bounded by their storage contract. Production source reads should -/// use [`execute_bounded`] to avoid allocating an entire source for one small read. +/// Legacy adapter for Host captures bounded by their storage contract. +/// Production Hosts use [`attest_input_delivery`] and [`execute_snapshot`] instead. pub fn execute( store: &SubjektivStore, context: &JournalJobContext, @@ -101,23 +101,149 @@ pub fn execute( }) } -/// Reads are delivered through this service before mutation authority is enabled. The durable -/// delivery ledger stores byte ranges only: Jobs may retain immutable descriptors without -/// accumulating a permanent second copy of the original Session in their request ledgers. -/// The Host verifies original provenance/digest/length and returns a bounded UTF-8 fragment. +pub(crate) fn migrate_input_delivery( + tx: &rusqlite::Transaction<'_>, +) -> feature_storage::Result<()> { + tx.execute_batch( + "CREATE TABLE journal_generation_input_deliveries ( + generation_id TEXT PRIMARY KEY NOT NULL REFERENCES journal_generations(id) ON DELETE CASCADE, + input_digest TEXT NOT NULL CHECK(length(input_digest)=71 AND substr(input_digest,1,7)='sha256:' AND substr(input_digest,8) NOT GLOB '*[^0-9a-f]*') + ); + CREATE TRIGGER journal_generation_input_delivery_no_update + BEFORE UPDATE ON journal_generation_input_deliveries + BEGIN SELECT RAISE(ABORT, 'Journal input delivery bindings are immutable'); END;", + )?; + Ok(()) +} + +fn validate_input_digest(input_digest: &str) -> JournalResult<()> { + // Preserve JobRequest::input_digest() verbatim, including its algorithm prefix. + if input_digest.strip_prefix("sha256:").is_none_or(|hex| { + hex.len() != 64 + || !hex + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) { + return Err(JournalError::InvalidInput( + "immutable Job input digest must be sha256: followed by 64 lowercase hex digits".into(), + )); + } + Ok(()) +} + +/// Trusted Host-only delivery evidence, never a model tool or a claim of understanding. +/// Before calling, the Host must validate the saved, immutable body-bearing input against the +/// bound Job's input digest, its Session/provenance/ranges, and the actual dispatch plan that +/// supplies that exact input. The service cannot verify bytes it does not own. This transaction +/// stores only the digest bound to the generation's immutable selection; it neither copies the +/// body nor changes records, position, reflection or checkpoint. Exact replay is allowed only +/// while the same generation and writer binding remain active; changed digests/ranges conflict. +pub fn attest_input_delivery( + store: &SubjektivStore, + context: &JournalJobContext, + input_digest: &str, + selected_ranges: &[JournalSourceRange], +) -> JournalResult<()> { + validate_input_digest(input_digest)?; + store.database.try_transaction(|tx| { + super::lifecycle::reject_erased(tx, &context.host)?; + let generation = super::store::job(store, tx, context, true)?; + if generation.selected_ranges != selected_ranges { + return Err(JournalError::Conflict( + "immutable Job input selection differs from generation".into(), + )); + } + let saved: Option = tx.query_row( + "SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1", + [&generation.id], |row| row.get(0), + ).optional()?; + if let Some(saved) = saved { + if saved != input_digest { + return Err(JournalError::Conflict( + "generation is bound to a different immutable Job input digest".into(), + )); + } + } else { + tx.execute( + "INSERT INTO journal_generation_input_deliveries(generation_id,input_digest) VALUES(?1,?2)", + params![generation.id, input_digest], + )?; + } + Ok(()) + }) +} + +fn require_input_delivery( + tx: &rusqlite::Connection, + context: &JournalJobContext, + input_digest: &str, +) -> JournalResult<()> { + validate_input_digest(input_digest)?; + let saved: Option = tx + .query_row( + "SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1", + [&context.generation_id], + |row| row.get(0), + ) + .optional()?; + if saved.as_deref() != Some(input_digest) { + return Err(JournalError::Conflict( + "trusted Host delivery of this immutable Job input is required".into(), + )); + } + Ok(()) +} + +/// Production sequential execution over Journal state and the Host-attested immutable input. +/// The digest is taken from the bound saved Job by the Host, not from model arguments. Every +/// request requires the matching delivery binding and an active generation/writer. Source text +/// is already in immutable input: Source targets fail explicitly, with no Session reader or +/// fallback. Receipt-backed record, position and checkpoint constraints remain unchanged. +pub fn execute_snapshot( + store: &SubjektivStore, + context: &JournalJobContext, + input_digest: &str, + request: JournalJobRequest, +) -> JournalResult { + execute_inner(store, context, request, Some(input_digest), |_, _, _| { + Err(JournalError::PermissionDenied( + "Source reads are unavailable for immutable Journal input".into(), + )) + }) +} + +/// Legacy/test source-read adapter. Production Hosts must use [`execute_snapshot`]. +/// This ledger attests bounded Source responses only and cannot satisfy the input-digest gate. pub fn execute_bounded( store: &SubjektivStore, context: &JournalJobContext, request: JournalJobRequest, + read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult, +) -> JournalResult { + execute_inner(store, context, request, None, read_source) +} + +fn execute_inner( + store: &SubjektivStore, + context: &JournalJobContext, + request: JournalJobRequest, + input_digest: Option<&str>, mut read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult, ) -> JournalResult { - let generation = store - .database - .try_with_connection(|tx| super::store::job(store, tx, context, true))?; + let generation = store.database.try_with_connection(|tx| { + super::lifecycle::reject_erased(tx, &context.host)?; + let generation = super::store::job(store, tx, context, true)?; + if let Some(input_digest) = input_digest { + require_input_delivery(tx, context, input_digest)?; + } + Ok::<_, JournalError>(generation) + })?; let mut delivered = None; let response = match request { JournalJobRequest::Execute { operation } => { - require_delivery(store, context, &generation.selected_ranges)?; + if input_digest.is_none() { + require_delivery(store, context, &generation.selected_ranges)?; + } let receipt = if matches!(operation.action, JournalAction::Checkpoint { .. }) { crate::api::checkpoint_journal(store, context, operation)? } else { @@ -197,6 +323,11 @@ pub fn execute_bounded( JournalJobRequest::ReadText { read } => { let text = match &read.target { JournalTextTarget::Source { pointer } => { + if input_digest.is_some() { + return Err(JournalError::PermissionDenied( + "Source target is unavailable: raw source bodies are in immutable Job input".into(), + )); + } let range = generation .selected_ranges .iter() diff --git a/crates/subjektiv/src/journal/store.rs b/crates/subjektiv/src/journal/store.rs index 373c06996..72008f1b1 100644 --- a/crates/subjektiv/src/journal/store.rs +++ b/crates/subjektiv/src/journal/store.rs @@ -202,7 +202,7 @@ pub(super) fn open(tx: &Connection, journal_id: &str) -> JournalResult JournalResult> { @@ -569,6 +569,60 @@ impl SubjektivStore { Ok(snapshot) }) } + /// Rebind a saved body input to a new attempt without resampling its initial state. + /// The first binding freezes the full initial Journal snapshot; retries reuse it and + /// retain operation receipts. Different Jobs cannot borrow that snapshot as authority. + pub fn prepare_journal_input( + &self, + host: &JournalHostContext, + input: &super::recording::JournalInput, + binding: JobAttemptBinding, + ) -> JournalResult { + label(&binding.job_id)?; + label(&binding.attempt_id)?; + self.database.try_transaction(|tx| { + check_host(self, tx, host, false)?; + check_session_host(self, tx, host)?; + let current = journal(tx, host, &input.journal_id, true)?; + let existing: Option = tx.query_row( + "SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3", + params![input.journal_id, binding.job_id, binding.attempt_id], |r| r.get(0)).optional()?; + if let Some(raw) = existing { + let g = serde_json::from_str(&raw)?; + input.validate_generation(&g)?; + return Ok(g); + } + // Retained receipt lookup does not reacquire writer authority. Only a new + // generation requires the current active Subject/Session writer lifetime. + check_host(self, tx, host, true)?; + if tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_generations WHERE journal_id=?1 AND active=1)", [&input.journal_id], |r| r.get::<_, bool>(0))? { + return Err(conflict("another active generation owns this Journal")); + } + let prior: Option = tx.query_row( + "SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 ORDER BY rowid LIMIT 1", + params![input.journal_id, binding.job_id], |r| r.get(0)).optional()?; + let mut g = if let Some(raw) = prior { + serde_json::from_str::(&raw)? + } else { + JournalGeneration { id: String::new(), journal_id: input.journal_id.clone(), job: binding.clone(), + selected_ranges: input.selected_ranges.clone(), initial_journal: current.clone(), + initial_open: open(tx, &input.journal_id)?, previous_fixed_end: previous_fixed(tx, &input.journal_id)?.map(|p| p.0) } + }; + input.validate_generation(&g)?; + check_ranges(tx, ¤t, &input.selected_ranges, true)?; + let end = prefix(&g.initial_journal, &input.selected_ranges)?; + let cursor = (current.process_order, current.process_byte); + if cursor < (g.initial_journal.process_order, g.initial_journal.process_byte) || cursor > end { + return Err(conflict("saved input no longer owns the current processing cursor")); + } + g.id = issued_id("journal-generation"); + g.job = binding; + tx.execute("INSERT INTO journal_generations(id,journal_id,job_id,attempt_id,request_json,snapshot_json,active) VALUES(?1,?2,?3,?4,?5,?6,1)", + params![g.id,g.journal_id,g.job.job_id,g.job.attempt_id,serde_json::to_string(&g.selected_ranges)?,serde_json::to_string(&g)?])?; + Ok(g) + }) + } + pub(super) fn validate_journal_job(&self, ctx: &JournalJobContext) -> JournalResult<()> { self.database.try_with_connection(|tx| { job(self, tx, ctx, true)?; diff --git a/crates/subjektiv/src/journal/tests.rs b/crates/subjektiv/src/journal/tests.rs index 7fae087db..2457fdea7 100644 --- a/crates/subjektiv/src/journal/tests.rs +++ b/crates/subjektiv/src/journal/tests.rs @@ -3,6 +3,1572 @@ //! runtime or provider is started. Temporary storage owns cleanup. Receipt insertion faults are //! deterministic SQLite triggers, not a claim of arbitrary process-crash durability. +mod input_delivery { + use super::*; + use transport::{JournalJobRequest as Request, JournalJobResponse as Response}; + + fn job_request(body: &str) -> ::job::JobRequest { + ::job::JobRequest { + job_id: "immutable-body-job".into(), + purpose: "Journal input delivery gate".into(), + input_ref: "journal:immutable-input".into(), + input: serde_json::json!({ "raw_body": body }), + instruction: "Record the immutable input body".into(), + profile: "builtin:subjektiv-journal-recording".into(), + serialization_key: None, + limits: Default::default(), + } + } + + fn digest() -> String { + job_request("保存済みの公開原文本文と経緯") + .input_digest() + .unwrap() + } + + fn malformed_digests() -> Vec { + vec![ + String::new(), + "sha256:not-a-digest".into(), + "a".repeat(64), + format!("sha256:{}", "A".repeat(64)), + format!("sha256:{}", "g".repeat(64)), + format!("sha256:{}", "a".repeat(63)), + format!("sha256:{}", "a".repeat(65)), + format!("SHA256:{}", "a".repeat(64)), + ] + } + + fn run(f: &Fixture, ctx: &JournalJobContext, request: Request) -> JournalResult { + service::execute_snapshot(f.store(), ctx, &digest(), request) + } + + fn attest(f: &Fixture, ctx: &JournalJobContext) { + service::attest_input_delivery(f.store(), ctx, &digest(), &f.ranges).unwrap(); + } + + #[test] + fn input_attestation_does_not_advance_journal_progress_and_replays_after_reopen() { + let mut f = Fixture::new(); + let (generation, ctx) = f.generation("immutable-input", f.ranges.clone()); + let before = f.state(); + let sources = f.sources(); + attest(&f, &ctx); + attest(&f, &ctx); + assert_eq!(f.state(), before); + assert_eq!(f.sources(), sources); + assert!(f.entries().is_empty()); + assert!(matches!( + service::finish(f.store(), &ctx), + Err(JournalError::Conflict(_)) + )); + assert!(f.store().journal_retention_required(&f.host).unwrap()); + f.reopen(); + attest(&f, &ctx); + assert_eq!(f.state(), before); + assert_eq!(f.sources(), sources); + f.store() + .database + .try_with_connection(|tx| { + let saved: (String, String) = tx.query_row( + "SELECT generation_id,input_digest FROM journal_generation_input_deliveries", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + assert_eq!(saved, (generation.id.clone(), digest())); + let source_deliveries: u64 = tx.query_row( + "SELECT COUNT(*) FROM journal_generation_deliveries", + [], + |r| r.get(0), + )?; + let receipts: u64 = + tx.query_row("SELECT COUNT(*) FROM journal_operations", [], |r| r.get(0))?; + assert_eq!((source_deliveries, receipts), (0, 0)); + Ok::<_, JournalError>(()) + }) + .unwrap(); + } + + #[test] + fn snapshot_gate_requires_matching_digest_and_exact_generation_selection() { + let mut f = Fixture::new(); + let (generation, ctx) = f.generation("digest-binding", f.ranges.clone()); + let record = f.operation("record", open_action("ongoing"), f.ranges.clone()); + for request in [ + Request::ReadState, + Request::Execute { + operation: record.clone(), + }, + ] { + assert!(matches!( + run(&f, &ctx, request), + Err(JournalError::Conflict(_)) + )); + } + let mut changed = f.ranges.clone(); + changed[0].bytes.start += 1; + for selection in [ + changed, + f.ranges[1..].to_vec(), + f.ranges.iter().rev().cloned().collect(), + ] { + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &digest(), &selection), + Err(JournalError::Conflict(_)) + )); + } + for malformed in malformed_digests() { + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &malformed, &f.ranges), + Err(JournalError::InvalidInput(_)) + )); + } + attest(&f, &ctx); + f.reopen(); + let different = job_request("改変された公開原文本文") + .input_digest() + .unwrap(); + assert_ne!(different, digest()); + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &different, &f.ranges), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + service::execute_snapshot(f.store(), &ctx, &different, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + service::execute_snapshot( + f.store(), + &ctx, + &different, + Request::Execute { + operation: record.clone() + } + ), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + run(&f, &ctx, Request::Execute { operation: record }), + Ok(Response::Saved(_)) + )); + f.close(&generation); + let (_, replacement) = f.generation("replacement-input", f.ranges.clone()); + assert!(matches!( + run(&f, &replacement, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + attest(&f, &replacement); + assert!(matches!( + run(&f, &replacement, Request::ReadState), + Ok(Response::State(_)) + )); + } + + #[test] + fn snapshot_execution_reads_journal_only_and_keeps_receipt_checkpoint_and_position_constraints() + { + let mut f = Fixture::new(); + let (_, ctx) = f.generation("source-free", f.ranges.clone()); + attest(&f, &ctx); + let source = Request::ReadText { + read: transport::JournalTextRead { + target: transport::JournalTextTarget::Source { + pointer: f.ranges[0].source.pointer.clone(), + }, + offset: 0, + limit: MAX_JOURNAL_READ_BYTES, + content_token: None, + }, + }; + let error = run(&f, &ctx, source).unwrap_err(); + assert!( + matches!(error, JournalError::PermissionDenied(ref message) if message.contains("Source target") && message.contains("immutable Job input")) + ); + let record = f.operation("saved-record", open_action("ongoing"), f.ranges.clone()); + let request = Request::Execute { + operation: record.clone(), + }; + let receipt = run(&f, &ctx, request.clone()).unwrap(); + // The lost response is recovered from the same operation after closing/reopening SQLite. + f.reopen(); + assert_eq!(run(&f, &ctx, request).unwrap(), receipt); + assert_eq!(f.entries().len(), 1); + assert_eq!(f.state().0.process_order, 0); + assert!(matches!( + service::finish(f.store(), &ctx), + Err(JournalError::Conflict(_)) + )); + let position = f.operation( + "saved-position", + JournalAction::UpdatePosition { + current_position: "現在地".into(), + }, + vec![], + ); + run( + &f, + &ctx, + Request::Execute { + operation: position, + }, + ) + .unwrap(); + for evidence in [ + vec![], + vec!["missing-record".into()], + vec!["saved-position".into()], + ] { + let operation = f.operation( + "invalid-checkpoint", + JournalAction::Checkpoint { + record_operation_ids: evidence, + }, + f.ranges.clone(), + ); + assert!(run(&f, &ctx, Request::Execute { operation }).is_err()); + assert_eq!( + (f.state().0.process_order, f.state().0.process_byte), + (0, 0) + ); + } + for (snapshot, expected) in [ + (transport::JournalSnapshot::Initial, ""), + (transport::JournalSnapshot::Current, "現在地"), + ] { + let response = run( + &f, + &ctx, + Request::ReadText { + read: transport::JournalTextRead { + target: transport::JournalTextTarget::Position { snapshot }, + offset: 0, + limit: 100, + content_token: None, + }, + }, + ) + .unwrap(); + assert!(matches!(response, Response::Text(page) if page.text == expected)); + } + for target in [ + transport::JournalTextTarget::OpenBody { + snapshot: transport::JournalSnapshot::Current, + }, + transport::JournalTextTarget::OpenOngoing { + snapshot: transport::JournalSnapshot::Current, + }, + ] { + assert!(matches!( + run( + &f, + &ctx, + Request::ReadText { + read: transport::JournalTextRead { + target, + offset: 0, + limit: 100, + content_token: None + } + } + ), + Ok(Response::Text(_)) + )); + } + assert!(matches!( + run( + &f, + &ctx, + Request::QueryOperation { + request_id: record.request_id.clone() + } + ), + Ok(Response::Operation(Some(_))) + )); + assert!( + matches!(run(&f, &ctx, Request::ListOperations { after: None, limit: 32 }), Ok(Response::Operations(page)) if page.items.len() == 2) + ); + assert!( + matches!(run(&f, &ctx, Request::ReadOperationCoverage { request_id: record.request_id.clone(), offset: 0, limit: 32 }), Ok(Response::Coverage(page)) if page.items == f.ranges) + ); + let checkpoint = f.operation( + "checkpoint", + JournalAction::Checkpoint { + record_operation_ids: vec![record.request_id], + }, + f.ranges.clone(), + ); + run( + &f, + &ctx, + Request::Execute { + operation: checkpoint, + }, + ) + .unwrap(); + f.reopen(); + assert_eq!( + (f.state().0.process_order, f.state().0.process_byte), + (2, 0) + ); + assert!(!f.store().journal_retention_required(&f.host).unwrap()); + let completion = service::finish(f.store(), &ctx).unwrap(); + assert_eq!((completion.process_order, completion.process_byte), (2, 0)); + assert!(matches!( + run(&f, &ctx, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges), + Err(JournalError::Conflict(_)) + )); + assert_eq!(service::finish(f.store(), &ctx).unwrap(), completion); + let attr = f.host.attribution.as_ref().unwrap(); + let binding = f + .store() + .claim_journal_writer(&f.host.subject_id, &attr.runtime_id, &attr.worker_id) + .unwrap(); + f.store().release_journal_writer(&binding).unwrap(); + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + run(&f, &ctx, Request::ReadState), + Err(JournalError::PermissionDenied(_)) + )); + } + + #[test] + fn retired_subject_rejects_late_attestation_and_snapshot_execution() { + let mut f = Fixture::new(); + let (_, ctx) = f.generation("retired-input", f.ranges.clone()); + attest(&f, &ctx); + let operation = f.operation( + "late-retired-record", + open_action("must not save"), + f.ranges.clone(), + ); + f.store().retire_subject(&f.host.subject_id).unwrap(); + f.reopen(); + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + run(&f, &ctx, Request::ReadState), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + run(&f, &ctx, Request::Execute { operation }), + Err(JournalError::PermissionDenied(_)) + )); + assert!(f.entries().is_empty()); + } + + #[test] + fn legacy_source_delivery_cannot_satisfy_immutable_input_gate() { + let f = Fixture::with_sizes(&[8]); + let (_, ctx) = f.generation("legacy-read", f.ranges.clone()); + service::execute_bounded( + f.store(), + &ctx, + Request::ReadText { + read: transport::JournalTextRead { + target: transport::JournalTextTarget::Source { + pointer: f.ranges[0].source.pointer.clone(), + }, + offset: 0, + limit: 8, + content_token: None, + }, + }, + |source, _, _| { + Ok(recording::JournalMaterial { + range: JournalSourceRange { + source: source.clone(), + bytes: JournalByteRange { start: 0, end: 8 }, + }, + text: "12345678".into(), + }) + }, + ) + .unwrap(); + assert!(matches!( + run(&f, &ctx, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + run( + &f, + &ctx, + Request::Execute { + operation: f.operation("new-record", open_action("ongoing"), f.ranges.clone()) + } + ), + Err(JournalError::Conflict(_)) + )); + } + + #[test] + fn input_delivery_migration_preserves_existing_subjects_and_enforces_binding_constraints() { + // Exercise the oldest Journal baseline, the immediate predecessor and fresh creation. + for baseline in [ + &crate::MIGRATIONS[..7], + &crate::MIGRATIONS[..12], + crate::MIGRATIONS, + ] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("migration"); + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-upgrade").unwrap(); + let registration = scope + .register(feature_storage::FeatureRegistration::new( + crate::SUBJEKTIV_FEATURE_ID, + baseline, + )) + .unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + let subject = store + .create_subject_with_behavior( + SubjectRole::new("preserved").unwrap(), + "Existing behavior".into(), + ) + .unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + drop(manager); + + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-upgrade").unwrap(); + let registration = SubjektivStore::register(&scope).unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone())); + let host = session_host(&store, &subject.id, "migration-session"); + let ranges = vec![source_range("migration-session", 0, 8)]; + let id = store + .accept_journal_sources(&host, "Migration", &ranges) + .unwrap(); + let binding = JobAttemptBinding { + job_id: "migration-job".into(), + attempt_id: "attempt".into(), + }; + let generation = store + .prepare_journal_generation(&host, &id, binding.clone(), ranges.clone()) + .unwrap(); + let ctx = JournalJobContext::validated(&store, &host, &generation.id, binding).unwrap(); + service::attest_input_delivery(&store, &ctx, &digest(), &ranges).unwrap(); + store + .close_journal_generation(&host, &generation.id, &generation.job) + .unwrap(); + let second = store + .prepare_journal_generation( + &host, + &id, + JobAttemptBinding { + job_id: "next-migration-job".into(), + attempt_id: "attempt".into(), + }, + ranges.clone(), + ) + .unwrap(); + store.database.try_transaction(|tx| { + let columns = tx.prepare("PRAGMA table_info(journal_generation_input_deliveries)")?.query_map([], |r| r.get::<_, String>(1))?.collect::, _>>()?; + assert_eq!(columns, ["generation_id", "input_digest"]); + assert!(tx.execute("INSERT INTO journal_generation_input_deliveries VALUES(?1,?2)", rusqlite::params![generation.id, digest()]).is_err()); + assert!(tx.execute("INSERT INTO journal_generation_input_deliveries VALUES('missing-generation',?1)", [digest()]).is_err()); + assert!(tx.execute("UPDATE journal_generation_input_deliveries SET input_digest=?1", [job_request("changed body").input_digest().unwrap()]).is_err()); + // Digest validation is a schema constraint as well as an API boundary check. + for invalid in malformed_digests() { + assert!(tx.execute("INSERT INTO journal_generation_input_deliveries VALUES(?1,?2)", rusqlite::params![second.id, invalid]).is_err()); + } + let saved: String = tx.query_row("SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1", [&generation.id], |r| r.get(0))?; + assert_eq!(saved, digest()); + Ok::<_, JournalError>(()) + }).unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + } + } + + #[test] + fn input_delivery_migration_failure_rolls_back_table_creation_and_retries() { + // A colliding trigger is created before migration so failure occurs after its new table DDL. + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("rollback"); + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-rollback").unwrap(); + let registration = scope + .register(feature_storage::FeatureRegistration::new( + crate::SUBJEKTIV_FEATURE_ID, + &crate::MIGRATIONS[..12], + )) + .unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + let subject = store + .create_subject_with_behavior( + SubjectRole::new("preserved").unwrap(), + "Retain on failure".into(), + ) + .unwrap(); + store.database.try_transaction(|tx| { + tx.execute_batch("CREATE TRIGGER journal_generation_input_delivery_no_update BEFORE UPDATE ON subjects BEGIN SELECT RAISE(ABORT, 'injected migration collision'); END;")?; + Ok::<_, JournalError>(()) + }).unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + drop(manager); + + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-rollback").unwrap(); + let registration = SubjektivStore::register(&scope).unwrap(); + assert!(matches!( + SubjektivStore::open(&scope, ®istration), + Err(crate::SubjektivError::Storage( + feature_storage::FeatureStorageError::MigrationFailed { .. } + )) + )); + drop(scope); + manager.shutdown().unwrap(); + drop(manager); + + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-rollback").unwrap(); + let registration = scope + .register(feature_storage::FeatureRegistration::new( + crate::SUBJEKTIV_FEATURE_ID, + &crate::MIGRATIONS[..12], + )) + .unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone())); + store.database.try_transaction(|tx| { + assert!(!tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='journal_generation_input_deliveries')", [], |r| r.get::<_, bool>(0))?); + tx.execute_batch("DROP TRIGGER journal_generation_input_delivery_no_update")?; + Ok::<_, JournalError>(()) + }).unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + drop(manager); + + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-rollback").unwrap(); + let registration = SubjektivStore::register(&scope).unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + assert_eq!(store.subject(&subject.id).unwrap(), Some(subject)); + store.database.try_with_connection(|tx| { + assert!(tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='journal_generation_input_deliveries')", [], |r| r.get::<_, bool>(0))?); + Ok::<_, JournalError>(()) + }).unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + } + + #[test] + fn late_input_attestation_and_execution_cannot_revive_cancelled_or_erased_generations() { + for erased in [false, true] { + let mut f = Fixture::new(); + let (generation, ctx) = f.generation("late-input", f.ranges.clone()); + attest(&f, &ctx); + let record = f.operation( + "late-record", + open_action("must not save"), + f.ranges.clone(), + ); + if erased { + f.store() + .erase_journal_session(&f.host, "erase-input") + .unwrap(); + } else { + f.close(&generation); + f.generation("replacement", f.ranges.clone()); + } + f.reopen(); + assert!(service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges).is_err()); + assert!(run(&f, &ctx, Request::ReadState).is_err()); + assert!(run(&f, &ctx, Request::Execute { operation: record }).is_err()); + f.store() + .database + .try_with_connection(|tx| { + let count: u64 = tx.query_row( + "SELECT COUNT(*) FROM journal_generation_input_deliveries", + [], + |r| r.get(0), + )?; + assert_eq!(count, u64::from(!erased)); + let records: u64 = + tx.query_row("SELECT COUNT(*) FROM journal_operations", [], |r| r.get(0))?; + assert_eq!(records, 0); + Ok::<_, JournalError>(()) + }) + .unwrap(); + } + } +} + +mod snapshot_inputs { + //! Component coverage joins shared canonical values, Journal input construction, SQLite + //! generations, and the production snapshot service. Synthetic Host-attested captures and + //! a saved Job file replace only capture/dispatch ownership; no Worker or Session reader runs. + use super::*; + use recording::JournalInput; + use session_store::capture::{ + CaptureBody, CaptureDto, CaptureEntryDto, CaptureLimits, CaptureSource, CaptureUsePolicy, + RecordingPosition, + }; + use session_store::public_material::{ + PublicMaterialDescriptor, PublicMaterialIdentity, PublicMaterialOrigin, + }; + use transport::{JournalJobRequest as Request, JournalJobResponse as Response}; + + fn canonical_capture(session: &str, bodies: &[String]) -> CaptureDto { + let capture = CaptureDto { + version: 1, + session_id: session.into(), + segment_id: Uuid::from_u128(3).to_string(), + view_entry_count: bodies.len() as u64, + entries: bodies + .iter() + .enumerate() + .map(|(index, body)| { + let segment_id = Uuid::from_u128(1 + (index / 50) as u128); + let entry_id = format!("synthetic-{index}"); + let origin = session_store::LoggedSessionHistoryOrigin::LocalHumanInput; + CaptureEntryDto { + entry_id: entry_id.clone(), + order: index as u64, + source: CaptureSource { + segment_id: segment_id.to_string(), + entry_index: index as u64, + provenance: serde_json::from_value(serde_json::json!("human_input")) + .unwrap(), + history_origin: Some(origin.clone()), + derived_from: vec![], + }, + policy: CaptureUsePolicy { + recording_material: true, + memory_evidence: false, + observation_readable: false, + }, + body: CaptureBody::CanonicalPublic { + descriptor: PublicMaterialDescriptor { + identity: PublicMaterialIdentity { + segment_id, + entry_ref: format!("E{entry_id}"), + }, + canonical_order: index as u64, + origin: PublicMaterialOrigin::History { origin }, + sha256: Sha256::digest(body.as_bytes()) + .iter() + .map(|b| format!("{b:02x}")) + .collect(), + total_bytes: body.len() as u64, + }, + byte_offset: 0, + text: body.clone(), + boundary: None, + }, + } + }) + .collect(), + }; + capture.validate(CaptureLimits::default()).unwrap(); + capture + } + + fn admitted_capture( + f: &Fixture, + bodies: &[String], + ) -> (JournalHostContext, String, CaptureDto) { + let host = session_host(f.store(), &f.host.subject_id, "canonical-input-session"); + let capture = canonical_capture("canonical-input-session", bodies); + let mut journal_id = None; + for ranges in recording::capture_ranges(&capture) + .unwrap() + .chunks(MAX_JOURNAL_RANGES) + { + let id = f + .store() + .accept_journal_sources(&host, "Canonical input", ranges) + .unwrap(); + if let Some(prior) = &journal_id { + assert_eq!(prior, &id); + } + journal_id = Some(id); + } + (host, journal_id.unwrap(), capture) + } + + fn operation( + f: &Fixture, + host: &JournalHostContext, + id: &str, + request_id: &str, + action: JournalAction, + ranges: Vec, + ) -> JournalOperation { + JournalOperation { + request_id: request_id.into(), + expected_state: f.store().read_journal(host, id).unwrap().state_token, + action, + covered_ranges: ranges, + } + } + + fn seed_initial_state( + f: &Fixture, + host: &JournalHostContext, + id: &str, + ranges: Vec, + ) { + let binding = JobAttemptBinding { + job_id: "seed-initial-state".into(), + attempt_id: "seed-attempt".into(), + }; + let generation = f + .store() + .prepare_journal_generation(host, id, binding.clone(), ranges) + .unwrap(); + let ctx = JournalJobContext::validated(f.store(), host, &generation.id, binding).unwrap(); + f.store() + .execute_journal( + &ctx, + operation( + f, + host, + id, + "seed-open", + open_action(&"Original full open body 日本語\n\"\\".repeat(300)), + vec![], + ), + ) + .unwrap(); + f.store() + .execute_journal( + &ctx, + operation( + f, + host, + id, + "seed-position", + JournalAction::UpdatePosition { + current_position: "Initial position 日本語\n\"\\".repeat(200), + }, + vec![], + ), + ) + .unwrap(); + f.store() + .close_journal_generation(host, &generation.id, &generation.job) + .unwrap(); + } + + fn job_request(job_id: &str, input: &JournalInput) -> ::job::JobRequest { + let request = ::job::JobRequest { + job_id: job_id.into(), + purpose: "Journal canonical input".into(), + input_ref: format!("journal:{}", input.journal_id), + input: serde_json::to_value(input).unwrap(), + instruction: "Process saved immutable bodies only".into(), + profile: "builtin:subjektiv-journal-recording".into(), + serialization_key: None, + limits: Default::default(), + }; + request.validate().unwrap(); + request + } + + fn run( + f: &Fixture, + ctx: &JournalJobContext, + request: &::job::JobRequest, + action: Request, + ) -> JournalResult { + service::execute_snapshot(f.store(), ctx, &request.input_digest().unwrap(), action) + } + + #[test] + fn complete_input_budget_and_successive_canonical_prefixes_preserve_all_escaped_utf8_bytes() { + let f = Fixture::new(); + let mut bodies: Vec<_> = (0..131) + .map(|i| format!("[user {i}] 公開原文\n\"\\\u{1}\t").repeat(8)) + .collect(); + bodies[60] = "[long public result] 日本語🦀\n\"\\\u{1}\t".repeat(14_000); + let (host, id, capture) = admitted_capture(&f, &bodies); + let all_ranges = recording::capture_ranges(&capture).unwrap(); + seed_initial_state(&f, &host, &id, all_ranges[..1].to_vec()); + let mut reconstructed = vec![String::new(); bodies.len()]; + let mut jobs = 0; + let mut partial = false; + for step in 0..20 { + let before = f.store().journal_resume_state(&host, &id).unwrap(); + if before.0.process_order == bodies.len() as u64 { + break; + } + let callback_count = std::cell::Cell::new(0); + let input = JournalInput::build(f.store(), &host, &id, |order, byte, budget| { + callback_count.set(callback_count.get() + 1); + assert_eq!( + (order, byte), + (before.0.process_order, before.0.process_byte) + ); + Ok(capture + .recording_slice( + RecordingPosition { + canonical_order: order, + byte_offset: byte, + }, + MAX_JOURNAL_RANGES, + budget, + ) + .unwrap() + .capture) + }) + .unwrap(); + assert_eq!(callback_count.get(), 1); + let request = job_request(&format!("bounded-{step}"), &input); + assert!( + serde_json::to_vec(&request.input).unwrap().len() <= ::job::MAX_JOB_INPUT_BYTES + ); + assert_eq!(JournalInput::decode(&request.input).unwrap(), input); + assert_eq!(input.initial_state_token, before.0.state_token); + assert_eq!( + input.initial_open_id, + before.1.as_ref().map(|e| e.id.clone()) + ); + let binding = JobAttemptBinding { + job_id: request.job_id.clone(), + attempt_id: "attempt".into(), + }; + let generation = f + .store() + .prepare_journal_input(&host, &input, binding.clone()) + .unwrap(); + input.validate_generation(&generation).unwrap(); + let ctx = + JournalJobContext::validated(f.store(), &host, &generation.id, binding).unwrap(); + service::attest_input_delivery( + f.store(), + &ctx, + &request.input_digest().unwrap(), + &input.selected_ranges, + ) + .unwrap(); + // Capture, input construction, generation preparation and delivery cannot reflect bytes. + assert_eq!(f.store().journal_resume_state(&host, &id).unwrap(), before); + for range in &input.selected_ranges { + let (_, offset, text) = input.capture.recording_text(range.source.order).unwrap(); + let target = &mut reconstructed[range.source.order as usize]; + assert_eq!(offset, target.len() as u64); + assert_eq!(range.bytes.start, offset); + assert_eq!(range.bytes.end, offset + text.len() as u64); + assert_eq!( + text, + &bodies[range.source.order as usize] + [range.bytes.start as usize..range.bytes.end as usize] + ); + target.push_str(text); + partial |= range.bytes.end < range.source.total_bytes; + } + let record_id = format!("bounded-record-{step}"); + run( + &f, + &ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + &record_id, + JournalAction::ConfirmUnchanged, + input.selected_ranges.clone(), + ), + }, + ) + .unwrap(); + run( + &f, + &ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + &format!("bounded-position-{step}"), + JournalAction::UpdatePosition { + current_position: format!("Processed prefix {step}"), + }, + vec![], + ), + }, + ) + .unwrap(); + run( + &f, + &ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + &format!("bounded-checkpoint-{step}"), + JournalAction::Checkpoint { + record_operation_ids: vec![record_id], + }, + input.selected_ranges, + ), + }, + ) + .unwrap(); + service::finish(f.store(), &ctx).unwrap(); + jobs += 1; + } + assert!( + jobs > 2, + "the long escaped input and 131 entries must require multiple prefixes" + ); + assert!( + partial, + "a long UTF-8 body must be split, not previewed or dropped" + ); + assert_eq!(reconstructed, bodies); + let journal = f.store().read_journal(&host, &id).unwrap(); + assert_eq!((journal.process_order, journal.process_byte), (131, 0)); + for range in all_ranges { + assert_eq!( + f.store() + .read_journal_source(&host, &id, range.source.order) + .unwrap() + .reflected_bytes, + range.source.total_bytes + ); + } + } + + #[test] + fn journal_input_decode_rejects_mismatched_ranges_policy_session_tampered_text_and_missing_body() + { + let f = Fixture::new(); + let (host, id, capture) = admitted_capture( + &f, + &["[user] 公開原文\n".into(), "[user] 次の原文\n".into()], + ); + let input = + JournalInput::build(f.store(), &host, &id, |_, _, _| Ok(capture.clone())).unwrap(); + let original = serde_json::to_value(input).unwrap(); + let cases: &[(&str, fn(&mut serde_json::Value))] = &[ + ("range", |v| { + v["selected_ranges"][0]["bytes"]["end"] = serde_json::json!(1) + }), + ("recording policy", |v| { + v["capture"]["entries"][0]["policy"]["recording_material"] = + serde_json::json!(false) + }), + ("cross-purpose policy", |v| { + v["capture"]["entries"][0]["policy"]["memory_evidence"] = serde_json::json!(true) + }), + ("session", |v| { + v["capture"]["session_id"] = serde_json::json!("different-session") + }), + ("full text digest", |v| { + let text = &mut v["capture"]["entries"][0]["body"]["text"]; + *text = serde_json::json!("X".repeat(text.as_str().unwrap().len())); + }), + ("Journal version", |v| v["version"] = serde_json::json!(2)), + ("capture version", |v| { + v["capture"]["version"] = serde_json::json!(2) + }), + ("missing body", |v| { + v["capture"]["entries"][0] + .as_object_mut() + .unwrap() + .remove("body"); + }), + ("missing text", |v| { + v["capture"]["entries"][0]["body"] + .as_object_mut() + .unwrap() + .remove("text"); + }), + ]; + for (name, corrupt) in cases { + let mut value = original.clone(); + corrupt(&mut value); + assert!( + JournalInput::decode(&value).is_err(), + "accepted corrupt input: {name}" + ); + } + } + + #[test] + fn released_writer_recovers_exact_saved_input_and_completion_without_renewing_mutation_authority() + { + let mut f = Fixture::new(); + // This Subject has only the tested Session; unrelated fixture history must not + // participate in the writer's pending-source release protection. + let subject = f + .store() + .create_subject_with_behavior( + SubjectRole::new("completion-recovery").unwrap(), + String::new(), + ) + .unwrap(); + let host = session_host(f.store(), &subject.id, "released-completion-session"); + let writer = f + .store() + .claim_journal_writer(&subject.id, "runtime", "worker") + .unwrap(); + let capture = canonical_capture( + "released-completion-session", + &["[user] 記録済みの公開本文\n".into()], + ); + let ranges = recording::capture_ranges(&capture).unwrap(); + let id = f + .store() + .accept_journal_sources(&host, "Retained completion", &ranges) + .unwrap(); + seed_initial_state(&f, &host, &id, ranges.clone()); + let input = JournalInput::build(f.store(), &host, &id, |_, _, _| Ok(capture)).unwrap(); + let request = job_request("retained-completion-job", &input); + let saved_request = serde_json::to_vec(&request).unwrap(); + let binding = JobAttemptBinding { + job_id: request.job_id.clone(), + attempt_id: "original-attempt".into(), + }; + let generation = f + .store() + .prepare_journal_input(&host, &input, binding.clone()) + .unwrap(); + let old_ctx = + JournalJobContext::validated(f.store(), &host, &generation.id, binding.clone()) + .unwrap(); + service::attest_input_delivery( + f.store(), + &old_ctx, + &request.input_digest().unwrap(), + &input.selected_ranges, + ) + .unwrap(); + run( + &f, + &old_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "completed-record", + JournalAction::ConfirmUnchanged, + ranges.clone(), + ), + }, + ) + .unwrap(); + run( + &f, + &old_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "completed-position", + JournalAction::UpdatePosition { + current_position: "Saved position after all selected input".into(), + }, + vec![], + ), + }, + ) + .unwrap(); + run( + &f, + &old_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "completed-checkpoint", + JournalAction::Checkpoint { + record_operation_ids: vec!["completed-record".into()], + }, + ranges, + ), + }, + ) + .unwrap(); + let saved_completion = service::finish(f.store(), &old_ctx).unwrap(); + let finished_journal = f.store().read_journal(&host, &id).unwrap(); + assert_eq!( + ( + saved_completion.process_order, + saved_completion.process_byte + ), + (1, 0) + ); + f.store().release_journal_writer(&writer).unwrap(); + assert!( + f.store() + .journal_worker_was_released(&subject.id, "runtime", "worker") + .unwrap() + ); + f.reopen(); + + // Reconciliation reads the same immutable Job input, not a new capture or the + // current Journal projection. Release did not erase this retained completion. + let request: ::job::JobRequest = serde_json::from_slice(&saved_request).unwrap(); + let input = JournalInput::decode(&request.input).unwrap(); + let retained = f + .store() + .prepare_journal_input(&host, &input, binding.clone()) + .unwrap(); + assert_eq!(retained, generation); + let recovery = + JournalJobContext::validated_recovery(f.store(), &host, &retained.id, binding.clone()) + .unwrap(); + assert_eq!( + service::finish(f.store(), &recovery).unwrap(), + saved_completion + ); + assert_eq!( + f.store().read_journal(&host, &id).unwrap(), + finished_journal + ); + assert!(matches!( + JournalJobContext::validated(f.store(), &host, &retained.id, binding.clone()), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + f.store().prepare_journal_input( + &host, + &input, + JobAttemptBinding { + job_id: binding.job_id.clone(), + attempt_id: "new-attempt".into() + } + ), + Err(JournalError::PermissionDenied(_)) + )); + let mut changed = input.clone(); + changed.initial_state_token.push('x'); + assert!(matches!( + f.store().prepare_journal_input(&host, &changed, binding), + Err(JournalError::Conflict(_)) + )); + let late_mutation = operation( + &f, + &host, + &id, + "late-released-position", + JournalAction::UpdatePosition { + current_position: "Must not replace saved position".into(), + }, + vec![], + ); + assert!(matches!( + run( + &f, + &old_ctx, + &request, + Request::Execute { + operation: late_mutation.clone() + } + ), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + f.store().execute_journal(&recovery, late_mutation), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + service::attest_input_delivery( + f.store(), + &old_ctx, + &request.input_digest().unwrap(), + &input.selected_ranges + ), + Err(JournalError::PermissionDenied(_)) + )); + assert_eq!( + f.store().read_journal(&host, &id).unwrap(), + finished_journal + ); + assert_eq!( + service::finish(f.store(), &recovery).unwrap(), + saved_completion + ); + } + + #[test] + fn journal_input_generation_validation_rejects_foreign_scope_and_changed_initial_proof() { + let f = Fixture::new(); + let (host, id, capture) = admitted_capture(&f, &["[user] original canonical body".into()]); + let ranges = recording::capture_ranges(&capture).unwrap(); + seed_initial_state(&f, &host, &id, ranges); + let input = JournalInput::build(f.store(), &host, &id, |_, _, _| Ok(capture)).unwrap(); + let generation = f + .store() + .prepare_journal_input( + &host, + &input, + JobAttemptBinding { + job_id: "generation-proof".into(), + attempt_id: "attempt".into(), + }, + ) + .unwrap(); + input.validate_generation(&generation).unwrap(); + let cases: &[(&str, fn(&mut JournalGeneration))] = &[ + ("Journal", |g| g.journal_id.push('x')), + ("Subject", |g| g.initial_journal.subject_id.push('x')), + ("Session", |g| g.initial_journal.source_session_id.push('x')), + ("initial content condition", |g| { + g.initial_journal.state_token.push('x') + }), + ("initial open entry", |g| { + g.initial_open.as_mut().unwrap().id.push('x') + }), + ("previous fixed boundary", |g| { + g.previous_fixed_end = Some(g.selected_ranges[0].source.pointer.clone()) + }), + ("selection", |g| g.selected_ranges[0].bytes.end -= 1), + ]; + for (name, change) in cases { + let mut changed = generation.clone(); + change(&mut changed); + assert!( + matches!( + input.validate_generation(&changed), + Err(JournalError::Conflict(_)) + ), + "accepted changed generation proof: {name}" + ); + } + } + + #[test] + fn same_saved_job_input_rebinds_full_initial_snapshot_and_reuses_partial_record_receipts_after_reopen() + { + let mut f = Fixture::new(); + let bodies = vec![ + "[user] 最初の原文は長いUTF8\n".repeat(30), + "[user] 続きの確定原文\n".repeat(20), + ]; + let (host, id, capture) = admitted_capture(&f, &bodies); + let ranges = recording::capture_ranges(&capture).unwrap(); + seed_initial_state(&f, &host, &id, ranges.clone()); + let input = JournalInput::build(f.store(), &host, &id, |_, _, _| Ok(capture)).unwrap(); + let request = job_request("durable-saved-input", &input); + let saved_path = f.temp.path().join("immutable-job.json"); + std::fs::write(&saved_path, serde_json::to_vec(&request).unwrap()).unwrap(); + let first_binding = JobAttemptBinding { + job_id: request.job_id.clone(), + attempt_id: "first".into(), + }; + let first = f + .store() + .prepare_journal_input(&host, &input, first_binding.clone()) + .unwrap(); + assert_eq!( + f.store() + .prepare_journal_input(&host, &input, first_binding.clone()) + .unwrap(), + first + ); + input.validate_generation(&first).unwrap(); + let first_ctx = + JournalJobContext::validated(f.store(), &host, &first.id, first_binding).unwrap(); + service::attest_input_delivery( + f.store(), + &first_ctx, + &request.input_digest().unwrap(), + &input.selected_ranges, + ) + .unwrap(); + let record = operation( + &f, + &host, + &id, + "durable-record", + JournalAction::Rewrite { + entry_id: first.initial_open.as_ref().unwrap().id.clone(), + title: "Saved activity".into(), + body: "Both original ranges reflected in this record".into(), + ongoing: "Continue work".into(), + }, + ranges.clone(), + ); + let saved_record = run( + &f, + &first_ctx, + &request, + Request::Execute { + operation: record.clone(), + }, + ) + .unwrap(); + let saved_position = run( + &f, + &first_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "first-position", + JournalAction::UpdatePosition { + current_position: "Position after the successful record".into(), + }, + vec![], + ), + }, + ) + .unwrap(); + let mut cut = bodies[0].len() / 2; + while !bodies[0].is_char_boundary(cut) { + cut -= 1; + } + let saved_checkpoint = run( + &f, + &first_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "partial-checkpoint", + JournalAction::Checkpoint { + record_operation_ids: vec![record.request_id.clone()], + }, + vec![fragment(&ranges[0], 0, cut as u64)], + ), + }, + ) + .unwrap(); + assert_eq!( + ( + f.store().read_journal(&host, &id).unwrap().process_order, + f.store().read_journal(&host, &id).unwrap().process_byte + ), + (0, cut as u64) + ); + f.store() + .close_journal_generation(&host, &first.id, &first.job) + .unwrap(); + f.reopen(); + // Only the saved input and SQLite are reopened. There is no latest-source callback. + let retry_request: ::job::JobRequest = + serde_json::from_slice(&std::fs::read(saved_path).unwrap()).unwrap(); + assert_eq!(retry_request, request); + let retry_input = JournalInput::decode(&retry_request.input).unwrap(); + let retry_binding = JobAttemptBinding { + job_id: retry_request.job_id.clone(), + attempt_id: "second".into(), + }; + let mut changed_initial = retry_input.clone(); + changed_initial.initial_state_token.push('x'); + assert!(matches!( + changed_initial.validate_generation(&first), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + f.store() + .prepare_journal_input(&host, &changed_initial, retry_binding.clone()), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + f.store().prepare_journal_input( + &host, + &retry_input, + JobAttemptBinding { + job_id: "different-job-cannot-borrow-initial-proof".into(), + attempt_id: "attempt".into() + } + ), + Err(JournalError::Conflict(_)) + )); + let retry = f + .store() + .prepare_journal_input(&host, &retry_input, retry_binding.clone()) + .unwrap(); + assert_ne!(retry.id, first.id); + assert_eq!(retry.initial_journal, first.initial_journal); + assert_eq!(retry.initial_open, first.initial_open); + assert_eq!(retry.previous_fixed_end, first.previous_fixed_end); + assert_eq!(retry.selected_ranges, first.selected_ranges); + assert_ne!( + retry.initial_journal.state_token, + f.store().read_journal(&host, &id).unwrap().state_token + ); + retry_input.validate_generation(&retry).unwrap(); + assert_eq!( + f.store() + .prepare_journal_input(&host, &retry_input, retry_binding.clone()) + .unwrap(), + retry + ); + let retry_ctx = + JournalJobContext::validated(f.store(), &host, &retry.id, retry_binding).unwrap(); + service::attest_input_delivery( + f.store(), + &retry_ctx, + &retry_request.input_digest().unwrap(), + &retry_input.selected_ranges, + ) + .unwrap(); + assert!(matches!( + run(&f, &first_ctx, &request, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + run( + &f, + &first_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "late-first-attempt", + JournalAction::ConfirmUnchanged, + ranges.clone() + ) + } + ), + Err(JournalError::Conflict(_)) + )); + for (request_id, saved) in [ + (record.request_id.as_str(), saved_record), + ("first-position", saved_position), + ("partial-checkpoint", saved_checkpoint), + ] { + let Response::Saved(receipt) = saved else { + panic!("saved operation receipt expected") + }; + assert_eq!( + f.store() + .query_journal_operation(&host, &id, request_id) + .unwrap(), + Some(receipt) + ); + } + assert!( + matches!(run(&f, &retry_ctx, &retry_request, Request::ReadOperationCoverage { request_id: record.request_id.clone(), offset: 0, limit: 32 }), Ok(Response::Coverage(page)) if page.items == ranges) + ); + for (target, expected) in [ + ( + transport::JournalTextTarget::Position { + snapshot: transport::JournalSnapshot::Initial, + }, + first.initial_journal.current_position.clone(), + ), + ( + transport::JournalTextTarget::OpenBody { + snapshot: transport::JournalSnapshot::Initial, + }, + first.initial_open.as_ref().unwrap().body.clone(), + ), + ( + transport::JournalTextTarget::OpenOngoing { + snapshot: transport::JournalSnapshot::Initial, + }, + first.initial_open.as_ref().unwrap().ongoing.clone(), + ), + ] { + assert!( + matches!(run(&f, &retry_ctx, &retry_request, Request::ReadText { read: transport::JournalTextRead { target, offset: 0, limit: MAX_JOURNAL_READ_BYTES, content_token: None } }), Ok(Response::Text(page)) if page.text == expected) + ); + } + let remainder = vec![ + fragment(&ranges[0], cut as u64, ranges[0].bytes.end), + ranges[1].clone(), + ]; + let missing_position = operation( + &f, + &host, + &id, + "retry-checkpoint-needs-position", + JournalAction::Checkpoint { + record_operation_ids: vec![record.request_id.clone()], + }, + remainder.clone(), + ); + assert!(matches!( + run( + &f, + &retry_ctx, + &retry_request, + Request::Execute { + operation: missing_position + } + ), + Err(JournalError::Conflict(_)) + )); + run( + &f, + &retry_ctx, + &retry_request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "retry-position", + JournalAction::UpdatePosition { + current_position: "Retry confirmed saved activity".into(), + }, + vec![], + ), + }, + ) + .unwrap(); + run( + &f, + &retry_ctx, + &retry_request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "remainder-checkpoint", + JournalAction::Checkpoint { + record_operation_ids: vec![record.request_id], + }, + remainder, + ), + }, + ) + .unwrap(); + let completion = service::finish(f.store(), &retry_ctx).unwrap(); + assert_eq!((completion.process_order, completion.process_byte), (2, 0)); + assert_eq!( + f.store() + .list_journal_entries(&host, &id, None, 32) + .unwrap() + .items + .len(), + 1 + ); + } +} + use super::*; use crate::SubjectRole; use feature_storage::{FeatureStorage, ScopedFeatureStorage}; diff --git a/crates/subjektiv/src/journal/transport.rs b/crates/subjektiv/src/journal/transport.rs index cb43bd9b0..4981f185b 100644 --- a/crates/subjektiv/src/journal/transport.rs +++ b/crates/subjektiv/src/journal/transport.rs @@ -1,8 +1,10 @@ //! Bounded, Host-bound Journal recording and sequential Job transport. //! //! Subject, Journal and generation authority never come from these model inputs. -//! Hosts validate live execution and public source bytes before selection; reading a -//! preview must not select the enclosing entry. Operation retries retain the entire +//! Hosts validate live execution and public source bytes before selection, save body-bearing +//! immutable Job input, and attest its exact digest and dispatch plan through the service. +//! Model requests cannot attest input delivery; Source targets are rejected by snapshot execution. +//! Initial/current Journal reads remain separate from input delivery. Operation retries retain the entire //! JournalOperation, including its original expected_state and covered_ranges. use serde::{Deserialize, Serialize}; @@ -50,8 +52,8 @@ pub struct JournalJobState { pub current_journal: JournalMetadata, pub current_open: Option, pub previous_fixed_end: Option, - /// Frozen permission to fetch this prefix, not evidence of delivery or reflection. - /// The sequential service gates mutations on its durable Source-read delivery ledger. + /// Frozen source selection delivered in immutable Job input, not evidence of reflection. + /// Production execution requires the trusted Host's matching input-digest attestation. pub selected_ranges: Vec, } @@ -83,7 +85,8 @@ pub enum JournalTextTarget { OpenOngoing { snapshot: JournalSnapshot, }, - /// The Host restricts this pointer to this Job's frozen public source input. + /// Legacy source-read adapter only. Production snapshot execution rejects this target; + /// raw source bodies must already be present in the immutable Job input. Source { pointer: JournalSourcePointer, }, diff --git a/crates/subjektiv/src/lib.rs b/crates/subjektiv/src/lib.rs index 86ba70a6e..43bd67689 100644 --- a/crates/subjektiv/src/lib.rs +++ b/crates/subjektiv/src/lib.rs @@ -639,6 +639,11 @@ static MIGRATIONS: &[FeatureMigration] = &[ "add explicit Subject permission sources and immutable child execution grants", permissions::migrate, ), + FeatureMigration::new( + 13, + "bind Journal generation delivery to immutable Job input digest", + journal::service::migrate_input_delivery, + ), ]; pub const REGISTRATION: FeatureRegistration = diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index e7daf4906..425bd0fb5 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -444,6 +444,31 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { material_pin(&self.state, request, true) } + async fn session_recording_capture( + &self, + request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result + { + let scope = material_scope(&self.state, &request.workspace_id)?; + #[cfg(feature = "fs-store")] + { + Ok(self + .state + .runtime + .session_recording_capture_scoped(&scope, &request)) + } + #[cfg(not(feature = "fs-store"))] + { + let _ = scope; + Ok( + runtime_api::SessionRecordingCaptureAvailability::Unavailable { + reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable, + message: "Committed Session storage unavailable".into(), + }, + ) + } + } + async fn session_material_catalog( &self, request: runtime_api::SessionMaterialCatalogRequest, diff --git a/crates/worker-runtime/src/http_server/tests/session_material_tests.rs b/crates/worker-runtime/src/http_server/tests/session_material_tests.rs index 733f658ac..f47845f92 100644 --- a/crates/worker-runtime/src/http_server/tests/session_material_tests.rs +++ b/crates/worker-runtime/src/http_server/tests/session_material_tests.rs @@ -150,6 +150,27 @@ impl Fixture { limit: 1, } } + fn recording_request(&self) -> runtime_api::SessionRecordingCaptureRequest { + runtime_api::SessionRecordingCaptureRequest { + workspace_id: self.scope.workspace_id.clone(), + source: self.source(), + expected_session_id: self.session.to_string(), + start: Default::default(), + max_entries: 1000, + max_serialized_bytes: 128 * 1024, + } + } + fn capture(&self, source: Source) -> session_store::capture::RecordingCaptureSlice { + let mut request = self.recording_request(); + request.source = source; + match self + .runtime + .session_recording_capture_scoped(&self.scope, &request) + { + runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice, + other => panic!("{other:?}"), + } + } fn catalog(&self) -> runtime_api::SessionMaterialCatalogPage { match self .runtime @@ -226,6 +247,9 @@ fn pending_session_proof_is_not_missing_material_and_pin_release_allows_empty_ar .is_err(), "retention intent is not source availability" ); + assert!( + matches!(f.runtime.session_recording_capture_scoped(&f.scope, &f.recording_request()), runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id } if session_id == f.session.to_string()) + ); // A missing committed source is never the same fact as a pending pointer. metadata .write( @@ -479,6 +503,7 @@ fn source_scope_session_and_active_pointer_mismatches_never_return_material() { #[test] fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() { let f = Fixture::new(); + let frozen = f.capture(f.source()); let original: Vec<_> = f.reader(&f.source()).descriptors().cloned().collect(); let pin = SessionRetentionPin(uuid::Uuid::now_v7()); f.runtime @@ -516,6 +541,7 @@ fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() let archived = Source::Archived { archive_id: "canonical-archive".into(), }; + assert_eq!(f.capture(archived.clone()), frozen); let reader = f.reader(&archived); assert_eq!(reader.descriptors().cloned().collect::>(), original); let body = reader.read_content(&original[0], 0, 16384).unwrap(); @@ -951,11 +977,52 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router .authorizer(SignedAuthorizer { identity, trust }) .build() .unwrap(); + let frozen = match client + .session_recording_capture(f.recording_request()) + .await + .unwrap() + { + runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice, + other => panic!("{other:?}"), + }; + frozen.capture.validate(Default::default()).unwrap(); + assert!(frozen.next.is_none()); + assert!( + serde_json::to_vec(&frozen).unwrap().len() + <= f.recording_request().max_serialized_bytes + ); + let snapshot = serde_json::to_vec(&frozen).unwrap(); + assert!(!String::from_utf8_lossy(&snapshot).contains("private-system-prompt")); + f.store + .append( + f.session, + f.segment, + &LogEntry::AnnotatedAssistantItem { + ts: 6, + entry: message("late-answer", LoggedRole::Assistant, "not in frozen input"), + }, + ) + .unwrap(); + assert_eq!(serde_json::to_vec(&frozen).unwrap(), snapshot); let page = match client.session_material_catalog(f.request()).await.unwrap() { Catalog::Page { page } => page, other => panic!("{other:?}"), }; let descriptor = page.descriptors[0].clone(); + assert_eq!( + frozen + .capture + .recording_text(0) + .unwrap() + .0 + .identity + .entry_ref, + descriptor.entry_ref + ); + assert_eq!( + frozen.capture.recording_text(0).unwrap().0.sha256, + descriptor.sha256 + ); let read = client .session_material_read(runtime_api::SessionMaterialReadRequest { workspace_id: "workspace-a".into(), @@ -1019,6 +1086,13 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router ); } assert!(!f.root().exists()); + assert!(matches!( + client + .session_recording_capture(f.recording_request()) + .await + .unwrap(), + runtime_api::SessionRecordingCaptureAvailability::Unavailable { .. } + )); assert!(matches!( client.session_material_catalog(f.request()).await.unwrap(), Catalog::Unavailable { .. } @@ -1050,41 +1124,54 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() { None, workspace_test_auth(vec![trust.clone()]), ); - let path = "/v1/session-public/material/catalog"; - let body = serde_json::to_vec(&f.request()).unwrap(); - let claims = signed_claims(&trust, "POST", path, &body); - let first = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await; - assert_eq!(first.status(), StatusCode::OK); - let replay = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await; - assert_eq!(replay.status(), StatusCode::UNAUTHORIZED); - let claims = signed_claims(&trust, "POST", path, &body); - let changed = signed_request( - app.clone(), - &identity, - &claims, - b"{}".to_vec(), - "workspace-a", - ) - .await; - assert_eq!(changed.status(), StatusCode::UNAUTHORIZED); - let mut wrong_scope = f.request(); - wrong_scope.workspace_id = "other-workspace".into(); - let wrong_body = serde_json::to_vec(&wrong_scope).unwrap(); - let claims = signed_claims(&trust, "POST", path, &wrong_body); - let response = signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await; - assert_eq!(response.status(), StatusCode::FORBIDDEN); - let unsigned = app - .oneshot( - Request::builder() - .method("POST") - .uri(path) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(body)) - .unwrap(), + for (path, body) in [ + ( + "/v1/session-public/material/catalog", + serde_json::to_vec(&f.request()).unwrap(), + ), + ( + "/v1/session-public/recording-capture", + serde_json::to_vec(&f.recording_request()).unwrap(), + ), + ] { + let claims = signed_claims(&trust, "POST", path, &body); + let first = + signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await; + assert_eq!(first.status(), StatusCode::OK); + let replay = + signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await; + assert_eq!(replay.status(), StatusCode::UNAUTHORIZED); + let claims = signed_claims(&trust, "POST", path, &body); + let changed = signed_request( + app.clone(), + &identity, + &claims, + b"{}".to_vec(), + "workspace-a", ) - .await - .unwrap(); - assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); + .await; + assert_eq!(changed.status(), StatusCode::UNAUTHORIZED); + let mut wrong_scope: serde_json::Value = serde_json::from_slice(&body).unwrap(); + wrong_scope["workspace_id"] = "other-workspace".into(); + let wrong_body = serde_json::to_vec(&wrong_scope).unwrap(); + let claims = signed_claims(&trust, "POST", path, &wrong_body); + let response = + signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let unsigned = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); + } } #[tokio::test] @@ -1170,3 +1257,152 @@ async fn explicit_material_erasure_requires_delete_capability_and_matching_scope ); assert!(f.root().exists()); } + +#[test] +fn recording_capture_rejects_scope_session_cursor_and_budget_without_returning_bodies() { + use runtime_api::SessionRecordingCaptureAvailability as Capture; + let f = Fixture::new(); + let request = f.recording_request(); + let frozen = f.capture(f.source()); + assert_eq!(frozen.capture.entries.len(), 3); + let mut request = request; + for invalid in 0..5 { + let mut bad = request.clone(); + match invalid { + 0 => bad.workspace_id = "other".into(), + 1 => bad.expected_session_id = session_store::new_session_id().to_string(), + 2 => bad.start.canonical_order = 100, + 3 => bad.max_entries = 0, + 4 => bad.max_serialized_bytes = 4 * 1024 * 1024 + 1, + _ => unreachable!(), + } + assert!(matches!( + f.runtime.session_recording_capture_scoped(&f.scope, &bad), + Capture::Unavailable { .. } + )); + } + let text = frozen.capture.recording_text(1).unwrap().2; + request.start = session_store::capture::RecordingPosition { + canonical_order: 1, + byte_offset: text.find('🍎').unwrap() as u64 + 1, + }; + assert!(matches!( + f.runtime + .session_recording_capture_scoped(&f.scope, &request), + Capture::Unavailable { + reason: Reason::InvalidCursor, + .. + } + )); + request.start = Default::default(); + request.max_entries = 1; + let Capture::Captured { slice } = f + .runtime + .session_recording_capture_scoped(&f.scope, &request) + else { + panic!() + }; + assert_eq!(slice.capture.entries.len(), 1); + assert_eq!(slice.next.unwrap().canonical_order, 1); +} + +#[test] +fn recording_capture_compaction_preserves_original_descriptors_and_excludes_summary_and_new_tail() { + let f = Fixture::new(); + let old = f.capture(f.source()); + let compact = session_store::new_segment_id(); + let mut summary = message( + "summary", + LoggedRole::System, + "derived summary is not original", + ); + summary.metadata.input_metadata = Some( + session_store::HistoryInputMetadata::new( + session_store::HistoryInputUsePolicy { + recording_material: session_store::InputEvidenceUse::Exclude, + memory_evidence: session_store::InputEvidenceUse::Exclude, + observation: session_store::InputObservationUse::Hidden, + compaction: session_store::InputCompactionUse::OpeningSummary, + }, + None, + None, + ) + .unwrap(), + ); + summary.metadata.derivation = Some(session_store::LoggedHistoryDerivation { + sources: vec![ + LoggedSessionHistoryEntryId("user-1".into()), + LoggedSessionHistoryEntryId("answer-1".into()), + ], + }); + f.store + .create_segment( + f.session, + compact, + &[ + LogEntry::AnnotatedSegmentStart { + ts: 6, + session_id: f.session, + system_prompt: Some("private".into()), + config: Default::default(), + history: vec![summary], + forked_from: None, + compacted_from: Some(session_store::SegmentOrigin { + segment_id: f.segment, + at_turn_index: 0, + }), + }, + LogEntry::AnnotatedAssistantItem { + ts: 7, + entry: message( + "new-answer", + LoggedRole::Assistant, + "new original after compaction", + ), + }, + ], + ) + .unwrap(); + WorkerAggregateStore::new( + f.root().parent().unwrap(), + format!("worker-runtime-{}", f.worker.worker_id), + ) + .unwrap() + .set_active( + &format!("worker-runtime-{}", f.worker.worker_id), + Some(session_store::WorkerActiveSegmentRef::active_segment( + f.session, compact, + )), + None, + ) + .unwrap(); + let frozen = f.capture(f.source()); + assert_eq!(frozen.capture.entries.len(), old.capture.entries.len() + 1); + for entry in &old.capture.entries { + assert_eq!( + frozen.capture.recording_text(entry.order).unwrap(), + old.capture.recording_text(entry.order).unwrap() + ); + } + assert!( + !serde_json::to_string(&frozen) + .unwrap() + .contains("derived summary is not original") + ); + let bytes = serde_json::to_vec(&frozen).unwrap(); + f.store + .append( + f.session, + compact, + &LogEntry::AnnotatedAssistantItem { + ts: 8, + entry: message("tail", LoggedRole::Assistant, "next Job only"), + }, + ) + .unwrap(); + assert_eq!(serde_json::to_vec(&frozen).unwrap(), bytes); + assert_eq!( + f.capture(f.source()).capture.entries.len(), + frozen.capture.entries.len() + 1 + ); +} diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 086969155..38dc5cb9f 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -4949,6 +4949,133 @@ impl Runtime { .map_err(crate::retention::retention_error) } + /// Freeze committed original public bodies once; no catalog/read pagination + /// or restored Worker is involved. Publication and erasure share this fence. + #[cfg(feature = "fs-store")] + pub fn session_recording_capture_scoped( + &self, + scope: &RuntimeWorkspaceScope, + request: &runtime_api::SessionRecordingCaptureRequest, + ) -> runtime_api::SessionRecordingCaptureAvailability { + use crate::worker_backend::SessionMaterialSourceError as Error; + use runtime_api::{ + SessionPublicUnavailableReason as R, SessionRecordingCaptureAvailability as A, + }; + use session_store::public_material::{PublicMaterialError as E, PublicMaterialLimits}; + let result = (|| -> Result<_, Error> { + if scope.workspace_id != request.workspace_id { + return Err("Session Workspace scope mismatch".into()); + } + if !(1..=4096).contains(&request.max_entries) + || !(1024..=4 * 1024 * 1024).contains(&request.max_serialized_bytes) + { + return Err(E::ResourceLimit.into()); + } + let session = request + .expected_session_id + .parse::() + .map_err(|_| "Invalid Session identity")?; + // Resolve attribution without reading archive bodies. Do not acquire + // the Runtime mutex under the Session lock: archive execution takes + // those locks in the opposite order. + let (root, archive) = self + .resolve_session_public_source_with_validation(scope, &request.source, false) + .map_err(|e| e.to_string())?; + let runtime_dir = { + let state = self.lock().map_err(|e| e.to_string())?; + state + .fs_store() + .ok_or("Session storage unavailable")? + .runtime_dir() + .to_path_buf() + }; + // This is the storage append fence used by live Runtime publishers, + // also fencing archive transfer and erase. Resolve the pointer under it. + let retention = crate::retention::runtime_session_retention(&runtime_dir); + let guard = retention.lock_live(session).map_err(|error| match error { + session_store::StoreError::SessionErased(_) => E::Erased, + _ => E::Unavailable, + })?; + let mut archive_to_verify = None; + let active = match (&request.source, archive) { + (runtime_api::SessionPublicSource::Retained { worker_id }, None) => { + let metadata = session_store::worker_metadata::WorkerAggregateStore::open_read_only( + root.parent().ok_or("Session root is unavailable")?, + crate::worker_backend::ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&WorkerRef::new(*worker_id)), + ).and_then(|store| store.read_read_only()).map_err(|e| e.to_string())?.ok_or("Session metadata missing")?; + if metadata.workspace_id.as_deref() != Some(scope.workspace_id.as_str()) { + return Err("Session Workspace mismatch".into()); + } + let active = metadata.active.ok_or("Session active pointer missing")?; + if active.session_id != session { + return Err("Session identity mismatch".into()); + } + match active.segment_id { + Some(segment) => segment, + None if request.start + == session_store::capture::RecordingPosition::default() + && crate::retention::unmaterialized_session( + root.parent().ok_or("Session root unavailable")?, + &scope.workspace_id, + ) + .map_err(|e| e.to_string())? + == Some(session) => + { + return Ok(A::NotMaterialized { + session_id: session.to_string(), + }); + } + None => return Err("Session active Segment missing".into()), + } + } + (runtime_api::SessionPublicSource::Archived { archive_id }, Some(manifest)) => { + archive_to_verify = Some((archive_id, manifest.clone())); + if manifest.source_session_id != request.expected_session_id { + return Err("Archive Session identity mismatch".into()); + } + manifest + .source_active_segment_id + .ok_or("Archive active Segment missing")? + } + _ => return Err("Session source unavailable".into()), + }; + let slice = session_store::capture::capture_recording_slice_worker_guarded( + &root, + session, + active, + PublicMaterialLimits::default(), + request.start, + request.max_entries, + request.max_serialized_bytes, + &guard, + )?; + if let Some((archive_id, manifest)) = archive_to_verify { + // Validate archive checksums only after capture froze every file + // end, still under the same publication/erase fence. + let (verified, verified_root) = FsWorkerRetentionProvider::new(&runtime_dir) + .session_archive(archive_id) + .map_err(|e| e.to_string())?; + if verified != manifest || verified_root != root { + return Err(E::Changed.into()); + } + } + Ok(A::Captured { slice }) + })(); + match result { + Ok(availability) => availability, + Err(error) => A::Unavailable { + reason: match error { + Error::Material(E::ResourceLimit) => R::ResourceLimit, + Error::Material(E::InvalidOffset | E::Changed) => R::InvalidCursor, + Error::Material(E::Missing | E::Erased) => R::RetentionMissing, + Error::Material(E::Inconsistent | E::Record(_)) => R::CorruptLog, + _ => R::StorageUnavailable, + }, + message: "Committed Session recording capture unavailable".into(), + }, + } + } + #[cfg(feature = "fs-store")] pub fn session_material_catalog_scoped( &self, diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index a67fd9d6a..83dd338a7 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -730,7 +730,7 @@ impl ProfileRuntimeWorkerFactory { .map_err(|error| format!("failed to clean restore run artifacts: {error}")) } - fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String { + pub(crate) fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String { format!("worker-runtime-{}", worker_ref.worker_id) } diff --git a/crates/worker/src/job/tests.rs b/crates/worker/src/job/tests.rs index f1eaefddf..4c867bd61 100644 --- a/crates/worker/src/job/tests.rs +++ b/crates/worker/src/job/tests.rs @@ -179,11 +179,11 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{}})), prose(), ]); - // This runner/Feature fixture has no selected sources; descriptor delivery gates - // for nonempty ranges are owned and tested by the Subjektiv service. + // This runner/Feature fixture has no selected sources; actual body-bearing input, + // trusted delivery and checkpoint gates are covered by the Host integration suites. let mut journal_request = request("builtin:subjektiv-journal-recording"); - journal_request.input = json!({"selected_ranges":[],"origins":[]}); - journal_request.instruction = "Read immutable selected_ranges through ReadJournalText Source before recording; descriptors are not source bodies.".into(); + journal_request.input = json!({"selected_ranges":[],"capture":{"entries":[]}}); + journal_request.instruction = "Record only the immutable canonical bodies in this Job input. Do not read original Sessions through ReadJournalText Source.".into(); let job = prepare_job_from_registry_with_grant( &journal_request, root.path(), @@ -249,7 +249,7 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s "ReadJournalText", "Source", "content_token", - "Conflict", + "input digest", ] { assert!( prompt.contains(contract), diff --git a/crates/workspace-server/src/assignment_tracking_tests.rs b/crates/workspace-server/src/assignment_tracking_tests.rs index 2e9085438..152b48f14 100644 --- a/crates/workspace-server/src/assignment_tracking_tests.rs +++ b/crates/workspace-server/src/assignment_tracking_tests.rs @@ -476,7 +476,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip (99, "Explicit hosting administrators and self-registration policy".into()), (100, "Provisional users and scoped passkey activation".into()), (101, "Durable explicit Subject body release".into()), - (102, "Durable failed Subject creation recovery".into()) + (102, "Durable failed Subject creation recovery".into()), + (103, "Job input retirement independent of immutable receipts".into()) ] ); for (table, rows) in retained.iter().zip(original.iter()) { diff --git a/crates/workspace-server/src/backend_job.rs b/crates/workspace-server/src/backend_job.rs index 6d804bc97..e49d552c5 100644 --- a/crates/workspace-server/src/backend_job.rs +++ b/crates/workspace-server/src/backend_job.rs @@ -463,10 +463,52 @@ impl BackendJobRequest { } } +/// Availability of the immutable input body, independent of execution outcome. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JobInputDisposition { + Available, + Reclaimed, + Erased, + Invalidated, +} + +impl JobInputDisposition { + pub fn as_str(self) -> &'static str { + match self { + Self::Available => "available", + Self::Reclaimed => "reclaimed", + Self::Erased => "erased", + Self::Invalidated => "invalidated", + } + } +} + +/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JobInputRetirementReason { + Completed, + ExplicitErasure, + LegacyInvalidation, +} + +impl JobInputRetirementReason { + pub fn disposition(self) -> JobInputDisposition { + match self { + Self::Completed => JobInputDisposition::Reclaimed, + Self::ExplicitErasure => JobInputDisposition::Erased, + Self::LegacyInvalidation => JobInputDisposition::Invalidated, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct BackendJobRecord { pub workspace_id: String, + /// Receipt metadata. After retirement `input` is null; use the verified loader to execute. pub request: BackendJobRequest, + pub input_digest: String, + pub input_disposition: JobInputDisposition, pub intent_fingerprint: String, pub state: BackendJobState, pub current_attempt: u8, @@ -479,6 +521,28 @@ pub struct BackendJobRecord { pub completed_at: Option, } +impl BackendJobRecord { + /// Verify the original immutable body and envelope before any execution or retry. + /// No Session read or replay request may repair unavailable/corrupt input. + pub fn load_executable_request(&self) -> Result<&BackendJobRequest> { + if self.input_disposition != JobInputDisposition::Available { + return Err(Error::InvalidInput(format!( + "Backend Job input is {}", + self.input_disposition.as_str() + ))); + } + self.request.validate()?; + if self.request.input_digest()? != self.input_digest + || self.request.fingerprint()? != self.intent_fingerprint + { + return Err(Error::Store( + "Backend Job immutable input or intent fingerprint mismatch".into(), + )); + } + Ok(&self.request) + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum BackendJobWorkerCleanupState { @@ -665,6 +729,8 @@ mod tests { "workspace_id": "workspace-a", "request": request(), "intent_fingerprint": request().fingerprint().unwrap(), + "input_digest": request().input_digest().unwrap(), + "input_disposition": "available", "state": "pending", "current_attempt": 2, "created_at": "created", @@ -1124,3 +1190,17 @@ mod tests { } } } + +pub(crate) const INPUT_RETENTION_TRIGGERS: &str = r#" +CREATE TRIGGER backend_job_identity_immutable BEFORE UPDATE OF + workspace_id, job_id, purpose, input_digest, input_ref, resource_key, intent_fingerprint ON backend_jobs +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job identity'); END; +CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON backend_jobs +WHEN ( + ((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated')) + OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased')) + AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') + AND json_type(NEW.request_json,'$.input') = 'null' +) IS NOT TRUE +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job input retirement'); END; +"#; diff --git a/crates/workspace-server/src/frozen_content_state_tests.rs b/crates/workspace-server/src/frozen_content_state_tests.rs index 954d42ccf..b2003d2ca 100644 --- a/crates/workspace-server/src/frozen_content_state_tests.rs +++ b/crates/workspace-server/src/frozen_content_state_tests.rs @@ -71,6 +71,11 @@ fn legacy_content_database(path: &Path) -> (Connection, String) { conn.execute("INSERT INTO workspace_config_tree_revisions VALUES('space',?1,?2,'toolchain','projection',?3,?4,'{}')", params![ordinal,snapshot.digest,manifest,created]).unwrap(); } + seed_legacy_content_job(&conn); + (conn, snapshot.digest) +} + +fn seed_legacy_content_job(conn: &Connection) { let request = BackendJobRequest { job_id: "legacy-job".into(), purpose: "migration_test".into(), @@ -93,7 +98,19 @@ fn legacy_content_database(path: &Path) -> (Connection, String) { VALUES('space','legacy-job','legacy-job:attempt:1',1,'old-number','dispatched','runtime','worker','2026-08-14T00:00:00Z','2026-08-13T00:00:00Z','2026-08-13T00:00:00Z'); INSERT INTO backend_job_deliveries(workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,created_at,updated_at) VALUES('space','delivery','legacy-job','legacy-job:attempt:1','runtime','target','pending','created','updated');") .unwrap(); - (conn, snapshot.digest) +} + +// Replay/resource checks use the current Store only after the full retained chain. +// Keep the focused partial DDL fixture above for the cutover's schema assertions. +fn legacy_content_job_database(path: &Path) -> Connection { + super::tests::prepare_retained_schema(path, 85); + let conn = Connection::open(path).unwrap(); + configure_sqlite(&conn).unwrap(); + conn.execute_batch("INSERT INTO accounts VALUES('owner','user','owner','Owner','created','updated'); + INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) + VALUES('space','owner','Space','active','created','updated');").unwrap(); + seed_legacy_content_job(&conn); + conn } #[test] @@ -432,7 +449,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ ] { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("content.db"); - let (conn, _) = legacy_content_database(&path); + let conn = legacy_content_job_database(&path); conn.execute( "UPDATE backend_job_attempts SET state=?1, runtime_id=CASE WHEN ?1='reserved' THEN NULL ELSE runtime_id END, @@ -441,11 +458,22 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ ) .unwrap(); migrate_content_state_v85_to_v86(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 86); + let cutover: (String, String, Option) = conn.query_row( + "SELECT j.state,a.state,a.worker_cleanup_state FROM backend_jobs j JOIN backend_job_attempts a USING(workspace_id,job_id)", + [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ).unwrap(); + assert_eq!( + cutover, + if state == "reserved" { + ("failed".into(), "failed".into(), Some("completed".into())) + } else { + ("unknown".into(), "unknown".into(), Some("pending".into())) + } + ); + apply_migrations(&conn).unwrap(); drop(conn); - let store = SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), - conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), - }; + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let job = store .get_backend_job("space", "legacy-job") .unwrap() @@ -478,7 +506,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("content.db"); - let (conn, _) = legacy_content_database(&path); + let conn = legacy_content_job_database(&path); let result = serde_json::json!({"finding":"preserved"}); let (digest, encoded) = job::result_digest(&result, 4096).unwrap(); conn.execute("UPDATE backend_jobs SET state='completed',result_json=?1,result_digest=?2,completed_at='finished'",params![encoded,digest]).unwrap(); @@ -490,11 +518,23 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() { ) .unwrap(); migrate_content_state_v85_to_v86(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 86); + let cutover: (String, String, String, String) = conn.query_row( + "SELECT j.state,j.result_json,a.worker_cleanup_state,a.worker_cleanup_completed_at FROM backend_jobs j JOIN backend_job_attempts a USING(workspace_id,job_id)", + [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), + ).unwrap(); + assert_eq!( + cutover, + ( + "completed".into(), + encoded, + "completed".into(), + "cleaned".into() + ) + ); + apply_migrations(&conn).unwrap(); drop(conn); - let store = SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), - conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), - }; + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let job = store .get_backend_job("space", "legacy-job") .unwrap() diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 0f9cd24a1..f657bac06 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -1326,6 +1326,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { Err("Session source permanent erasure is unavailable on this Runtime".into()) } + fn session_recording_capture( + &self, + _request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result { + Err("committed recording capture is unavailable on this Runtime".into()) + } + fn session_material_catalog( &self, _request: runtime_api::SessionMaterialCatalogRequest, @@ -2336,6 +2343,21 @@ impl RuntimeRegistry { }) } + pub fn session_recording_capture( + &self, + runtime_id: &str, + request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result { + validate_backend_identifier("runtime_id", runtime_id)?; + self.runtime(runtime_id)? + .session_recording_capture(request) + .map_err(|message| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: runtime_id.into(), + code: "session_recording_capture_failed".into(), + message, + }) + } + pub fn session_material_catalog( &self, runtime_id: &str, @@ -3441,6 +3463,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { }) } + fn session_recording_capture( + &self, + request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result { + if request.workspace_id != self.workspace_id { + return Err("recording capture Workspace mismatch".into()); + } + let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend"); + Ok(self + .runtime + .session_recording_capture_scoped(&scope, &request)) + } + fn session_material_catalog( &self, request: runtime_api::SessionMaterialCatalogRequest, @@ -5677,6 +5712,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { .map_err(|diagnostic| diagnostic.message) } + fn session_recording_capture( + &self, + request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result { + self.run_runtime_api( + self.request_timeout, + MAX_REMOTE_RUNTIME_RESPONSE_BYTES, + move |client| async move { client.session_recording_capture(request).await }, + ) + .map_err(|diagnostic| diagnostic.message) + } + fn session_material_catalog( &self, request: runtime_api::SessionMaterialCatalogRequest, diff --git a/crates/workspace-server/src/implementation_result_tests.rs b/crates/workspace-server/src/implementation_result_tests.rs index c5a7cd7d2..a46555429 100644 --- a/crates/workspace-server/src/implementation_result_tests.rs +++ b/crates/workspace-server/src/implementation_result_tests.rs @@ -381,6 +381,10 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; + DROP TRIGGER backend_job_input_retirement; + DROP TRIGGER backend_job_identity_immutable; + ALTER TABLE backend_jobs DROP COLUMN input_disposition; + UPDATE __yoi_schema_migrations SET version=102 WHERE version=103; DROP TABLE subject_create_recoveries; UPDATE __yoi_schema_migrations SET version=101 WHERE version=102; DROP TABLE subject_worker_releases; @@ -517,6 +521,10 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow // Reconstruct the prior composition before its new business capture existed. conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture; DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; + DROP TRIGGER backend_job_input_retirement; + DROP TRIGGER backend_job_identity_immutable; + ALTER TABLE backend_jobs DROP COLUMN input_disposition; + UPDATE __yoi_schema_migrations SET version=102 WHERE version=103; DROP TABLE subject_create_recoveries; UPDATE __yoi_schema_migrations SET version=101 WHERE version=102; DROP TABLE subject_worker_releases; @@ -679,7 +687,11 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() { if historical { conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture; DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; - DROP TABLE subject_create_recoveries; + DROP TRIGGER backend_job_input_retirement; + DROP TRIGGER backend_job_identity_immutable; + ALTER TABLE backend_jobs DROP COLUMN input_disposition; + UPDATE __yoi_schema_migrations SET version=102 WHERE version=103; + DROP TABLE subject_create_recoveries; UPDATE __yoi_schema_migrations SET version=101 WHERE version=102; DROP TABLE subject_worker_releases; UPDATE __yoi_schema_migrations SET version=100 WHERE version=101; diff --git a/crates/workspace-server/src/job_notification_routing.rs b/crates/workspace-server/src/job_notification_routing.rs index a5bf62efa..c3245a497 100644 --- a/crates/workspace-server/src/job_notification_routing.rs +++ b/crates/workspace-server/src/job_notification_routing.rs @@ -2,8 +2,8 @@ //! lifecycle, are the durable source. Fixed Worker recipients never follow leases. use super::*; use crate::backend_job::{ - BackendJobAttemptState, BackendJobDeliveryRecord, BackendJobDeliveryState, - BackendJobResultAcceptance, BackendJobState, + BackendJobAttemptState, BackendJobDeliveryRecord, BackendJobDeliveryState, BackendJobRequest, + BackendJobState, }; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -103,6 +103,32 @@ pub(in crate::store) fn install(conn: &Connection) -> Result<()> { Ok(()) } +struct NotificationResultSnapshot { + request: BackendJobRequest, + result: Option, + result_digest: Option, +} + +// The schema-96 import runs before later Job lifecycle migrations. Read only +// the retained result envelope, not the executable Job's current-schema columns. +fn read_result_snapshot( + conn: &Connection, + workspace: &str, + job: &str, +) -> Result { + let (request, result, result_digest): (String, Option, Option) = conn + .query_row( + "SELECT request_json, result_json, result_digest FROM backend_jobs WHERE workspace_id=?1 AND job_id=?2", + params![workspace, job], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + Ok(NotificationResultSnapshot { + request: decode(&request)?, + result: result.map(|value| decode(&value)).transpose()?, + result_digest, + }) +} + pub(in crate::store) fn capture( conn: &Connection, workspace: &str, @@ -115,8 +141,7 @@ pub(in crate::store) fn capture( attempt_id: attempt.into(), delivery_id: id.clone(), }; - let job = read_backend_job(conn, workspace, job)? - .ok_or_else(|| Error::Store("Job notification source missing".into()))?; + let job = read_result_snapshot(conn, workspace, job)?; let target = &job.request.notification_target; let facts = serde_json::json!({"principal_kind":if target.is_some(){Some("worker")}else{None}, "principal_id":null,"runtime_id":target.as_ref().map(|w|&w.runtime_id),"worker_id":target.as_ref().map(|w|&w.worker_id),"status":null,"case_state":"open","released":false,"backend_job":source,"recipient_kind":"fixed_worker","job_purpose":job.request.purpose,"result_digest":job.result_digest}); conn.execute("INSERT OR IGNORE INTO notification_sources(workspace_id,ticket_id,input_event_id,role,assignment_id,kind,facts_json) VALUES(?1,NULL,?2,'backend_job','','backend_job_result',?3)",params![workspace,id,json(&facts)?])?; @@ -290,22 +315,13 @@ fn import_legacy(conn: &Connection, workspace: &str, job_id: &str, attempt_id: & | "notification_too_large", ) => ("defer", NotificationReason::SourceUnavailable), _ => { - let job = read_backend_job(conn, workspace, job_id)? - .ok_or_else(|| Error::Store("Legacy Job result missing".into()))?; - let attempt = read_backend_job_attempt(conn, workspace, job_id, attempt_id)? - .ok_or_else(|| Error::Store("Legacy Job attempt missing".into()))?; - let acceptance = BackendJobResultAcceptance { - job, - attempt, - replayed: true, - }; - let facts = if acceptance.job.request.notification_target.as_ref() != Some(&old.target) - { + let snapshot = read_result_snapshot(conn, workspace, job_id)?; + let facts = if snapshot.request.notification_target.as_ref() != Some(&old.target) { JobNotificationFacts::Failed(NotificationFailureCode::PreparationFailed) } else { - legacy_result_facts(&acceptance) + legacy_result_facts(&snapshot, job_id, attempt_id) }; - let decision = policy(acceptance.job.request.notification_target.is_some(), &facts); + let decision = policy(snapshot.request.notification_target.is_some(), &facts); match facts { JobNotificationFacts::Result(body) => content = Some(body), JobNotificationFacts::Failed(code) => failure = Some(code), @@ -338,28 +354,26 @@ fn import_legacy(conn: &Connection, workspace: &str, job_id: &str, attempt_id: & Ok(()) } -fn legacy_result_facts(acceptance: &BackendJobResultAcceptance) -> JobNotificationFacts { - let content = if acceptance.job.request.purpose == crate::ticket_item_checker::PURPOSE { - let Some(result) = &acceptance.job.result else { +fn legacy_result_facts( + snapshot: &NotificationResultSnapshot, + job_id: &str, + attempt_id: &str, +) -> JobNotificationFacts { + let content = if snapshot.request.purpose == crate::ticket_item_checker::PURPOSE { + let Some(result) = &snapshot.result else { return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult); }; - match crate::ticket_item_checker::validate_result(&acceptance.job.request.input, result) { + match crate::ticket_item_checker::validate_result(&snapshot.request.input, result) { Ok(result) if result.findings.is_empty() => return JobNotificationFacts::NoFindings, Ok(_) => {} Err(_) => return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult), } - crate::ticket_item_checker::render_result(acceptance) + crate::ticket_item_checker::render_result(&snapshot.request.input, result) } else { Ok(format!( - "Backend Job `{}` completed attempt `{}` for `{}`. Durable result digest: {}.", - acceptance.job.request.job_id, - acceptance.attempt.attempt_id, - acceptance.job.request.purpose, - acceptance - .job - .result_digest - .as_deref() - .unwrap_or("unavailable") + "Backend Job `{job_id}` completed attempt `{attempt_id}` for `{}`. Durable result digest: {}.", + snapshot.request.purpose, + snapshot.result_digest.as_deref().unwrap_or("unavailable") )) }; match content { diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 489e78728..30244ac7f 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -789,6 +789,7 @@ CREATE TABLE "worker_registry" ( FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE ); CREATE TABLE backend_jobs ( + input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated')), workspace_id TEXT NOT NULL, job_id TEXT NOT NULL, purpose TEXT NOT NULL, @@ -1642,6 +1643,19 @@ CREATE TABLE IF NOT EXISTS ticket_assignment_work_releases ( FOREIGN KEY (workspace_id, assignment_id) REFERENCES ticket_worker_assignments(workspace_id, assignment_id) ON DELETE CASCADE ); + +CREATE TRIGGER backend_job_identity_immutable BEFORE UPDATE OF + workspace_id, job_id, purpose, input_digest, input_ref, resource_key, intent_fingerprint ON backend_jobs +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job identity'); END; +CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON backend_jobs +WHEN ( + ((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated')) + OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased')) + AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') + AND json_type(NEW.request_json,'$.input') = 'null' +) IS NOT TRUE +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job input retirement'); END; + -- Ticket case assignment work constraints (schema 91). DROP VIEW IF EXISTS ticket_active_worker_assignments; CREATE VIEW ticket_active_worker_assignments AS diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 809f7d203..6ca61c905 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -4635,11 +4635,79 @@ impl WorkspaceApi { &self, reservation: BackendJobReservation, ) -> Result { + // Reread even a terminal/stale reservation before deciding whether it can + // dispatch or replay: an explicit erasure may have retired its saved input. + let journal_gate = reservation + .job + .request + .grants + .subjektiv_journal + .as_ref() + .map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id)) + .transpose()?; + let _journal_guard = journal_gate + .as_ref() + .map(|gate| { + gate.lock() + .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into())) + }) + .transpose()?; + let current_job = self + .store + .get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)? + .ok_or_else(|| Error::Store("reserved Backend Job disappeared".into()))?; + let current_attempt = self + .store + .get_backend_job_attempt( + &self.config.workspace_id, + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + )? + .ok_or_else(|| Error::Store("reserved Backend attempt disappeared".into()))?; + let reservation = BackendJobReservation { + job: current_job, + attempt: current_attempt, + replayed: reservation.replayed, + resource_reused: reservation.resource_reused, + }; + if matches!( + reservation.job.input_disposition, + crate::backend_job::JobInputDisposition::Erased + | crate::backend_job::JobInputDisposition::Invalidated + ) { + return Err(Error::WorkspacePermissionDenied( + "erased or invalidated Backend Job input cannot dispatch".into(), + )); + } if reservation.job.state != BackendJobState::Pending || reservation.attempt.state != BackendJobAttemptState::Reserved || reservation.attempt.worker.is_some() { - return Ok(reservation); + return Ok(BackendJobReservation { + replayed: true, + ..reservation + }); + } + reservation.job.load_executable_request()?; + if subjektiv_journal::migrate_reserved_legacy(self, &reservation)? { + let job = self + .store + .get_backend_job(self.workspace_id(), &reservation.job.request.job_id)? + .ok_or_else(|| Error::Store("retired legacy Job missing".into()))?; + let attempt = self + .store + .get_backend_job_attempt( + self.workspace_id(), + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + )? + .ok_or_else(|| Error::Store("retired legacy attempt missing".into()))?; + return Ok(BackendJobReservation { + job, + attempt, + replayed: reservation.replayed, + resource_reused: reservation.resource_reused, + }); } subjektiv_journal::prepare(self, &reservation)?; let (reservation, claimed) = self.store.claim_backend_job_attempt_dispatch( @@ -4844,6 +4912,7 @@ impl WorkspaceApi { .to_string(), }); } + subjektiv_journal::attest_delivery(self, &reservation)?; let job = self .store .get_backend_job(&self.config.workspace_id, &request.job_id)? @@ -5220,6 +5289,16 @@ impl WorkspaceApi { subjektiv_journal::close(self, &job.request, ¤t)?; if current.state == BackendJobAttemptState::Completed { subjektiv_journal::drain_after_completion(self, &job.request); + if job.input_disposition + == crate::backend_job::JobInputDisposition::Available + { + self.store.reclaim_backend_job_input( + self.workspace_id(), + job_id, + &job.input_digest, + crate::backend_job::JobInputRetirementReason::Completed, + )?; + } } } } diff --git a/crates/workspace-server/src/server/subjektiv_journal.rs b/crates/workspace-server/src/server/subjektiv_journal.rs index 5abfd49b2..fccee17a6 100644 --- a/crates/workspace-server/src/server/subjektiv_journal.rs +++ b/crates/workspace-server/src/server/subjektiv_journal.rs @@ -4,7 +4,7 @@ use super::*; use crate::backend_job::{ BackendJobGrants, BackendJobRecord, SubjektivJournalGrant, journal_serialization_key, }; -use subjektiv::journal::recording::{JournalMaterial, select_materials_bounded}; +use subjektiv::journal::recording::JournalInput; use subjektiv::journal::transport::*; use subjektiv::journal::{self, *}; @@ -92,20 +92,82 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) - }; let store = store(api)?; let host = host(api, &store, grant)?; - store - .prepare_journal_generation( + let input = saved_input(&reservation.job.request, grant)?; + let g = store + .prepare_journal_input( &host, - &grant.journal_id, + &input, subjektiv::api::JobAttemptBinding { job_id: reservation.job.request.job_id.clone(), attempt_id: reservation.attempt.attempt_id.clone(), }, - grant.selected_ranges.clone(), ) .map_err(domain)?; + input.validate_generation(&g).map_err(domain)?; Ok(()) } +fn legacy_descriptor(request: &BackendJobRequest) -> bool { + let Some(grant) = request.grants.subjektiv_journal.as_ref() else { + return false; + }; + let Some(object) = request.input.as_object() else { + return false; + }; + object.len() == 2 + && object.get("journal_id").and_then(|v| v.as_str()) == Some(&grant.journal_id) + && object.get("selected_ranges") + == serde_json::to_value(&grant.selected_ranges).ok().as_ref() +} + +/// Retire only the explicitly recognized descriptor wire. Malformed body snapshots are +/// failures, not migration requests and never a reason to reselect from current Sessions. +pub(super) fn migrate_reserved_legacy( + api: &WorkspaceApi, + reservation: &BackendJobReservation, +) -> Result { + if !legacy_descriptor(&reservation.job.request) { + return Ok(false); + } + reservation.job.load_executable_request()?; + close(api, &reservation.job.request, &reservation.attempt)?; + api.store.finish_backend_job_attempt( + api.workspace_id(), + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + BackendJobAttemptState::Failed, + "journal_descriptor_invalidated", + "Descriptor-only Journal input retired; saved checkpoints retained", + &Utc::now().to_rfc3339(), + )?; + api.store.reclaim_backend_job_input( + api.workspace_id(), + &reservation.job.request.job_id, + &reservation.job.input_digest, + crate::backend_job::JobInputRetirementReason::LegacyInvalidation, + )?; + let queued = api.clone(); + let request = reservation.job.request.clone(); + api.host_executors.host()?.dispatch(move || { + drain_after_completion(&queued, &request); + })?; + Ok(true) +} + +fn saved_input(request: &BackendJobRequest, grant: &SubjektivJournalGrant) -> Result { + let input = JournalInput::decode(&request.input).map_err(domain)?; + if input.subject_id != grant.subject_id + || input.journal_id != grant.journal_id + || input.capture.session_id != grant.session_id + || input.selected_ranges != grant.selected_ranges + { + return Err(Error::WorkspacePermissionDenied( + "Journal saved input/capability mismatch".into(), + )); + } + Ok(input) +} + fn live( api: &WorkspaceApi, worker: &RuntimeWorkerRef, @@ -125,6 +187,7 @@ fn live( .store .get_backend_job(api.workspace_id(), &binding.job_id)? .ok_or_else(|| Error::WorkspacePermissionDenied("Journal Job missing".into()))?; + job.load_executable_request()?; let attempt = api .store .get_backend_job_attempt(api.workspace_id(), &binding.job_id, &binding.attempt_id)? @@ -180,15 +243,44 @@ pub(super) fn execute( worker: &RuntimeWorkerRef, request: JournalJobRequest, ) -> Result { + let (grant, _, _) = live(api, worker)?; + let gate = lifecycle_lock(api, &grant.subject_id)?; + let _guard = gate + .lock() + .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?; + // Dispatch holds this same gate until matching Runtime acceptance is attested. + // Revalidate after waiting: erasure, cancellation or replacement may have won. let (grant, job, attempt) = live(api, worker)?; let store = store(api)?; let host = host(api, &store, &grant)?; let g = generation(&store, &host, &grant, &job, &attempt)?; + let input = saved_input(&job.request, &grant)?; + input.validate_generation(&g).map_err(domain)?; let ctx = JournalJobContext::validated(&store, &host, &g.id, g.job.clone()).map_err(domain)?; - journal::service::execute_bounded(&store, &ctx, request, |source, start, limit| { - read_source(api, &grant, source, start, limit) - .map_err(|e| JournalError::Conflict(e.to_string())) - }) + journal::service::execute_snapshot(&store, &ctx, &attempt.input_digest, request).map_err(domain) +} + +/// Called only after matching tracked Runtime acceptance of the exact rendered saved input. +/// The durable request and Worker binding alone do not establish body delivery. +pub(super) fn attest_delivery( + api: &WorkspaceApi, + reservation: &BackendJobReservation, +) -> Result<()> { + let Some(grant) = &reservation.job.request.grants.subjektiv_journal else { + return Ok(()); + }; + let input = saved_input(&reservation.job.request, grant)?; + let store = store(api)?; + let host = host(api, &store, grant)?; + let g = generation(&store, &host, grant, &reservation.job, &reservation.attempt)?; + input.validate_generation(&g).map_err(domain)?; + let ctx = JournalJobContext::validated(&store, &host, &g.id, g.job.clone()).map_err(domain)?; + journal::service::attest_input_delivery( + &store, + &ctx, + &reservation.attempt.input_digest, + &input.selected_ranges, + ) .map_err(domain) } @@ -196,21 +288,29 @@ pub(super) fn finish( api: &WorkspaceApi, worker: &RuntimeWorkerRef, ) -> Result { + let (grant, _, _) = live(api, worker)?; + let gate = lifecycle_lock(api, &grant.subject_id)?; + let _guard = gate + .lock() + .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?; + // Dispatch holds this same gate until matching Runtime acceptance is attested. + // Revalidate after waiting: erasure, cancellation or replacement may have won. let (grant, job, attempt) = live(api, worker)?; let store = store(api)?; let host = host(api, &store, &grant)?; + let input = saved_input(&job.request, &grant)?; // prepare is an exact binding lookup too, including the saved completion of a closed generation. let g = store - .prepare_journal_generation( + .prepare_journal_input( &host, - &grant.journal_id, + &input, subjektiv::api::JobAttemptBinding { job_id: job.request.job_id, attempt_id: attempt.attempt_id, }, - grant.selected_ranges.clone(), ) .map_err(domain)?; + input.validate_generation(&g).map_err(domain)?; let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone()) .map_err(domain)?; journal::service::finish(&store, &ctx).map_err(domain) @@ -277,21 +377,6 @@ pub(super) fn close( Ok(()) } -fn read_source( - api: &WorkspaceApi, - grant: &SubjektivJournalGrant, - source: &JournalSource, - start: u64, - limit: usize, -) -> Result { - let store = store(api)?; - let attr = store - .session_attribution(&grant.session_id) - .map_err(|e| Error::Store(e.to_string()))? - .ok_or_else(|| Error::WorkspacePermissionDenied("Session attribution absent".into()))?; - read_canonical(&material_reader(api, &attr)?, source, start, limit) -} - pub(super) fn bind_host_session( api: &WorkspaceApi, attr: crate::subjektiv::SubjectSessionAttribution, @@ -524,22 +609,63 @@ fn dispatch( .close_journal_generation(&host, &g.id, &g.job) .map_err(domain)?; } - let reader = material_reader(api, &attribution)?; - let materials = select_materials_bounded( - &store, - &host, - journal_id, - 96 * 1024, - |source, start, limit| { - read_canonical(&reader, source, start, limit) - .map_err(|e| JournalError::Conflict(e.to_string())) - }, - ) + let mut after = None; + loop { + let jobs = api.store.list_backend_journal_jobs( + api.workspace_id(), + &attribution.session_id, + after.as_deref(), + 100, + )?; + for job in &jobs { + if job + .request + .grants + .subjektiv_journal + .as_ref() + .is_none_or(|g| g.journal_id != journal_id) + || job.input_disposition != crate::backend_job::JobInputDisposition::Available + { + continue; + } + if job.state == BackendJobState::Pending || job.state == BackendJobState::Unknown { + return Ok(()); + } + if job.state == BackendJobState::Failed && !legacy_descriptor(&job.request) { + job.load_executable_request()?; + saved_input( + &job.request, + job.request + .grants + .subjektiv_journal + .as_ref() + .expect("checked Journal grant"), + )?; + let reservation = api.store.reserve_backend_job_retry( + api.workspace_id(), + &job.request.job_id, + &Utc::now().to_rfc3339(), + )?; + return queue_reservation(api, reservation); + } + } + if jobs.len() < 100 { + break; + } + after = jobs.last().map(|j| j.request.job_id.clone()); + } + let input = JournalInput::build(&store, &host, journal_id, |order, offset, budget| { + capture_input(api, &attribution, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string())) + }) .map_err(domain)?; - if materials.is_empty() { + if input.selected_ranges.is_empty() { return Ok(()); } - let request = recording_request(&host, attribution, journal_id, materials)?; + store + .accept_journal_sources(&host, "Session activity", &input.selected_ranges) + .map_err(domain)?; + let request = recording_request(&host, attribution, journal_id, input)?; // The intent is saved before generation binding or dispatch. If binding/dispatch fails, // Reserved recovery repeats this exact intent, never creates a replacement blindly. api.resolve_backend_job_profile(&request)?; @@ -548,6 +674,10 @@ fn dispatch( &request, &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), )?; + queue_reservation(api, reservation) +} + +fn queue_reservation(api: &WorkspaceApi, reservation: BackendJobReservation) -> Result<()> { let queued = api.clone(); if let Err(error)=api.host_executors.host()?.dispatch(move || { if let Err(error)=queued.dispatch_backend_job_reservation(reservation) { @@ -578,33 +708,21 @@ fn admit_sources( )); } let mut id = store.journal_for_session(&host).map_err(domain)?; - let reader = material_reader(api, &attribution)?; - if reader.not_materialized { - if store.journal_retention_required(&host).map_err(domain)? { - return Err(Error::RepositoryConflict( - "Admitted Journal original Session is not materialized".into(), - )); - } - retain_source( - api.runtime.as_ref(), - api.workspace_id(), - &attribution, - material_source(api, &attribution)?, - true, - )?; - return Ok(JournalRecordingResponse::NoNewSources); - } + let (order, offset) = if let Some(id) = id.as_deref() { + let journal = store.read_journal(&host, id).map_err(domain)?; + (journal.process_order, journal.process_byte) + } else { + (0, 0) + }; + let capture = capture_input(api, &attribution, order, offset, 48 * 1024)?; retain_source( api.runtime.as_ref(), api.workspace_id(), &attribution, material_source(api, &attribution)?, - false, + capture.segment_id == "not-materialized", )?; - let sources: Vec<_> = reader - .descriptors() - .map(|d| source_from_descriptor(&attribution.session_id, d)) - .collect(); + let sources = journal::recording::capture_ranges(&capture).map_err(domain)?; for batch in sources.chunks(MAX_JOURNAL_RANGES) { id = Some( store @@ -995,7 +1113,7 @@ pub(super) fn release_subject( { // Legacy saved-before-admission Sessions are not optional. Prove canonical // source identity first, then bind only this original Host-attributed Session. - let _canonical = material_reader(api, &attr)?; + let _canonical = capture_input(api, &attr, 0, 0, 48 * 1024)?; let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?; if host @@ -1281,80 +1399,6 @@ fn material_source( }, ) } -fn material_reader<'a>( - api: &'a WorkspaceApi, - attribution: &crate::subjektiv::SubjectSessionAttribution, -) -> Result> { - MaterialReader::open( - api.runtime.as_ref(), - api.workspace_id(), - &attribution.runtime_id, - material_source(api, attribution)?, - &attribution.session_id, - ) -} -fn source_from_descriptor( - session: &str, - d: &session_store::public_material::PublicMaterialDescriptor, -) -> JournalSourceRange { - JournalSourceRange { - source: JournalSource { - session_id: session.into(), - pointer: JournalSourcePointer { - segment_id: d.identity.segment_id, - entry_ref: d.identity.entry_ref.clone(), - }, - order: d.canonical_order, - content_sha256: d.sha256.clone(), - total_bytes: d.total_bytes, - }, - bytes: JournalByteRange { - start: 0, - end: d.total_bytes, - }, - } -} -fn read_canonical( - reader: &MaterialReader<'_>, - source: &JournalSource, - start: u64, - limit: usize, -) -> Result { - if reader.session_id != source.session_id { - return Err(Error::WorkspacePermissionDenied( - "Journal source Session mismatch".into(), - )); - } - let descriptor = reader - .descriptors() - .find(|d| source_from_descriptor(&source.session_id, d).source == *source) - .ok_or_else(|| { - Error::Store("Journal original public source changed or unavailable".into()) - })?; - // The reader streams the complete canonical digest while returning only this bounded window. - let page = reader - .read_content(descriptor, start, limit) - .map_err(|e| Error::Store(e.to_string()))?; - if page.sha256 != source.content_sha256 - || page.total_bytes != source.total_bytes - || page.next_offset != start.saturating_add(page.content.len() as u64) - { - return Err(Error::RepositoryConflict( - "Journal canonical source identity changed".into(), - )); - } - Ok(JournalMaterial { - range: JournalSourceRange { - source: source.clone(), - bytes: JournalByteRange { - start, - end: page.next_offset, - }, - }, - text: page.content, - }) -} - /// Close the first-run save/notification gap: Runtime can commit an initial turn before /// the create handler has finalized the Backend registry required by recording callbacks. /// Reconcile only after creation succeeds, using the ordinary current-owner/Feature checks. @@ -1475,7 +1519,7 @@ pub(super) fn recording_request( host: &JournalHostContext, attribution: crate::subjektiv::SubjectSessionAttribution, journal_id: &str, - materials: Vec, + input: JournalInput, ) -> Result { let binding_id = host .writer_binding_id() @@ -1488,18 +1532,88 @@ pub(super) fn recording_request( subject_id: attribution.subject_id, session_id: attribution.session_id, journal_id: journal_id.into(), - selected_ranges: materials.iter().map(|m| m.range.clone()).collect(), + selected_ranges: input.selected_ranges.clone(), }; - Ok(BackendJobRequest { - job_id:format!("journal-recording:{}",Uuid::new_v4()),purpose:"subjektiv_journal".into(), - input_ref:format!("subjektiv://{}/journal/{journal_id}",grant.subject_id), - input:serde_json::json!({"journal_id":journal_id,"selected_ranges":grant.selected_ranges}), - instruction:"Record ONLY this frozen original public selected_ranges prefix. The input contains descriptors, NOT source bodies. First ReadJournalState and use bounded ReadJournalText Source until every byte of every selected range is returned; record operations are refused until then. ReadJournalText also exposes initial/current position/open text. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, update current position and CheckpointJournal using retained record receipts. Never infer event completion from turn/task/job/compaction boundaries. Preserve exact operation ID/full input after unknown outcomes. SubmitBackendJobResult only after the full selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(), - profile:"builtin:subjektiv-journal-recording".into(), - grants:BackendJobGrants {subjektiv_journal:Some(grant),..Default::default()}, - serialization_key:Some(journal_serialization_key(journal_id)),source_worker:None,notification_target:None, - limits:crate::backend_job::BackendJobLimits {timeout_seconds:600,..Default::default()}, - }) + let request = BackendJobRequest { + job_id: format!("journal-recording:{}",Uuid::new_v4()), purpose: "subjektiv_journal".into(), + input_ref: format!("subjektiv://{}/journal/{journal_id}",grant.subject_id), + input: serde_json::to_value(input).map_err(|e| Error::Store(e.to_string()))?, + instruction: "Record ONLY the immutable body-bearing capture supplied in this input. These exact canonical source bytes, identities and ranges are the whole input for this Job; do not fetch original Sessions or use ReadJournalText Source. First ReadJournalState; initial/current Journal position/open text remain separately paged by ReadJournalText. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, current position and receipt-backed CheckpointJournal. Input delivery is not understanding or reflection. Never infer event completion from turn/task/job/compaction boundaries. Preserve exact operation ID/full input after Unknown. SubmitBackendJobResult only after the complete selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(), + profile: "builtin:subjektiv-journal-recording".into(), + grants: BackendJobGrants { subjektiv_journal: Some(grant), ..Default::default() }, + serialization_key: Some(journal_serialization_key(journal_id)), source_worker: None, notification_target: None, + limits: crate::backend_job::BackendJobLimits { timeout_seconds: 600, ..Default::default() }, + }; + saved_input( + &request, + request + .grants + .subjektiv_journal + .as_ref() + .expect("constructed grant"), + )?; + request.validate()?; + Ok(request) +} + +pub(super) fn capture_input( + api: &WorkspaceApi, + attr: &crate::subjektiv::SubjectSessionAttribution, + order: u64, + offset: u64, + budget: usize, +) -> Result { + match api + .runtime + .session_recording_capture( + &attr.runtime_id, + runtime_api::SessionRecordingCaptureRequest { + workspace_id: api.workspace_id().into(), + source: material_source(api, attr)?, + expected_session_id: attr.session_id.clone(), + start: session_store::capture::RecordingPosition { + canonical_order: order, + byte_offset: offset, + }, + max_entries: MAX_JOURNAL_RANGES, + max_serialized_bytes: budget, + }, + ) + .map_err(RuntimeRegistryError::into_error)? + { + runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => { + if slice.capture.session_id != attr.session_id + || serde_json::to_vec(&slice) + .map_err(|e| Error::Store(e.to_string()))? + .len() + > budget + { + return Err(Error::RepositoryConflict( + "recording capture identity/budget mismatch".into(), + )); + } + Ok(slice.capture) + } + runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id } + if session_id == attr.session_id && order == 0 && offset == 0 => + { + Ok(session_store::capture::CaptureDto { + version: 1, + session_id, + segment_id: "not-materialized".into(), + view_entry_count: 0, + entries: vec![], + }) + } + runtime_api::SessionRecordingCaptureAvailability::Unavailable { reason, .. } => { + Err(Error::Store(format!( + "committed Journal capture unavailable ({reason:?}); no latest-source fallback" + ))) + } + _ => Err(Error::Store( + "committed Journal capture identity mismatch; no latest-source fallback".into(), + )), + } } fn removal_attribution( @@ -1575,6 +1689,16 @@ fn finish_erased_release( "explicit erase tombstone missing".into(), )); } + for session in store + .subject_session_attributions(&attr.subject_id) + .map_err(|e| Error::Store(e.to_string()))? + .into_iter() + .filter(|session| { + session.runtime_id == attr.runtime_id && session.worker_id == attr.worker_id + }) + { + require_erased_job_copies(api, &session.session_id)?; + } // Pre-binding bodies have no write token. This stable receipt label is not a writer // grant: the domain erase operation records an inactive lifetime without binding Sessions. let erased_body_id = format!("erased-body:{}", attr.worker_id); @@ -1835,183 +1959,6 @@ pub(super) async fn finish_route( } /// A bounded Runtime transport view, not a second history index or a local-path fallback. -/// Reopening it observes a new catalog; each generation still compares its frozen source identity. -struct MaterialReader<'a> { - runtime: &'a RuntimeRegistry, - workspace_id: &'a str, - runtime_id: String, - source: runtime_api::SessionPublicSource, - session_id: String, - not_materialized: bool, - descriptors: Vec, -} -fn material_descriptor( - d: runtime_api::SessionMaterialDescriptor, -) -> Result { - Ok(session_store::public_material::PublicMaterialDescriptor { - identity: session_store::public_material::PublicMaterialIdentity { - segment_id: Uuid::parse_str(&d.segment_id).map_err(|e| Error::Store(e.to_string()))?, - entry_ref: d.entry_ref, - }, - canonical_order: d.canonical_order, - origin: serde_json::from_str(&d.origin_json).map_err(|e| Error::Store(e.to_string()))?, - sha256: d.sha256, - total_bytes: d.total_bytes, - }) -} -impl<'a> MaterialReader<'a> { - fn open( - runtime: &'a RuntimeRegistry, - workspace_id: &'a str, - runtime_id: &str, - source: runtime_api::SessionPublicSource, - session_id: &str, - ) -> Result { - let mut descriptors = Vec::new(); - let mut offset = 0; - let mut condition = None; - loop { - let page = match runtime - .session_material_catalog( - runtime_id, - runtime_api::SessionMaterialCatalogRequest { - workspace_id: workspace_id.into(), - source: source.clone(), - expected_session_id: session_id.into(), - expected_catalog_sha256: condition.clone(), - offset, - limit: 100, - }, - ) - .map_err(|e| e.into_error())? - { - runtime_api::SessionMaterialCatalogAvailability::NotMaterialized { - session_id: observed, - } => { - if observed != session_id || offset != 0 || condition.is_some() { - return Err(Error::RepositoryConflict( - "Journal Session materialization proof changed during catalog read" - .into(), - )); - } - return Ok(Self { - runtime, - workspace_id, - runtime_id: runtime_id.into(), - source, - session_id: session_id.into(), - not_materialized: true, - descriptors, - }); - } - runtime_api::SessionMaterialCatalogAvailability::Page { page } => page, - runtime_api::SessionMaterialCatalogAvailability::Unavailable { - reason, - message, - } => { - return Err(Error::Store(format!( - "Journal original material unavailable ({reason:?}): {message}" - ))); - } - }; - if page.session_id != session_id - || condition - .as_ref() - .is_some_and(|c| c != &page.catalog_sha256) - || page.descriptors.len() > 100 - || descriptors.len().saturating_add(page.descriptors.len()) > 1_000_000 - { - return Err(Error::RepositoryConflict( - "Journal canonical Session catalog changed or exceeded its bound".into(), - )); - } - condition = Some(page.catalog_sha256); - for d in page.descriptors { - let d = material_descriptor(d)?; - if d.canonical_order != descriptors.len() as u64 { - return Err(Error::RepositoryConflict( - "Journal canonical catalog order is discontinuous".into(), - )); - } - descriptors.push(d); - } - match page.next_offset { - Some(next) if next > offset && next == descriptors.len() as u64 => offset = next, - None => break, - _ => { - return Err(Error::RepositoryConflict( - "Journal catalog continuation did not advance".into(), - )); - } - } - } - Ok(Self { - runtime, - workspace_id, - runtime_id: runtime_id.into(), - source, - session_id: session_id.into(), - not_materialized: false, - descriptors, - }) - } - fn descriptors( - &self, - ) -> impl Iterator { - self.descriptors.iter() - } - fn read_content( - &self, - descriptor: &session_store::public_material::PublicMaterialDescriptor, - offset: u64, - limit: usize, - ) -> Result { - let page = match self - .runtime - .session_material_read( - &self.runtime_id, - runtime_api::SessionMaterialReadRequest { - workspace_id: self.workspace_id.into(), - source: self.source.clone(), - expected_session_id: self.session_id.clone(), - expected_descriptor: runtime_api::SessionMaterialDescriptor { - segment_id: descriptor.identity.segment_id.to_string(), - entry_ref: descriptor.identity.entry_ref.clone(), - canonical_order: descriptor.canonical_order, - origin_json: serde_json::to_string(&descriptor.origin) - .map_err(|e| Error::Store(e.to_string()))?, - sha256: descriptor.sha256.clone(), - total_bytes: descriptor.total_bytes, - }, - byte_offset: offset, - max_content_bytes: limit.clamp(4, 16 * 1024), - }, - ) - .map_err(|e| e.into_error())? - { - runtime_api::SessionMaterialReadAvailability::Page { page } => page, - runtime_api::SessionMaterialReadAvailability::Unavailable { reason, message } => { - return Err(Error::Store(format!( - "Journal original material read unavailable ({reason:?}): {message}" - ))); - } - }; - if material_descriptor(page.descriptor)? != *descriptor - || page.content.len() > limit.clamp(4, 16 * 1024) - { - return Err(Error::RepositoryConflict( - "Journal canonical material identity changed".into(), - )); - } - Ok(session_store::public_material::PublicMaterialContent { - content: page.content, - next_offset: page.next_offset, - total_bytes: descriptor.total_bytes, - sha256: descriptor.sha256.clone(), - }) - } -} - fn erase_source( runtime: &RuntimeRegistry, workspace: &str, @@ -2145,17 +2092,29 @@ pub(super) fn reconcile_unknown( let host = host(api, &store, grant)?; let erased = store.journal_session_erased(&host).map_err(domain)?; if !erased { - let g = store - .prepare_journal_generation( - &host, - &grant.journal_id, - subjektiv::api::JobAttemptBinding { - job_id: job.request.job_id.clone(), - attempt_id: attempt.attempt_id.clone(), - }, - grant.selected_ranges.clone(), - ) - .map_err(domain)?; + job.load_executable_request()?; + let binding = subjektiv::api::JobAttemptBinding { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + }; + let g = if legacy_descriptor(&job.request) { + // Only reconcile already-saved legacy receipts; never redispatch descriptors. + store + .prepare_journal_generation( + &host, + &grant.journal_id, + binding, + grant.selected_ranges.clone(), + ) + .map_err(domain)? + } else { + let input = saved_input(&job.request, grant)?; + let generation = store + .prepare_journal_input(&host, &input, binding) + .map_err(domain)?; + input.validate_generation(&generation).map_err(domain)?; + generation + }; let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone()) .map_err(domain)?; match journal::service::finish(&store, &ctx) { @@ -2187,6 +2146,14 @@ pub(super) fn reconcile_unknown( ); if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) { drain_after_completion(api, &job.request); + if job.input_disposition == crate::backend_job::JobInputDisposition::Available { + api.store.reclaim_backend_job_input( + api.workspace_id(), + &job.request.job_id, + &job.input_digest, + crate::backend_job::JobInputRetirementReason::Completed, + )?; + } } else { api.schedule_backend_job_worker_cleanup(¤t); } @@ -2226,8 +2193,8 @@ pub(super) fn reconcile_unknown( &Utc::now().to_rfc3339(), )?; if !erased { - // The old immutable input may include reflected bytes. Select the remaining - // prefix afresh only AFTER definitive lifecycle/domain fences; never replace Unknown. + // Resume the same saved body input only after canonical execution/domain fences. + // dispatch selects its retained intent before it can capture any newer source. let attr = store .session_attribution(&grant.session_id) .map_err(|e| Error::Store(e.to_string()))? @@ -2256,6 +2223,30 @@ fn reconcile_session_jobs( .store .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? .ok_or_else(|| Error::Store("Journal current attempt missing".into()))?; + if !erased + && legacy_descriptor(&job.request) + && job.input_disposition == crate::backend_job::JobInputDisposition::Available + { + job.load_executable_request()?; + close(api, &job.request, &attempt)?; + if !attempt.state.terminal() { + attempt = api.store.finish_backend_job_attempt( + api.workspace_id(), + &job.request.job_id, + &id, + BackendJobAttemptState::Failed, + "journal_descriptor_invalidated", + "Descriptor-only Journal input retired; saved checkpoints retained", + &Utc::now().to_rfc3339(), + )?; + } + api.store.reclaim_backend_job_input( + api.workspace_id(), + &job.request.job_id, + &job.input_digest, + crate::backend_job::JobInputRetirementReason::LegacyInvalidation, + )?; + } if erased && !attempt.state.terminal() { // The domain tombstone precedes this cancellation and also fences any // crash window before it. No erased Job can prepare or use tools again. @@ -2269,6 +2260,30 @@ fn reconcile_session_jobs( &Utc::now().to_rfc3339(), )?; } + if erased { + api.store.reclaim_backend_job_input( + api.workspace_id(), + &job.request.job_id, + &job.input_digest, + crate::backend_job::JobInputRetirementReason::ExplicitErasure, + )?; + // Every historical execution can own a delivered copy, not just the current + // attempt or an unacknowledged result. Cleanup is canonical Purge and retryable. + for number in 1..=job.current_attempt { + let historical_id = crate::backend_job::attempt_id(&job.request.job_id, number); + let historical = api + .store + .get_backend_job_attempt( + api.workspace_id(), + &job.request.job_id, + &historical_id, + )? + .ok_or_else(|| Error::Store("historical Journal attempt missing".into()))?; + if historical.state.terminal() { + api.schedule_backend_job_worker_cleanup(&historical); + } + } + } if attempt.state == BackendJobAttemptState::Unknown { reconcile_unknown(api, job, &attempt)?; } @@ -2284,6 +2299,41 @@ fn reconcile_session_jobs( Ok(()) } +fn require_erased_job_copies(api: &WorkspaceApi, session: &str) -> Result<()> { + let mut after = None; + loop { + let jobs = api.store.list_backend_journal_jobs( + api.workspace_id(), + session, + after.as_deref(), + 100, + )?; + for job in &jobs { + if job.input_disposition != crate::backend_job::JobInputDisposition::Erased { + return Err(Error::RepositoryConflict( + "Journal input erasure remains incomplete".into(), + )); + } + for number in 1..=job.current_attempt { + let id = crate::backend_job::attempt_id(&job.request.job_id, number); + let attempt = api + .store + .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? + .ok_or_else(|| Error::Store("erased Journal attempt missing".into()))?; + if attempt.worker.is_some() + && attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) + { + return Err(Error::RepositoryConflict("Journal delivered-copy Purge remains pending; retry the same erasure after cleanup".into())); + } + } + } + if jobs.len() < 100 { + return Ok(()); + } + after = jobs.last().map(|j| j.request.job_id.clone()); + } +} + impl WorkspaceApi { /// Host-only erasure saves the domain fence, cancels Jobs, then erases originals. /// Runtime requires a stopped source Worker. Physical failure leaves the domain @@ -2301,6 +2351,10 @@ impl WorkspaceApi { .ok_or_else(|| { Error::WorkspacePermissionDenied("Session attribution missing".into()) })?; + let gate = lifecycle_lock(self, &attr.subject_id)?; + let _guard = gate + .lock() + .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?; let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?; store .erase_journal_session(&host, request_id) @@ -2311,7 +2365,8 @@ impl WorkspaceApi { self.workspace_id(), &attr, material_source(self, &attr)?, - ) + )?; + require_erased_job_copies(self, &attr.session_id) } } diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs index 4cc4b5c24..72ff4d8cc 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs @@ -14,10 +14,33 @@ use std::sync::{Arc, Mutex}; #[derive(Clone, Default)] struct ConnectedJournalModel { dialogs: Arc>>, + journal_inputs: Arc>>, } impl ConnectedJournalModel { fn journal_response(request: &Request) -> Vec { + // Consume the actual Controller-rendered input, not a fixture generation or + // a Source tool response. Decode also checks complete bodies/ranges/digests. + let envelope = request + .items + .iter() + .filter_map(Item::as_text) + .find(|text| text.lines().any(|line| line.starts_with("input_json: "))) + .expect("rendered immutable Backend Job envelope"); + let input_json = envelope + .lines() + .find_map(|line| line.strip_prefix("input_json: ")) + .unwrap(); + let input_value: Value = serde_json::from_str(input_json).unwrap(); + let frozen = subjektiv::journal::recording::JournalInput::decode(&input_value).unwrap(); + assert_eq!( + serde_json::to_string(&input_value).unwrap(), + input_json, + "render preserves the exact persisted JSON bytes" + ); + assert!(frozen.capture.entries.iter().all(|entry| matches!( + &entry.body, session_store::capture::CaptureBody::CanonicalPublic { text, .. } if !text.is_empty()))); + let ranges = serde_json::to_value(&frozen.selected_ranges).unwrap(); // Runtime Controllers deliberately retain alias-routed core schemas for their // lifetime even with zero attachments. Verify invocation authority, not schema // absence (standalone Internal Jobs use a different composition). @@ -89,7 +112,7 @@ impl ConnectedJournalModel { "request_id":format!("checkpoint:{}",state["value"]["generation_id"].as_str().unwrap()), "expected_state":position["value"]["state_token"], "action":{"action":"checkpoint","record_operation_ids":[format!("record:{}",state["value"]["generation_id"].as_str().unwrap())]}, - "covered_ranges":state["value"]["selected_ranges"], + "covered_ranges":ranges, }), ) } else if let Some(record) = result("record") { @@ -106,41 +129,14 @@ impl ConnectedJournalModel { ) } else if let Some(state) = result("state") { let state = &state["value"]; - // Deliver the actual frozen Runtime source through authenticated Job tools, rather - // than injecting a fixture Journal or claiming unseen source was processed. - for (index, range) in state["selected_ranges"] - .as_array() - .unwrap() - .iter() - .enumerate() - { - let mut offset = range["bytes"]["start"].as_u64().unwrap(); - let mut token = Value::Null; - loop { - let id = format!("source-{index}-{offset}"); - if let Some(page) = result(&id) { - let page = &page["value"]; - if let Some(next) = page["next_byte"].as_u64() { - assert!(next > offset); - offset = next; - token = page["content_token"].clone(); - } else { - break; - } - } else { - assert!( - request - .tools - .iter() - .any(|tool| tool.name == "ReadJournalText") - ); - return vec![Event::tool_use_start(0, &id, "ReadJournalText"), Event::tool_input_delta(0, json!({ - "target":{"target":"source","pointer":range["source"]["pointer"]}, - "offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token, - }).to_string()), Event::tool_use_stop(0)]; - } - } - } + assert_eq!( + state["selected_ranges"], ranges, + "mutation coverage must come from the rendered frozen input" + ); + assert_eq!( + state["initial_journal"]["state_token"], + input_value["initial_state_token"] + ); let (tool, action) = if state["current_open"].is_null() { ( "CreateJournalEntry", @@ -158,7 +154,7 @@ impl ConnectedJournalModel { json!({ "request_id":format!("record:{}",state["generation_id"].as_str().unwrap()), "expected_state":state["current_journal"]["state_token"], - "action":action,"covered_ranges":state["selected_ranges"], + "action":action,"covered_ranges":ranges, }), ) } else { @@ -209,6 +205,18 @@ impl LlmClient for ConnectedJournalModel { .iter() .any(|tool| tool.name == "ReadJournalState") { + let envelope = request + .items + .iter() + .filter_map(Item::as_text) + .find(|text| text.lines().any(|line| line.starts_with("input_json: "))) + .expect("actual Journal input in model render") + .to_owned(); + let mut inputs = self.journal_inputs.lock().unwrap(); + if !inputs.contains(&envelope) { + inputs.push(envelope); + } + drop(inputs); Self::journal_response(&request) } else if request.tools.iter().any(|tool| { matches!( @@ -458,6 +466,29 @@ async fn backend_real_runtime_journal_release_and_next_body_persisted_startup() .store .list_backend_journal_jobs(TEST_WORKSPACE_ID, &old_attr.session_id, None, 100) .unwrap(); + // Compare actual render with the durable identity, even if completion already + // reclaimed the input body. Reconstituting only for comparison is test-local. + for saved in &recorded { + let inputs = model.journal_inputs.lock().unwrap(); + let prefix = format!("job_id: {}", saved.request.job_id); + let rendered = inputs + .iter() + .find(|text| text.lines().any(|line| line == prefix)) + .expect("every saved Journal Job reached the real Controller"); + let actual = rendered + .lines() + .find_map(|line| line.strip_prefix("input_json: ")) + .unwrap(); + let mut original = saved.request.clone(); + original.input = serde_json::from_str(actual).unwrap(); + assert_eq!(original.input_digest().unwrap(), saved.input_digest); + let attempt = crate::backend_job::attempt_id(&saved.request.job_id, saved.current_attempt); + assert_eq!(*rendered, original.worker_input(&attempt).unwrap()); + let input = subjektiv::journal::recording::JournalInput::decode(&original.input).unwrap(); + assert_eq!(input.capture.session_id, old_attr.session_id); + assert!(actual.contains("User chose B rather than A")); + assert!(actual.contains("Partial B implementation")); + } let uncertain_job = &recorded[0].request.job_id; let conn = rusqlite::Connection::open(&api.config.database_path).unwrap(); let saved_result: (String, String) = conn.query_row( diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs index 81ff5817a..364438db2 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs @@ -188,6 +188,55 @@ struct Fixture { source: RuntimeWorkerRef, id: String, } +fn fixture_capture(range: &JournalSourceRange, text: &str) -> session_store::capture::CaptureDto { + use session_store::capture::*; + let descriptor = session_store::public_material::PublicMaterialDescriptor { + identity: session_store::public_material::PublicMaterialIdentity { + segment_id: range.source.pointer.segment_id, + entry_ref: range.source.pointer.entry_ref.clone(), + }, + canonical_order: range.source.order, + origin: session_store::public_material::PublicMaterialOrigin::History { + origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + }, + sha256: range.source.content_sha256.clone(), + total_bytes: range.source.total_bytes, + }; + CaptureDto { + version: 1, + session_id: range.source.session_id.clone(), + segment_id: range.source.pointer.segment_id.to_string(), + view_entry_count: range.source.order + 1, + entries: vec![CaptureEntryDto { + entry_id: range + .source + .pointer + .entry_ref + .trim_start_matches('E') + .into(), + order: range.source.order, + source: CaptureSource { + segment_id: range.source.pointer.segment_id.to_string(), + entry_index: range.source.order, + provenance: protocol::SessionEntryProvenance::HumanInput, + history_origin: Some(session_store::LoggedSessionHistoryOrigin::LocalHumanInput), + derived_from: vec![], + }, + policy: CaptureUsePolicy { + recording_material: true, + memory_evidence: false, + observation_readable: false, + }, + body: CaptureBody::CanonicalPublic { + descriptor, + byte_offset: range.bytes.start, + text: text.into(), + boundary: None, + }, + }], + } +} + impl Fixture { async fn new() -> Self { let dir = tempfile::tempdir().unwrap(); @@ -236,7 +285,16 @@ impl Fixture { job_id: "journal-job-test".into(), purpose: "subjektiv_journal".into(), input_ref: format!("journal://{id}"), - input: serde_json::json!({"selected_ranges":[range]}), + input: serde_json::to_value( + subjektiv::journal::recording::JournalInput::build( + &store, + &host, + &id, + |_, _, _| Ok(fixture_capture(&range, text)), + ) + .unwrap(), + ) + .unwrap(), instruction: "Record this public input with sequential tools".into(), profile: "builtin:subjektiv-journal-recording".into(), grants: BackendJobGrants { @@ -537,16 +595,23 @@ fn message(text: &str) -> session_store::LoggedItem { } impl RuntimeFixture { async fn new() -> Self { + Self::with_history(vec![session_store::LoggedHistoryEntry { + item: message(SOURCE_SENTINEL), + metadata: session_store::LoggedSessionHistoryMetadata { + input_metadata: None, + entry_id: session_store::LoggedSessionHistoryEntryId("read-source-original".into()), + origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + derivation: None, + }, + }]) + .await + } + async fn with_history(history: Vec) -> Self { let dir = tempfile::tempdir().unwrap(); let (api, execution) = test_api_with_recording_backend(dir.path()).await; let source = spawn_ticket_check_source(&api, "original-source"); let session = session_store::new_session_id(); - seed_session( - &session_root(dir.path(), &source), - session, - message(SOURCE_SENTINEL), - session_store::LoggedSessionHistoryOrigin::LocalHumanInput, - ); + seed_session_history(&session_root(dir.path(), &source), session, history); let store = open_subjektiv_store(&api).unwrap(); let subject = store .create_subject(crate::subjektiv::SubjectRole::new("source-reader-test").unwrap()) @@ -600,7 +665,7 @@ impl RuntimeFixture { else { panic!("canonical catalog required") }; - assert_eq!(page.descriptors.len(), 1); + assert!(!page.descriptors.is_empty()); let d = &page.descriptors[0]; let range = JournalSourceRange { source: JournalSource { @@ -621,20 +686,24 @@ impl RuntimeFixture { let id = store .accept_journal_sources(&host, "source", &[range.clone()]) .unwrap(); - let request = subjektiv_journal::recording_request( + let input = subjektiv::journal::recording::JournalInput::build( + &store, &host, - attr, &id, - vec![subjektiv::journal::recording::JournalMaterial { - range: range.clone(), - text: SOURCE_SENTINEL.into(), - }], + |order, offset, budget| { + subjektiv_journal::capture_input(&api, &attr, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string())) + }, ) .unwrap(); + store + .accept_journal_sources(&host, "bounded captured prefix", &input.selected_ranges) + .unwrap(); + let request = subjektiv_journal::recording_request(&host, attr, &id, input).unwrap(); assert!( - !serde_json::to_string(&request) - .unwrap() - .contains(SOURCE_SENTINEL) + request.input["capture"]["entries"] + .as_array() + .is_some_and(|entries| !entries.is_empty()) ); let reserved = api .store @@ -666,47 +735,53 @@ impl RuntimeFixture { ) .unwrap(); } - fn dispatch_and_read(&self) -> BackendJobReservation { + fn dispatch_input(&self) -> BackendJobReservation { let dispatched = self .api .dispatch_backend_job_reservation(self.reserved.clone()) .unwrap(); let worker = dispatched.attempt.worker.as_ref().unwrap(); - let response = subjektiv_journal::execute( - &self.api, - worker, - JournalJobRequest::ReadText { - read: JournalTextRead { - target: JournalTextTarget::Source { - pointer: self.range.source.pointer.clone(), - }, - offset: 0, - limit: 16384, - content_token: None, - }, - }, - ) - .unwrap(); - let encoded = serde_json::to_string(&response).unwrap(); + let encoded = self + .execution + .inputs + .lock() + .unwrap() + .last() + .unwrap() + .1 + .clone(); + assert_eq!( + encoded, + self.request + .worker_input(&dispatched.attempt.attempt_id) + .unwrap() + ); assert!(encoded.contains(SOURCE_SENTINEL)); + assert!( + subjektiv_journal::execute( + &self.api, + worker, + JournalJobRequest::ReadText { + read: JournalTextRead { + target: JournalTextTarget::Source { + pointer: self.range.source.pointer.clone() + }, + offset: 0, + limit: 16384, + content_token: None + } + } + ) + .is_err(), + "body-bearing Jobs cannot read their original Session" + ); + // Deterministic execution stands in for Controller persistence here; actual Runtime + // input storage is exercised separately by its signed capture/delivery integration. seed_session( &session_root(self.dir.path(), worker), session_store::new_session_id(), - session_store::LoggedItem::ToolResult { - call_id: "read-original".into(), - summary: "ReadJournalText".into(), - content: Some(encoded), - attachments: vec![], - disposition: agen::tool::ToolResultDisposition::Success, - is_error: false, - }, - session_store::LoggedSessionHistoryOrigin::ToolOutput { - worker: session_store::LoggedWorkerSubject { - workspace_id: Some(TEST_WORKSPACE_ID.into()), - runtime_id: Some(worker.runtime_id.clone()), - worker_id: worker.worker_id.clone(), - }, - }, + message(&encoded), + session_store::LoggedSessionHistoryOrigin::LocalHumanInput, ); dispatched } @@ -748,6 +823,60 @@ impl RuntimeFixture { ) .unwrap(); } + /// Pending is NOT erasure success: observe durable Purge and retry the exact key. + fn erase_pending_or_acknowledged(&self, key: &str) -> bool { + match self + .api + .erase_subjektiv_journal_session(&self.session.to_string(), key) + { + Ok(()) => false, + Err(Error::RepositoryConflict(message)) => { + assert_eq!( + message, + "Journal delivered-copy Purge remains pending; retry the same erasure after cleanup" + ); + true + } + Err(error) => panic!("unexpected erasure failure: {error}"), + } + } + fn assert_erasure_acknowledged(&self, key: &str) { + self.api + .erase_subjektiv_journal_session(&self.session.to_string(), key) + .unwrap(); + let saved = self + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &self.request.job_id) + .unwrap() + .unwrap(); + assert_eq!( + saved.input_disposition, + crate::backend_job::JobInputDisposition::Erased + ); + assert_eq!(saved.input_digest, self.request.input_digest().unwrap()); + assert!( + !serde_json::to_string(&saved.request) + .unwrap() + .contains(SOURCE_SENTINEL) + ); + assert!(saved.load_executable_request().is_err()); + self.assert_no_erased_resurrection(); + } + fn assert_no_erased_resurrection(&self) { + self.execution.take_inputs(); + // Internal dispatch must reread and reject retired input, including stale + // terminal reservations. Exact public API replay is a separate contract. + assert!( + self.api + .dispatch_backend_job_reservation(self.reserved.clone()) + .is_err() + ); + assert!( + self.execution.take_inputs().is_empty(), + "stale saved body must not resurrect a delivered copy" + ); + } fn assert_purged(&self, worker: &RuntimeWorkerRef) { assert!(!session_root(self.dir.path(), worker).exists()); let removal = self @@ -791,7 +920,7 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang .worker_retention_policy(TEST_WORKSPACE_ID) .unwrap() .unwrap(); - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); assert!(session_root(f.dir.path(), worker).exists()); match outcome { @@ -823,12 +952,7 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang } "erased" => { f.stop_source(); - f.api - .erase_subjektiv_journal_session( - &f.session.to_string(), - "erase-original-session", - ) - .unwrap(); + f.erase_pending_or_acknowledged("erase-original-session"); } _ => unreachable!(), } @@ -850,12 +974,9 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang ); assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); if outcome == "erased" { + f.assert_erasure_acknowledged("erase-original-session"); assert!(f.store.journal_session_erased(&f.host).unwrap()); - assert!( - f.api - .dispatch_backend_job_reservation(f.reserved.clone()) - .is_err() - ); + f.assert_no_erased_resurrection(); } } } @@ -863,7 +984,7 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang #[tokio::test] async fn unknown_saved_completion_reconciles_same_binding_without_redispatch_after_lost_result() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); f.checkpoint(worker); let saved = subjektiv_journal::finish(&f.api, worker).unwrap(); // response/result delivery is lost @@ -900,7 +1021,7 @@ async fn unknown_saved_completion_reconciles_same_binding_without_redispatch_aft #[tokio::test] async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_cleanup_fails() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); let g = f .store @@ -966,6 +1087,28 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ ) .is_err() ); + let retained = f + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() + .unwrap(); + assert_eq!(retained.input_digest, f.request.input_digest().unwrap()); + assert_eq!( + retained.load_executable_request().unwrap().input, + f.request.input + ); + assert!( + f.api + .store + .reserve_backend_job_retry( + TEST_WORKSPACE_ID, + &f.request.job_id, + &Utc::now().to_rfc3339() + ) + .is_err(), + "unknown cannot be reissued as a new attempt before the fence" + ); assert!(f.store.journal_retention_required(&f.host).unwrap()); assert!(session_root(f.dir.path(), worker).exists()); assert_eq!( @@ -1044,6 +1187,42 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ 1, "source pin survives failed/partial Job cleanup" ); + f.execution.take_inputs(); + let retry = f + .api + .store + .reserve_backend_job_retry( + TEST_WORKSPACE_ID, + &f.request.job_id, + &Utc::now().to_rfc3339(), + ) + .unwrap(); + assert_eq!(retry.job.request.input, f.request.input); + assert_eq!(retry.job.input_digest, f.request.input_digest().unwrap()); + let retry = f.api.dispatch_backend_job_reservation(retry).unwrap(); + let resumed = retry.attempt.worker.as_ref().unwrap(); + let delivered = f.execution.take_inputs(); + assert_eq!(delivered.len(), 1); + assert_eq!( + delivered[0].1, + f.request.worker_input(&retry.attempt.attempt_id).unwrap() + ); + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, "partial-record") + .unwrap(), + Some(recorded) + ); + assert_eq!( + f.store + .list_journal_entries(&f.host, &f.id, None, 16) + .unwrap() + .items + .len(), + 1 + ); + assert!(subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok()); + assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); } #[tokio::test] @@ -1126,7 +1305,7 @@ async fn journal_reserved_restart_dispatches_exact_intent_once_and_unbound_unkno #[tokio::test] async fn archived_original_session_remains_readable_then_releases_its_pin_after_saved_completion() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); let root = f.dir.path().join(".test-embedded-runtime-store"); let retention = worker_runtime::retention::runtime_session_retention(&root); @@ -1158,8 +1337,23 @@ async fn archived_original_session_remains_readable_then_releases_its_pin_after_ .unwrap() .expect("durable archive binding"); assert!(!session_root(f.dir.path(), &f.source).exists()); - // Same frozen descriptor; material_source must now resolve through the durable - // archive mapping, not attempt a stale retained Worker path. + // Archive movement neither changes nor reclaims the saved immutable body. + // The Job cannot fall back to a retained or archived Session read. + let saved = f + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() + .unwrap(); + assert_eq!( + saved.load_executable_request().unwrap().input, + f.request.input + ); + assert!( + serde_json::to_string(&saved.request.input) + .unwrap() + .contains(SOURCE_SENTINEL) + ); let reread = subjektiv_journal::execute( &f.api, worker, @@ -1174,11 +1368,13 @@ async fn archived_original_session_remains_readable_then_releases_its_pin_after_ }, }, ) - .unwrap(); - assert!( - serde_json::to_string(&reread) - .unwrap() - .contains(SOURCE_SENTINEL) + .unwrap_err(); + assert!(matches!(reread, Error::WorkspacePermissionDenied(message) + if message.contains("raw source bodies are in immutable Job input"))); + assert_eq!( + retention.pins(f.session).unwrap().len(), + 1, + "delivery alone never releases original pin" ); f.checkpoint(worker); let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); @@ -1251,7 +1447,7 @@ async fn journal_profile_name_without_host_job_binding_keeps_normal_archive_poli async fn saved_checkpoint_unknown_auto_resolves_without_waiting_for_failed_session_cleanup() { for recovery in ["transition", "restart"] { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); f.checkpoint(worker); // Deliberately do not call finish: checkpoint receipt is durable. f.execution.take_inputs(); @@ -1353,10 +1549,9 @@ async fn saved_checkpoint_unknown_auto_resolves_without_waiting_for_failed_sessi } #[tokio::test] -async fn stopped_partial_journal_recovery_fences_old_attempt_before_dispatching_only_remaining_work() - { +async fn stopped_partial_journal_recovery_fences_old_attempt_and_resumes_same_immutable_input() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); let g = f .store @@ -1420,11 +1615,14 @@ async fn stopped_partial_journal_recovery_fences_old_attempt_before_dispatching_ .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) .unwrap() .unwrap(); - assert_eq!(old.state, BackendJobState::Failed); + assert_eq!(old.state, BackendJobState::Pending); + assert_eq!(old.request.job_id, f.request.job_id); + assert_eq!(old.input_digest, f.request.input_digest().unwrap()); assert_eq!( - old.failure_category.as_deref(), - Some("journal_attempt_fenced") + old.load_executable_request().unwrap().input, + f.request.input ); + assert_eq!(old.current_attempt, dispatched.attempt.attempt + 1); f.assert_purged(worker); assert_eq!( f.store @@ -1446,11 +1644,20 @@ async fn stopped_partial_journal_recovery_fences_old_attempt_before_dispatching_ .active_journal_generation(&f.host, &f.id) .unwrap() .unwrap(); - assert_ne!(next.job, g.job); - assert_eq!(next.initial_open.unwrap().title, "unfinished"); + assert_eq!(next.job.job_id, g.job.job_id); + assert_ne!(next.job.attempt_id, g.job.attempt_id); + assert_eq!( + next.initial_open, g.initial_open, + "original initial snapshot stays bound to same input" + ); + assert_eq!(next.selected_ranges, g.selected_ranges); let new_inputs = f.execution.take_inputs(); assert_eq!(new_inputs.len(), 1); - assert!(!new_inputs[0].1.contains(SOURCE_SENTINEL)); + assert!(new_inputs[0].1.contains(SOURCE_SENTINEL)); + assert_eq!( + new_inputs[0].1, + f.request.worker_input(&next.job.attempt_id).unwrap() + ); assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); f.api.recover_backend_jobs().unwrap(); assert!( @@ -1463,7 +1670,7 @@ async fn stopped_partial_journal_recovery_fences_old_attempt_before_dispatching_ async fn failed_cancelled_and_erased_job_sessions_retry_purge_after_cleanup_failure() { for outcome in ["failed", "cancelled", "erased"] { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); f.api .store @@ -1476,12 +1683,10 @@ async fn failed_cancelled_and_erased_job_sessions_retry_purge_after_cleanup_fail .unwrap(); if outcome == "erased" { f.stop_source(); - f.api - .erase_subjektiv_journal_session( - &f.session.to_string(), - "erase-with-blocked-cleanup", - ) - .unwrap(); + assert!( + f.erase_pending_or_acknowledged("erase-with-blocked-cleanup"), + "pinned delivered copy must prevent erasure acknowledgement" + ); } else { f.api .finish_backend_job_attempt( @@ -1534,12 +1739,9 @@ async fn failed_cancelled_and_erased_job_sessions_retry_purge_after_cleanup_fail .await; f.assert_purged(worker); if outcome == "erased" { + f.assert_erasure_acknowledged("erase-with-blocked-cleanup"); assert!(f.store.journal_session_erased(&f.host).unwrap()); - assert!( - f.api - .dispatch_backend_job_reservation(f.reserved.clone()) - .is_err() - ); + f.assert_no_erased_resurrection(); } } } @@ -1547,7 +1749,7 @@ async fn failed_cancelled_and_erased_job_sessions_retry_purge_after_cleanup_fail #[tokio::test] async fn explicit_erasure_interrupts_completed_journal_cleanup_waiting_for_run_teardown() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); f.checkpoint(worker); let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); @@ -1585,9 +1787,7 @@ async fn explicit_erasure_interrupts_completed_journal_cleanup_waiting_for_run_t "normal cleanup still waits for tracked response teardown" ); f.stop_source(); - f.api - .erase_subjektiv_journal_session(&f.session.to_string(), "erase-while-completed-run-busy") - .unwrap(); + f.erase_pending_or_acknowledged("erase-while-completed-run-busy"); // No Idle/finish marker is published by this fixture. The erasure fence must // cause a real Runtime stop/Purge instead of leaving plaintext indefinitely. wait_for_backend_job_worker_cleanup( @@ -1598,6 +1798,7 @@ async fn explicit_erasure_interrupts_completed_journal_cleanup_waiting_for_run_t ) .await; f.assert_purged(worker); + f.assert_erasure_acknowledged("erase-while-completed-run-busy"); assert_eq!( f.api .store @@ -2189,7 +2390,7 @@ async fn erased_subject_removal_retries_interrupted_physical_erasure_under_the_p #[tokio::test] async fn physical_erase_failure_keeps_domain_tombstone_and_job_cancellation_for_exact_retry() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); // Original source is not stopped: Runtime must reject physical erasure, after // the Backend has durably fenced all Journal adoption and cancelled its Job. assert!( @@ -2215,9 +2416,16 @@ async fn physical_erase_failure_keeps_domain_tombstone_and_job_cancellation_for_ .is_err() ); f.stop_source(); - f.api - .erase_subjektiv_journal_session(&f.session.to_string(), "erase-unsettled-source") - .unwrap(); + f.erase_pending_or_acknowledged("erase-unsettled-source"); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + f.assert_erasure_acknowledged("erase-unsettled-source"); + f.assert_purged(dispatched.attempt.worker.as_ref().unwrap()); assert!(!session_root(f.dir.path(), &f.source).exists()); assert_eq!( f.api @@ -2807,3 +3015,1345 @@ async fn legacy_journal_grant_preserves_old_json_and_only_recovers_original_curr "legacy grant must never map to a later writer" ); } + +/// Real embedded Runtime capture + SQLite + production dispatch; synthetic ordinary +/// Session entries only, no fixed Worker data, fixture generator or model calls. +mod immutable { + use super::*; + use session_store::{LogEntry, Store as _, WorkerSessionStore}; + use std::collections::BTreeMap; + use subjektiv::journal::recording::JournalInput; + + fn history(id: &str, item: session_store::LoggedItem) -> session_store::LoggedHistoryEntry { + session_store::LoggedHistoryEntry { + item, + metadata: session_store::LoggedSessionHistoryMetadata { + input_metadata: None, + entry_id: session_store::LoggedSessionHistoryEntryId(id.into()), + origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + derivation: None, + }, + } + } + fn append(f: &RuntimeFixture, segment: Uuid, id: &str) { + WorkerSessionStore::new(session_root(f.dir.path(), &f.source)) + .unwrap() + .append( + f.session, + segment, + &LogEntry::AnnotatedAssistantItem { + ts: 2, + entry: history(id, message(id)), + }, + ) + .unwrap(); + } + fn checkpoint_ranges(f: &RuntimeFixture, worker: &RuntimeWorkerRef, input: &JournalInput) { + let state = f.store.read_journal(&f.host, &f.id).unwrap(); + let record_id = format!("immutable-record:{}", input.initial_state_token); + let operation = JournalOperation { + request_id: record_id.clone(), + expected_state: state.state_token, + action: JournalAction::ConfirmUnchanged, + covered_ranges: input.selected_ranges.clone(), + }; + let response = subjektiv_journal::execute( + &f.api, + worker, + JournalJobRequest::Execute { + operation: operation.clone(), + }, + ) + .unwrap(); + // Exact operation replay after a lost response cannot duplicate or advance the cursor. + assert_eq!( + subjektiv_journal::execute(&f.api, worker, JournalJobRequest::Execute { operation }) + .unwrap(), + response + ); + let JournalJobResponse::Saved(record) = response else { + panic!("record receipt") + }; + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, &record_id) + .unwrap(), + Some(record.clone()) + ); + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + state.process_order + ); + let JournalJobResponse::Saved(position) = subjektiv_journal::execute( + &f.api, + worker, + JournalJobRequest::Execute { + operation: JournalOperation { + request_id: format!("immutable-position:{}", input.initial_state_token), + expected_state: record.state_token, + action: JournalAction::UpdatePosition { + current_position: "Original prefix saved; remaining work unconfirmed" + .into(), + }, + covered_ranges: vec![], + }, + }, + ) + .unwrap() else { + panic!("position receipt") + }; + subjektiv_journal::execute( + &f.api, + worker, + JournalJobRequest::Execute { + operation: JournalOperation { + request_id: format!("immutable-checkpoint:{}", input.initial_state_token), + expected_state: position.state_token, + action: JournalAction::Checkpoint { + record_operation_ids: vec![record_id], + }, + covered_ranges: input.selected_ranges.clone(), + }, + }, + ) + .unwrap(); + } + + #[tokio::test] + async fn backend_immutable_input_survives_missing_source_and_predispatch_host_reopen() { + let RuntimeFixture { + dir, + api, + execution, + store, + host, + source, + session, + request, + reserved, + range, + id, + } = RuntimeFixture::new().await; + let frozen = serde_json::to_vec(&request.input).unwrap(); + let digest = request.input_digest().unwrap(); + subjektiv_journal::prepare(&api, &reserved).unwrap(); + let generation = store + .active_journal_generation(&host, &id) + .unwrap() + .unwrap(); + let root = session_root(dir.path(), &source); + let unavailable = dir.path().join("source-unavailable-during-job"); + std::fs::rename(&root, &unavailable).unwrap(); + let config = api.config.clone(); + drop(api); + drop(store); + // Reopen after input save but before dispatch, with no readable original source. + let api = WorkspaceApi::new_with_execution_backend_and_broker( + config.clone(), + Arc::new(test_control_store(&config)), + execution.clone(), + BackendResourceBroker::default(), + HostExecutors::default(), + None, + ) + .await + .unwrap(); + let store = open_subjektiv_store(&api).unwrap(); + let f = RuntimeFixture { + dir, + api, + execution, + store, + host, + source, + session, + request, + reserved, + range, + id, + }; + let saved = f + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() + .unwrap(); + assert_eq!( + serde_json::to_vec(&saved.load_executable_request().unwrap().input).unwrap(), + frozen + ); + assert_eq!(saved.input_digest, digest); + let dispatched = f + .api + .dispatch_backend_job_reservation(f.reserved.clone()) + .unwrap(); + assert_eq!( + f.store + .active_journal_generation(&f.host, &f.id) + .unwrap() + .unwrap(), + generation + ); + let worker = dispatched.attempt.worker.as_ref().unwrap(); + assert!(!root.exists()); + assert_eq!( + f.execution.take_inputs().last().unwrap().1, + f.request + .worker_input(&dispatched.attempt.attempt_id) + .unwrap() + ); + f.checkpoint(worker); + let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); + assert_eq!(completion.process_order, 1); + assert!( + !root.exists(), + "checkpoint completed with original capture/catalog/read unavailable" + ); + std::fs::rename(&unavailable, &root).unwrap(); + f.api + .accept_backend_job_result( + worker, + &BackendJobResultSubmission { + job_id: f.request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id.clone(), + input_digest: digest, + result: serde_json::to_value(completion).unwrap(), + }, + ) + .unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + } + + #[tokio::test] + async fn backend_over_100_entries_long_utf8_cap_continuation_and_postcapture_compaction() { + let long = "原文🙂é".repeat(36_000); + let mut entries: Vec<_> = (0..130) + .map(|n| { + history( + &format!("ordinary-{n}"), + message(&format!("ordinary original {n:03}")), + ) + }) + .collect(); + entries.push(history( + "long-tool", + session_store::LoggedItem::ToolResult { + call_id: "ordinary-tool".into(), + summary: "NOT THE ORIGINAL BODY".into(), + content: Some(long.clone()), + attachments: vec![], + disposition: Default::default(), + is_error: false, + }, + )); + let f = RuntimeFixture::with_history(entries).await; + let attr = f + .store + .session_attribution(&f.session.to_string()) + .unwrap() + .unwrap(); + let original_segment = f.range.source.pointer.segment_id; + let frozen = serde_json::to_vec(&f.request.input).unwrap(); + let rebuilt = JournalInput::build(&f.store, &f.host, &f.id, |order, offset, budget| { + let captured = subjektiv_journal::capture_input(&f.api, &attr, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string()))?; + // Append after Host capture but before the enclosing Journal input is finalized. + append(&f, original_segment, "append-before-dispatch"); + Ok(captured) + }) + .unwrap(); + assert_eq!( + serde_json::to_vec(&serde_json::to_value(&rebuilt).unwrap()).unwrap(), + frozen + ); + assert!( + !serde_json::to_string(&rebuilt) + .unwrap() + .contains("append-before-dispatch") + ); + let first = f + .api + .dispatch_backend_job_reservation(f.reserved.clone()) + .unwrap(); + // A real compaction header carries exactly one excluded opening summary + // and its complete original lineage; an empty header is intentionally invalid. + let mut summary = history( + "compaction-summary", + session_store::LoggedItem::Message { + role: session_store::LoggedRole::System, + content: vec![session_store::LoggedContentPart::Text { + text: "PRIVATE_COMPACTION_SUMMARY_NEVER_CAPTURE".into(), + }], + }, + ); + summary.metadata.input_metadata = Some( + session_store::HistoryInputMetadata::new( + session_store::HistoryInputUsePolicy { + recording_material: session_store::InputEvidenceUse::Exclude, + memory_evidence: session_store::InputEvidenceUse::Exclude, + observation: session_store::InputObservationUse::Hidden, + compaction: session_store::InputCompactionUse::OpeningSummary, + }, + None, + None, + ) + .unwrap(), + ); + summary.metadata.origin = session_store::LoggedSessionHistoryOrigin::Provided { + producer: "ordinary-compaction".into(), + }; + summary.metadata.derivation = Some(session_store::LoggedHistoryDerivation { + sources: (0..130) + .map(|n| session_store::LoggedSessionHistoryEntryId(format!("ordinary-{n}"))) + .chain([ + session_store::LoggedSessionHistoryEntryId("long-tool".into()), + session_store::LoggedSessionHistoryEntryId("append-before-dispatch".into()), + ]) + .collect(), + }); + // Actual compacted Segment/pointer movement while the first Job is executing. + let compact = Uuid::new_v4(); + let root = session_root(f.dir.path(), &f.source); + WorkerSessionStore::new(&root) + .unwrap() + .create_segment( + f.session, + compact, + &[LogEntry::AnnotatedSegmentStart { + ts: 3, + session_id: f.session, + system_prompt: Some("PRIVATE_SYSTEM_NEVER_CAPTURE".into()), + config: Default::default(), + history: vec![summary], + forked_from: None, + compacted_from: Some(session_store::SegmentOrigin { + segment_id: original_segment, + at_turn_index: 0, + }), + }], + ) + .unwrap(); + use session_store::WorkerMetadataStore as _; + let aggregate = root.parent().unwrap(); + let name = format!("worker-runtime-{}", f.source.worker_id); + session_store::WorkerAggregateStore::new(aggregate, &name) + .unwrap() + .write( + &session_store::WorkerMetadata::new( + &name, + Some(session_store::WorkerActiveSegmentRef::active_segment( + f.session, compact, + )), + ) + .with_workspace_id(TEST_WORKSPACE_ID), + ) + .unwrap(); + append(&f, compact, "append-during-job"); + assert_eq!( + serde_json::to_vec( + &f.api + .store + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() + .unwrap() + .request + .input + ) + .unwrap(), + frozen + ); + + let mut reconstructed: BTreeMap = BTreeMap::new(); + let mut current = first; + let mut generations = 0; + let mut split_tool = 0; + loop { + generations += 1; + assert!(generations < 20, "bounded continuation made no progress"); + let input = JournalInput::decode(¤t.job.request.input).unwrap(); + assert!( + serde_json::to_vec(¤t.job.request.input) + .unwrap() + .len() + <= job::MAX_JOB_INPUT_BYTES + ); + assert_eq!( + input.selected_ranges, + current + .job + .request + .grants + .subjektiv_journal + .as_ref() + .unwrap() + .selected_ranges + ); + let before = f.store.read_journal(&f.host, &f.id).unwrap(); + assert_eq!(input.selected_ranges[0].source.order, before.process_order); + assert_eq!(input.selected_ranges[0].bytes.start, before.process_byte); + for (entry, range) in input.capture.entries.iter().zip(&input.selected_ranges) { + let session_store::capture::CaptureBody::CanonicalPublic { + text, byte_offset, .. + } = &entry.body + else { + panic!("original canonical bytes") + }; + let slot = reconstructed + .entry(range.source.order) + .or_insert_with(|| (range.source.clone(), String::new())); + assert_eq!(slot.0, range.source); + assert_eq!( + *byte_offset, + slot.1.len() as u64, + "no overlap or skipped bytes across Jobs" + ); + assert_eq!(range.bytes.end - range.bytes.start, text.len() as u64); + slot.1.push_str(text); + if entry.entry_id == "long-tool" { + split_tool += 1; + } + assert!(!text.contains("PRIVATE_SYSTEM_NEVER_CAPTURE")); + assert!(!text.contains("PRIVATE_COMPACTION_SUMMARY_NEVER_CAPTURE")); + } + let worker = current.attempt.worker.as_ref().unwrap(); + let encoded = current + .job + .request + .worker_input(¤t.attempt.attempt_id) + .unwrap(); + assert!( + f.execution + .take_inputs() + .iter() + .any(|(_, text)| text == &encoded) + ); + // Host delivery is not a processing checkpoint. + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + before.process_order + ); + checkpoint_ranges(&f, worker, &input); + let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); + let done = completion.process_order == 133 && completion.process_byte == 0; + f.api + .accept_backend_job_result( + worker, + &BackendJobResultSubmission { + job_id: current.job.request.job_id.clone(), + attempt_id: current.attempt.attempt_id.clone(), + input_digest: current.job.input_digest.clone(), + result: serde_json::to_value(completion).unwrap(), + }, + ) + .unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + ¤t.job.request.job_id, + ¤t.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + if done { + break; + } + current = tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let jobs = f + .api + .store + .list_backend_journal_jobs( + TEST_WORKSPACE_ID, + &f.session.to_string(), + None, + 100, + ) + .unwrap(); + for job in jobs + .into_iter() + .filter(|job| job.state == BackendJobState::Pending) + { + let attempt = f + .api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &job.request.job_id, + &crate::backend_job::attempt_id( + &job.request.job_id, + job.current_attempt, + ), + ) + .unwrap() + .unwrap(); + if attempt.state == BackendJobAttemptState::Dispatched { + return BackendJobReservation { + job, + attempt, + replayed: true, + resource_reused: false, + }; + } + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap_or_else(|_| { + let state = f.store.read_journal(&f.host, &f.id).unwrap(); + let probe = JournalInput::build(&f.store, &f.host, &f.id, |order, offset, budget| { + subjektiv_journal::capture_input(&f.api, &attr, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string())) + }).map(|input| input.selected_ranges.iter().map(|range| (range.source.order, range.bytes.clone())).collect::>()); + panic!("production drain stalled after {generations} generations at {}/{}, next capture={probe:?}", state.process_order, state.process_byte); + }); + } + assert!(generations > 3); + assert!( + split_tool > 1, + "long UTF-8 source must continue across whole-envelope caps" + ); + assert_eq!(reconstructed.len(), 133); + for (order, (source, bytes)) in &reconstructed { + assert_eq!( + bytes.len() as u64, + source.total_bytes, + "complete canonical source {order}" + ); + let digest: String = Sha256::digest(bytes.as_bytes()) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + assert_eq!( + digest, source.content_sha256, + "exact original body, not summary or preview" + ); + } + assert!(reconstructed[&130].1.contains(&long)); + assert!(reconstructed[&131].1.contains("append-before-dispatch")); + assert!(reconstructed[&132].1.contains("append-during-job")); + } +} + +#[tokio::test] +async fn explicit_erasure_reclaims_over_100_historical_body_inputs_and_waits_for_delivered_copy_ack() + { + let mut f = RuntimeFixture::new().await; + f.api + .store + .finish_backend_job_attempt( + TEST_WORKSPACE_ID, + &f.request.job_id, + &f.reserved.attempt.attempt_id, + BackendJobAttemptState::Failed, + "ordinary_predispatch_failure", + "saved input remains resumable", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + // Synthetic ordinary historical Job intents: no execution/model fixture. Failed + // attempts deliberately keep resume bodies, so erasure must page beyond 100. + for n in 0..105 { + let mut request = f.request.clone(); + request.job_id = format!("historical-body-{n:03}"); + let reserved = f + .api + .store + .reserve_backend_job(TEST_WORKSPACE_ID, &request, &Utc::now().to_rfc3339()) + .unwrap(); + assert_eq!(reserved.job.request.input, f.request.input); + f.api + .store + .finish_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &reserved.attempt.attempt_id, + BackendJobAttemptState::Failed, + "ordinary_historical_failure", + "saved input still required for resume", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + } + f.reserved = f + .api + .store + .reserve_backend_job_retry( + TEST_WORKSPACE_ID, + &f.request.job_id, + &Utc::now().to_rfc3339(), + ) + .unwrap(); + let delivered = f.dispatch_input(); + let worker = delivered.attempt.worker.as_ref().unwrap(); + f.api + .store + .update_worker_retention( + TEST_WORKSPACE_ID, + worker, + "pinned", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + f.stop_source(); + let key = "erase-all-historical-job-bodies"; + assert!( + f.erase_pending_or_acknowledged(key), + "blocked copy must never acknowledge erasure" + ); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &delivered.attempt.attempt_id, + BackendJobWorkerCleanupState::Failed, + ) + .await; + assert!( + session_root(f.dir.path(), worker).exists(), + "failure injection remains effective" + ); + let first = f + .api + .store + .list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.session.to_string(), None, 100) + .unwrap(); + assert_eq!(first.len(), 100); + let rest = f + .api + .store + .list_backend_journal_jobs( + TEST_WORKSPACE_ID, + &f.session.to_string(), + Some(&first.last().unwrap().request.job_id), + 100, + ) + .unwrap(); + assert_eq!(rest.len(), 6); + for saved in first.iter().chain(&rest) { + assert_eq!( + saved.input_disposition, + crate::backend_job::JobInputDisposition::Erased + ); + assert_eq!(saved.input_digest, f.request.input_digest().unwrap()); + assert!(saved.load_executable_request().is_err()); + assert!( + !serde_json::to_string(&saved.request) + .unwrap() + .contains(SOURCE_SENTINEL) + ); + } + // Domain and input reclamation alone are not a delivered-copy cleanup ack. + assert!(f.erase_pending_or_acknowledged(key)); + f.api + .store + .update_worker_retention( + TEST_WORKSPACE_ID, + worker, + "normal", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + f.api.recover_backend_jobs().unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &delivered.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + f.assert_erasure_acknowledged(key); + f.assert_purged(worker); + assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); +} + +mod lifecycle_barrier { + use super::*; + use std::sync::mpsc::{self, Receiver, Sender}; + use std::time::Duration; + + const DEADLINE: Duration = Duration::from_secs(20); + // Bounded observation of an already signalled callback while the actual mutex + // is demonstrably held. This is a channel timeout, never a scheduling sleep. + const HELD_WINDOW: Duration = Duration::from_millis(100); + + pub(super) fn spawn( + work: impl FnOnce() -> T + Send + 'static, + ) -> (std::thread::JoinHandle<()>, Receiver) { + let runtime = tokio::runtime::Handle::current(); + let (tx, rx) = mpsc::channel(); + let join = std::thread::spawn(move || { + let _runtime = runtime.enter(); + let result = work(); + tx.send(result).unwrap(); + }); + (join, rx) + } + pub(super) fn joined(task: (std::thread::JoinHandle<()>, Receiver)) -> T { + let result = task + .1 + .recv_timeout(DEADLINE) + .expect("bounded callback progress; no gate/Runtime acknowledgement cycle"); + task.0.join().expect("callback thread joined"); + result + } + fn acceptance_barrier(f: &RuntimeFixture) -> (Receiver, Sender<()>) { + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let root = f.dir.path().to_path_buf(); + let encoded = f + .request + .worker_input(&f.reserved.attempt.attempt_id) + .unwrap(); + f.execution.before_input_returns(move |worker| { + let worker = + RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, worker.worker_id.to_string()); + seed_session( + &session_root(&root, &worker), + session_store::new_session_id(), + message(&encoded), + session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + ); + entered_tx.send(worker).unwrap(); + // Model the Runtime's SubmissionAccepted boundary, NOT Tool completion. + // A real Controller can send the first Tool concurrently. Waiting for + // that Tool here would manufacture the very circular wait under test. + release_rx + .recv_timeout(DEADLINE) + .expect("test released Runtime acceptance"); + }); + (entered_rx, release_tx) + } + fn dispatch( + f: &RuntimeFixture, + ) -> ( + std::thread::JoinHandle<()>, + Receiver>, + ) { + let api = f.api.clone(); + let reserved = f.reserved.clone(); + spawn(move || api.dispatch_backend_job_reservation(reserved)) + } + fn operation(f: &RuntimeFixture) -> JournalOperation { + let generation = f + .store + .active_journal_generation(&f.host, &f.id) + .unwrap() + .unwrap(); + JournalOperation { + request_id: "first-tool-after-acceptance".into(), + expected_state: generation.initial_journal.state_token, + action: JournalAction::ConfirmUnchanged, + covered_ranges: vec![f.range.clone()], + } + } + fn assert_no_preaccept_attestation(f: &RuntimeFixture) { + let generation = f + .store + .active_journal_generation(&f.host, &f.id) + .unwrap() + .unwrap(); + let context = + JournalJobContext::validated(&f.store, &f.host, &generation.id, generation.job) + .unwrap(); + let error = subjektiv::journal::service::execute_snapshot( + &f.store, + &context, + &f.request.input_digest().unwrap(), + JournalJobRequest::ReadState, + ) + .unwrap_err(); + assert!( + error.to_string().contains("trusted Host delivery"), + "{error}" + ); + assert!( + f.store + .query_journal_operation(&f.host, &f.id, "first-tool-after-acceptance") + .unwrap() + .is_none() + ); + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + 0 + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn dispatch_first_tool_waits_for_runtime_acceptance_then_progresses_without_ack_cycle() { + let f = RuntimeFixture::new().await; + let (entered, release) = acceptance_barrier(&f); + let dispatch = dispatch(&f); + let worker = entered.recv_timeout(DEADLINE).unwrap(); + let gate = subjektiv_journal::lifecycle_lock( + &f.api, + &f.request + .grants + .subjektiv_journal + .as_ref() + .unwrap() + .subject_id, + ) + .unwrap(); + assert!(matches!( + gate.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + )); + let (attempted_tx, attempted_rx) = mpsc::channel(); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let op = operation(&f); + let tool = spawn(move || { + attempted_tx.send(()).unwrap(); + subjektiv_journal::execute( + &api, + &callback_worker, + JournalJobRequest::Execute { operation: op }, + ) + }); + attempted_rx.recv_timeout(DEADLINE).unwrap(); + let (finish_tx, finish_rx) = mpsc::channel(); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let early_finish = spawn(move || { + finish_tx.send(()).unwrap(); + subjektiv_journal::finish(&api, &callback_worker) + }); + finish_rx.recv_timeout(DEADLINE).unwrap(); + assert!( + matches!( + early_finish.1.recv_timeout(HELD_WINDOW), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "finish must wait for the same acceptance gate, not mutate before attestation" + ); + assert!( + matches!( + tool.1.recv_timeout(HELD_WINDOW), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "first Tool must not bypass dispatch's held Subject gate" + ); + assert_no_preaccept_attestation(&f); + release.send(()).unwrap(); + // Dispatch must independently finish after acceptance, even with the Tool waiting. + let dispatched = joined(dispatch).unwrap(); + let unfinished = joined(early_finish).unwrap_err().to_string(); + assert!( + !unfinished.contains("trusted Host delivery"), + "acceptance attestation now exists" + ); + let JournalJobResponse::Saved(record) = joined(tool).unwrap() else { + panic!("record receipt") + }; + assert_eq!(dispatched.attempt.worker.as_ref(), Some(&worker)); + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, "first-tool-after-acceptance") + .unwrap(), + Some(record.clone()) + ); + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + 0, + "acceptance and mutation alone are not a checkpoint" + ); + let position = save_fixture_position(&f.api, &worker, record.state_token); + subjektiv_journal::execute( + &f.api, + &worker, + JournalJobRequest::Execute { + operation: JournalOperation { + request_id: "checkpoint-after-barrier".into(), + expected_state: position, + action: JournalAction::Checkpoint { + record_operation_ids: vec!["first-tool-after-acceptance".into()], + }, + covered_ranges: vec![f.range.clone()], + }, + }, + ) + .unwrap(); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let completion = joined(spawn(move || { + subjektiv_journal::finish(&api, &callback_worker) + })) + .unwrap(); + f.api + .accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: f.request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id.clone(), + input_digest: f.request.input_digest().unwrap(), + result: serde_json::to_value(completion).unwrap(), + }, + ) + .unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + f.assert_purged(&worker); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn erase_and_tool_race_dispatch_acceptance_without_inversion_and_require_cleanup_ack() { + let f = RuntimeFixture::new().await; + f.stop_source(); + let (entered, release) = acceptance_barrier(&f); + let dispatch = dispatch(&f); + let worker = entered.recv_timeout(DEADLINE).unwrap(); + f.api + .store + .update_worker_retention( + TEST_WORKSPACE_ID, + &worker, + "pinned", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + let gate = subjektiv_journal::lifecycle_lock( + &f.api, + &f.request + .grants + .subjektiv_journal + .as_ref() + .unwrap() + .subject_id, + ) + .unwrap(); + assert!(matches!( + gate.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + )); + let (attempted_tx, attempted_rx) = mpsc::channel(); + let api = f.api.clone(); + let session = f.session.to_string(); + let key = "erase-racing-dispatch-acceptance"; + let erase_attempted = attempted_tx.clone(); + let erasure = spawn(move || { + erase_attempted.send(()).unwrap(); + api.erase_subjektiv_journal_session(&session, key) + }); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let op = operation(&f); + let tool = spawn(move || { + attempted_tx.send(()).unwrap(); + subjektiv_journal::execute( + &api, + &callback_worker, + JournalJobRequest::Execute { operation: op }, + ) + }); + attempted_rx.recv_timeout(DEADLINE).unwrap(); + attempted_rx.recv_timeout(DEADLINE).unwrap(); + assert!(matches!( + erasure.1.recv_timeout(HELD_WINDOW), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert!(matches!( + tool.1.recv_timeout(HELD_WINDOW), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert!( + !f.store.journal_session_erased(&f.host).unwrap(), + "erase must wait for the same dispatch gate" + ); + assert_no_preaccept_attestation(&f); + release.send(()).unwrap(); + let dispatched = joined(dispatch).unwrap(); + let first = joined(erasure).unwrap_err(); + assert!(matches!(first, Error::RepositoryConflict(message) + if message == "Journal delivered-copy Purge remains pending; retry the same erasure after cleanup")); + // Mutex waiter order is deliberately unspecified. A Tool either saves before + // the tombstone or revalidates and is rejected; it cannot cross the fence. + if let Ok(response) = joined(tool) { + assert!(matches!(response, JournalJobResponse::Saved(_))); + } + assert!(f.store.journal_session_erased(&f.host).unwrap()); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let op = JournalOperation { + request_id: "post-tombstone-must-not-save".into(), + expected_state: "irrelevant-after-erasure".into(), + action: JournalAction::ConfirmUnchanged, + covered_ranges: vec![f.range.clone()], + }; + assert!(joined(spawn(move || { + subjektiv_journal::execute(&api, &callback_worker, JournalJobRequest::ReadState) + .is_err() + && subjektiv_journal::execute( + &api, + &callback_worker, + JournalJobRequest::Execute { operation: op }, + ) + .is_err() + && subjektiv_journal::finish(&api, &callback_worker).is_err() + }))); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Failed, + ) + .await; + assert!(session_root(f.dir.path(), &worker).exists()); + assert!(f.erase_pending_or_acknowledged(key)); + f.api + .store + .update_worker_retention( + TEST_WORKSPACE_ID, + &worker, + "normal", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + f.api.recover_backend_jobs().unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + f.assert_erasure_acknowledged(key); + f.assert_purged(&worker); + } +} + +mod journal_migration { + use super::lifecycle_barrier::{joined, spawn}; + use super::*; + use subjektiv::journal::recording::JournalInput; + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn reserved_legacy_descriptor_invalidates_exact_identity_and_migrates_only_after_saved_checkpoint() + { + let f = SubjectRemovalFixture::new().await; + let capture = subjektiv_journal::capture_input(&f.api, &f.attr, 0, 0, 48 * 1024).unwrap(); + let ranges = subjektiv::journal::recording::capture_ranges(&capture).unwrap(); + assert_eq!(ranges.len(), 2); + let id = f + .store + .accept_journal_sources(&f.host, "legacy admitted originals", &ranges) + .unwrap(); + let input = JournalInput::build(&f.store, &f.host, &id, |order, offset, budget| { + subjektiv_journal::capture_input(&f.api, &f.attr, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string())) + }) + .unwrap(); + let mut legacy = + subjektiv_journal::recording_request(&f.host, f.attr.clone(), &id, input).unwrap(); + legacy.job_id = "reserved-descriptor-only-legacy".into(); + legacy.input = json!({"journal_id":id, "selected_ranges":ranges}); + legacy + .grants + .subjektiv_journal + .as_mut() + .unwrap() + .binding_id + .clear(); + let reserved = f + .api + .store + .reserve_backend_job(TEST_WORKSPACE_ID, &legacy, &Utc::now().to_rfc3339()) + .unwrap(); + assert_eq!(reserved.attempt.state, BackendJobAttemptState::Reserved); + assert_eq!(reserved.job.request, legacy); + let digest = legacy.input_digest().unwrap(); + let fingerprint = legacy.fingerprint().unwrap(); + // A previous generation saved the first prefix before its acknowledgement + // was lost. Seed that real domain checkpoint, not a model success claim. + let binding = subjektiv::api::JobAttemptBinding { + job_id: "previous-checkpointed-generation".into(), + attempt_id: "previous-attempt".into(), + }; + let generation = f + .store + .prepare_journal_generation(&f.host, &id, binding.clone(), vec![ranges[0].clone()]) + .unwrap(); + let ctx = JournalJobContext::validated(&f.store, &f.host, &generation.id, binding.clone()) + .unwrap(); + let record = f + .store + .execute_journal( + &ctx, + JournalOperation { + request_id: "legacy-saved-record".into(), + expected_state: generation.initial_journal.state_token, + action: JournalAction::Create { + title: "Saved before migration".into(), + body: "First original reflected; second still pending".into(), + ongoing: "Continue second original only".into(), + state: JournalEntryState::Open, + source_end: None, + correction_refs: vec![], + }, + covered_ranges: vec![ranges[0].clone()], + }, + ) + .unwrap(); + let position = f + .store + .execute_journal( + &ctx, + JournalOperation { + request_id: "legacy-saved-position".into(), + expected_state: record.state_token.clone(), + action: JournalAction::UpdatePosition { + current_position: "First original saved; remaining work unconfirmed".into(), + }, + covered_ranges: vec![], + }, + ) + .unwrap(); + let checkpoint = f + .store + .checkpoint_journal( + &ctx, + JournalOperation { + request_id: "legacy-saved-checkpoint".into(), + expected_state: position.state_token, + action: JournalAction::Checkpoint { + record_operation_ids: vec!["legacy-saved-record".into()], + }, + covered_ranges: vec![ranges[0].clone()], + }, + ) + .unwrap(); + f.store + .close_journal_generation(&f.host, &generation.id, &binding) + .unwrap(); + assert_eq!(f.store.read_journal(&f.host, &id).unwrap().process_order, 1); + let api = f.api.clone(); + let migrated = joined(spawn(move || { + api.dispatch_backend_job_reservation(reserved) + })) + .unwrap(); + assert_eq!(migrated.job.request.job_id, legacy.job_id); + let old = f + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &legacy.job_id) + .unwrap() + .unwrap(); + assert_eq!(old.state, BackendJobState::Failed); + assert_eq!( + old.failure_category.as_deref(), + Some("journal_descriptor_invalidated") + ); + assert_eq!( + old.input_disposition, + crate::backend_job::JobInputDisposition::Invalidated + ); + assert_eq!(old.input_digest, digest); + assert_eq!(old.intent_fingerprint, fingerprint); + assert_eq!(old.request.job_id, legacy.job_id); + assert!(old.request.input.is_null()); + assert!(old.load_executable_request().is_err()); + assert_eq!(old.current_attempt, 1); + let old_attempt = f + .api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &legacy.job_id, + &crate::backend_job::attempt_id(&legacy.job_id, 1), + ) + .unwrap() + .unwrap(); + assert_eq!(old_attempt.input_digest, digest); + assert!( + old_attempt.worker.is_none(), + "descriptor-only input must never reach a Worker" + ); + + let (new, attempt) = tokio::time::timeout(std::time::Duration::from_secs(20), async { + loop { + for job in f + .api + .store + .list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100) + .unwrap() + { + if job.request.job_id == legacy.job_id { + continue; + } + let attempt = f + .api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &job.request.job_id, + &crate::backend_job::attempt_id( + &job.request.job_id, + job.current_attempt, + ), + ) + .unwrap() + .unwrap(); + if attempt.state == BackendJobAttemptState::Dispatched { + return (job, attempt); + } + } + tokio::task::yield_now().await; + } + }) + .await + .expect("migration dispatch progresses without a lifecycle gate inversion"); + assert_ne!(new.request.job_id, legacy.job_id); + let frozen = JournalInput::decode(&new.request.input).unwrap(); + assert_eq!(frozen.selected_ranges, vec![ranges[1].clone()]); + assert_eq!(frozen.initial_state_token, checkpoint.state_token); + assert_eq!( + f.store + .query_journal_operation(&f.host, &id, "legacy-saved-record") + .unwrap(), + Some(record) + ); + assert_eq!( + f.store + .query_journal_operation(&f.host, &id, "legacy-saved-checkpoint") + .unwrap(), + Some(checkpoint) + ); + assert_eq!( + f.store + .list_journal_entries(&f.host, &id, None, 16) + .unwrap() + .items + .len(), + 1 + ); + let api = f.api.clone(); + let worker = attempt.worker.clone().unwrap(); + assert!( + joined(spawn(move || subjektiv_journal::execute( + &api, + &worker, + JournalJobRequest::ReadState + ))) + .is_ok() + ); + let api = f.api.clone(); + let stale = migrated.clone(); + assert!( + joined(spawn(move || api.dispatch_backend_job_reservation(stale))).is_err(), + "invalidated internal reservations cannot dispatch or silently replay" + ); + // Stop this intentionally unfinished new body Job without pretending its + // remaining original was reflected; normal cleanup still needs its receipt. + f.api + .finish_backend_job_attempt( + &new.request.job_id, + &attempt.attempt_id, + BackendJobAttemptState::Failed, + "test_finished", + "remaining original deliberately unprocessed", + ) + .unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &new.request.job_id, + &attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert_eq!(f.store.read_journal(&f.host, &id).unwrap().process_order, 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn tampered_or_missing_body_snapshot_is_not_legacy_and_never_falls_back_to_available_source() + { + for missing in [false, true] { + let f = RuntimeFixture::new().await; + f.api + .store + .finish_backend_job_attempt( + TEST_WORKSPACE_ID, + &f.request.job_id, + &f.reserved.attempt.attempt_id, + BackendJobAttemptState::Failed, + "predispatch_fixture", + "release resource before independently saved malformed request", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + let mut malformed = f.request.clone(); + malformed.job_id = format!("malformed-body-snapshot-{missing}"); + if missing { + malformed.input.as_object_mut().unwrap().remove("capture"); + } else { + malformed.input["capture"]["entries"][0]["body"]["text"] = + json!("tampered body, same canonical descriptor"); + } + assert!(JournalInput::decode(&malformed.input).is_err()); + let bad = f + .api + .store + .reserve_backend_job(TEST_WORKSPACE_ID, &malformed, &Utc::now().to_rfc3339()) + .unwrap(); + assert_eq!(bad.job.request.input, malformed.input); + assert!( + bad.job.load_executable_request().is_ok(), + "generic immutable digest is valid; semantic body integrity is not" + ); + f.execution.take_inputs(); + let api = f.api.clone(); + let candidate = bad.clone(); + assert!( + !joined(spawn(move || subjektiv_journal::migrate_reserved_legacy( + &api, &candidate + ))) + .unwrap() + ); + let api = f.api.clone(); + assert!(joined(spawn(move || api.dispatch_backend_job_reservation(bad))).is_err()); + assert!(f.execution.take_inputs().is_empty()); + assert!( + session_root(f.dir.path(), &f.source).exists(), + "original source is available but must not repair saved input" + ); + let jobs = f + .api + .store + .list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.session.to_string(), None, 100) + .unwrap(); + assert_eq!( + jobs.len(), + 2, + "malformed body must not create a replacement/migration Job" + ); + let saved = jobs + .iter() + .find(|job| job.request.job_id == malformed.job_id) + .unwrap(); + assert_eq!( + saved.input_disposition, + crate::backend_job::JobInputDisposition::Available + ); + assert_eq!(saved.input_digest, malformed.input_digest().unwrap()); + assert_eq!(saved.request.input, malformed.input); + assert!( + f.store + .active_journal_generation(&f.host, &f.id) + .unwrap() + .is_none() + ); + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + 0 + ); + } + } +} diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index d742f0ec6..6da4ad1bd 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -119,7 +119,7 @@ pub use ticket::{ }; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 102; +const LATEST_SCHEMA_VERSION: i64 = 103; const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants"; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; @@ -437,6 +437,11 @@ const MIGRATIONS: &[Migration] = &[ name: "Durable failed Subject creation recovery", apply: migrate_subject_create_recovery_v101_to_v102, }, + Migration { + version: 103, + name: "Job input retirement independent of immutable receipts", + apply: migrate_job_input_retention_v102_to_v103, + }, ]; #[derive(Clone, Copy)] @@ -1771,6 +1776,15 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { request: &BackendJobRequest, now: &str, ) -> Result; + /// Host-only retirement; fence domain mutations and delivered snapshots first for erasure. + /// Completed reclamation additionally requires canonical Worker cleanup for all attempts. + fn reclaim_backend_job_input( + &self, + workspace_id: &str, + job_id: &str, + expected_input_digest: &str, + reason: crate::backend_job::JobInputRetirementReason, + ) -> Result; fn reserve_backend_job_retry( &self, workspace_id: &str, @@ -7358,6 +7372,48 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } + fn reclaim_backend_job_input( + &self, + workspace_id: &str, + job_id: &str, + expected_input_digest: &str, + reason: crate::backend_job::JobInputRetirementReason, + ) -> Result { + use crate::backend_job::{JobInputDisposition, JobInputRetirementReason}; + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let job = read_backend_job(&tx, workspace_id, job_id)? + .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + if job.input_digest != expected_input_digest { + return Err(Error::InvalidInput("Backend Job input digest mismatch".into())); + } + let desired = reason.disposition(); + if job.input_disposition == desired { + tx.commit()?; + reclaim_backend_job_input_storage(conn)?; + return Ok(job); + } + if reason == JobInputRetirementReason::Completed { + job.load_executable_request()?; + let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))? + .ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?; + if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed + || job.result.is_none() || backend_job_has_unfinished_cleanup(&tx, workspace_id, job_id)? + { return Err(Error::InvalidInput("input reclaim requires known completed result and completed Worker cleanup".into())); } + } + if job.input_disposition != JobInputDisposition::Available + && reason != JobInputRetirementReason::ExplicitErasure + { return Err(Error::InvalidInput("retired Backend Job input cannot be restored or reclassified".into())); } + let mut metadata = job.request.clone(); + metadata.input = serde_json::Value::Null; + tx.execute("UPDATE backend_jobs SET request_json=?3, input_disposition=?4 WHERE workspace_id=?1 AND job_id=?2", params![workspace_id,job_id,serde_json::to_string(&metadata).map_err(|e| Error::Store(e.to_string()))?,desired.as_str()])?; + let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::Store("retired Backend Job disappeared".into()))?; + tx.commit()?; + reclaim_backend_job_input_storage(conn)?; + Ok(job) + }) + } + fn reserve_backend_job_retry( &self, workspace_id: &str, @@ -7368,6 +7424,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let mut job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + job.load_executable_request()?; let previous_id = attempt_id(job_id, job.current_attempt); let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)? .ok_or_else(|| Error::Store("Backend Job current attempt is missing".to_string()))?; @@ -7407,7 +7464,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } tx.execute( "INSERT INTO backend_job_attempts (workspace_id, job_id, attempt_id, attempt, input_digest, state, deadline_at, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, 'reserved', ?6, ?7, ?7)", - params![workspace_id, job_id, next_id, next as i64, job.request.input_digest()?, deadline, now], + params![workspace_id, job_id, next_id, next as i64, job.input_digest, deadline, now], )?; job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::Store("retried Backend Job is missing".to_string()))?; @@ -7446,7 +7503,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { "SELECT job.job_id FROM backend_jobs job JOIN backend_job_attempts attempt ON attempt.workspace_id = job.workspace_id AND attempt.job_id = job.job_id AND attempt.attempt = job.current_attempt - WHERE job.workspace_id = ?1 AND job.state = 'pending' + WHERE job.workspace_id = ?1 AND job.state = 'pending' AND job.input_disposition = 'available' AND attempt.state = 'dispatched' AND attempt.runtime_id = ?2 AND attempt.worker_id = ?3 AND attempt.runtime_run_id IS NOT NULL", params![workspace_id, worker.runtime_id.as_str(), worker.worker_id.as_str()], @@ -7506,6 +7563,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; if attempt.state != BackendJobAttemptState::Reserved { @@ -7543,7 +7601,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result> { self.with_conn(|conn| { let mut statement = conn.prepare( - "SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state = 'reserved' ORDER BY created_at, attempt_id LIMIT ?2", + "SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='reserved' AND j.input_disposition='available' ORDER BY a.created_at, a.attempt_id LIMIT ?2", )?; let identities = statement.query_map(params![workspace_id, limit as i64], |row| { Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) @@ -7568,9 +7626,10 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; - if attempt.attempt != job.current_attempt || attempt.input_digest != job.request.input_digest()? { + if attempt.attempt != job.current_attempt || attempt.input_digest != job.input_digest { return Err(Error::InvalidInput( "Backend Job attempt is stale for the current input digest".to_string(), )); @@ -7629,6 +7688,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?; + job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| { Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")) @@ -7682,7 +7743,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?; let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?; - if submission.input_digest != job.request.input_digest()? + if submission.input_digest != job.input_digest || attempt.input_digest != submission.input_digest || attempt.attempt != job.current_attempt { @@ -7709,6 +7770,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { tx.commit()?; return Ok(BackendJobResultAcceptance { job, attempt, replayed: true }); } + job.load_executable_request()?; if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending { return Err(Error::InvalidInput( "Backend Job attempt no longer accepts results".to_string(), @@ -7776,7 +7838,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { // Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence. if &attempt != expected || job.current_attempt != attempt.attempt || job.request.grants.subjektiv_journal.is_none() - || attempt.input_digest != job.request.input_digest()? + || attempt.input_digest != job.input_digest || !matches!(job.state, BackendJobState::Unknown | BackendJobState::Completed | BackendJobState::Failed) { return Err(Error::InvalidInput("Journal reconciliation binding changed".into())); } @@ -12109,6 +12171,20 @@ fn read_memory_staging_resolution_record( }) } +// The receipt/disposition commit precedes physical cleanup. An error here is +// safely retried with the same expected digest/reason, not a new execution. +fn reclaim_backend_job_input_storage(conn: &Connection) -> Result<()> { + conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?; + let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?; + if busy != 0 { + return Err(Error::Store( + "Backend Job input retired; SQLite body cleanup blocked by a reader, retry retirement" + .into(), + )); + } + Ok(()) +} + fn backend_job_deadline(now: &str, timeout_seconds: u32) -> Result { let now = chrono::DateTime::parse_from_rfc3339(now) .map_err(|error| Error::InvalidInput(format!("invalid Backend Job timestamp: {error}")))?; @@ -12172,7 +12248,7 @@ pub(crate) fn terminal_journal_job_worker( Ok(job.request.grants.subjektiv_journal.is_some() && attempt.state.terminal() && attempt.worker.as_ref() == Some(worker) - && attempt.input_digest == job.request.input_digest()?) + && attempt.input_digest == job.input_digest) } fn read_backend_job( @@ -12181,13 +12257,13 @@ fn read_backend_job( job_id: &str, ) -> Result> { let row = conn.query_row( - "SELECT request_json, intent_fingerprint, state, current_attempt, result_json, result_digest, failure_category, failure_detail, created_at, updated_at, completed_at FROM backend_jobs WHERE workspace_id = ?1 AND job_id = ?2", + "SELECT request_json, intent_fingerprint, state, current_attempt, result_json, result_digest, failure_category, failure_detail, created_at, updated_at, completed_at, input_digest, input_disposition FROM backend_jobs WHERE workspace_id = ?1 AND job_id = ?2", params![workspace_id, job_id], |row| Ok(( row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?, row.get::<_, i64>(3)?, row.get::<_, Option>(4)?, row.get::<_, Option>(5)?, row.get::<_, Option>(6)?, row.get::<_, Option>(7)?, row.get::<_, String>(8)?, - row.get::<_, String>(9)?, row.get::<_, Option>(10)?, + row.get::<_, String>(9)?, row.get::<_, Option>(10)?, row.get::<_, String>(11)?, row.get::<_, String>(12)?, )), ).optional()?; let Some(( @@ -12202,11 +12278,20 @@ fn read_backend_job( created_at, updated_at, completed_at, + input_digest, + input_disposition, )) = row else { return Ok(None); }; - let request: BackendJobRequest = serde_json::from_str(&request_json) + let request_value: serde_json::Value = serde_json::from_str(&request_json) + .map_err(|error| Error::Store(format!("invalid stored Backend Job request: {error}")))?; + if request_value.get("input").is_none() { + return Err(Error::Store( + "stored Backend Job input field is missing".into(), + )); + } + let request: BackendJobRequest = serde_json::from_value(request_value) .map_err(|error| Error::Store(format!("invalid stored Backend Job request: {error}")))?; let result = result_json .map(|value| serde_json::from_str(&value)) @@ -12218,6 +12303,9 @@ fn read_backend_job( workspace_id: workspace_id.to_string(), request, intent_fingerprint, + input_digest, + input_disposition: serde_json::from_value(serde_json::Value::String(input_disposition)) + .map_err(|e| Error::Store(format!("invalid Job input disposition: {e}")))?, state: BackendJobState::parse(&state)?, current_attempt, result, @@ -15130,6 +15218,24 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> { Ok(()) } +fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 102 { + return Err(Error::Store( + "Job input retention migration requires schema 102".into(), + )); + } + let tx = conn.unchecked_transaction()?; + tx.execute_batch("ALTER TABLE backend_jobs ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));")?; + tx.execute_batch(crate::backend_job::INPUT_RETENTION_TRIGGERS)?; + tx.execute( + "INSERT INTO __yoi_schema_migrations(version,name) VALUES(103,?1)", + ["Job input retirement independent of immutable receipts"], + )?; + verify_workspace_resource_constraints(&tx)?; + tx.commit()?; + Ok(()) +} + fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> { if current_schema_version(conn)? != 101 { return Err(Error::Store( @@ -18534,7 +18640,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx assert!(conn.execute("INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) VALUES('invalid','missing','Invalid','active','1','1')", []).is_err()); } - fn prepare_retained_schema(path: &Path, version: i64) { + pub(super) fn prepare_retained_schema(path: &Path, version: i64) { // Cache only empty preconditions, built from frozen DDL and real retained // migrations. The test's seeded rows and migration under test are never shared. static TEMPLATES: std::sync::OnceLock>> = @@ -18914,6 +19020,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()}, WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()}, WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()}, + WorkspaceSchemaMigrationStep {version:103, name:"Job input retirement independent of immutable receipts".into()}, ] ); @@ -19026,6 +19133,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx (100, "Provisional users and scoped passkey activation".to_string()), (101, "Durable explicit Subject body release".to_string()), (102, "Durable failed Subject creation recovery".to_string()), +(103, "Job input retirement independent of immutable receipts".to_string()), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -21293,7 +21401,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx drop(conn); let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap(); reopened - .with_conn(|c| verify_schema_history(c, 102)) + .with_conn(|c| verify_schema_history(c, LATEST_SCHEMA_VERSION)) .unwrap(); } @@ -22232,7 +22340,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx (99, "Explicit hosting administrators and self-registration policy".into()), (100, "Provisional users and scoped passkey activation".into()), (101, "Durable explicit Subject body release".into()), - (102, "Durable failed Subject creation recovery".into())]); + (102, "Durable failed Subject creation recovery".into()), + (103, "Job input retirement independent of immutable receipts".into())]); for (table, rows) in unchanged.iter().zip(before.iter()) { assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}"); } @@ -26150,6 +26259,321 @@ INSERT INTO worker_registry ( .unwrap(); } + // Storage component tests: seed already-saved result/cleanup receipts, without + // constructing an execution Worker or claiming a model/dispatch integration. + fn seed_completed_job_receipts(store: &SqliteWorkspaceStore, id: &str, cleanup: &str) { + let result = serde_json::json!({"stored":true}); + let (digest, json) = result_digest(&result, job::ABSOLUTE_MAX_RESULT_BYTES).unwrap(); + store.with_conn(|conn| { + conn.execute("UPDATE backend_job_attempts SET state='completed',result_json=?2,result_digest=?3,worker_cleanup_state=?4,worker_cleanup_completed_at=?5 WHERE job_id=?1", params![id,json,digest,cleanup,RESOURCE_JOB_NOW])?; + conn.execute("UPDATE backend_jobs SET state='completed',result_json=?2,result_digest=?3,completed_at=?4 WHERE job_id=?1", params![id,json,digest,RESOURCE_JOB_NOW])?; + Ok(()) + }).unwrap(); + } + + #[tokio::test] + async fn backend_job_input_reclaim_preserves_receipts_reopen_and_exact_replay() { + use crate::backend_job::{JobInputDisposition, JobInputRetirementReason}; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("input.sqlite"); + let store = SqliteWorkspaceStore::open_api_fixture(&path).unwrap(); + seed_resource_job_workspace(&store).await; + let original = resource_job_request("reclaim", "subject-a", "candidate"); + let reservation = store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + assert!( + store + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + &reservation.job.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + seed_completed_job_receipts(&store, "reclaim", "pending"); + assert!( + store + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + &reservation.job.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + store.with_conn(|conn| { conn.execute("UPDATE backend_job_attempts SET worker_cleanup_state='completed' WHERE job_id='reclaim'",[])?; Ok(()) }).unwrap(); + let before = store + .get_backend_job("resource-workspace", "reclaim") + .unwrap() + .unwrap(); + let attempt = store + .get_backend_job_attempt( + "resource-workspace", + "reclaim", + &reservation.attempt.attempt_id, + ) + .unwrap() + .unwrap(); + assert!( + store + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + "wrong", + JobInputRetirementReason::Completed + ) + .is_err() + ); + let retired = store + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + &before.input_digest, + JobInputRetirementReason::Completed, + ) + .unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed); + assert!(retired.request.input.is_null()); + assert!(retired.load_executable_request().is_err()); + assert_eq!(retired.input_digest, original.input_digest().unwrap()); + assert_eq!(retired.intent_fingerprint, before.intent_fingerprint); + assert_eq!(retired.request.grants, original.grants); + assert_eq!(retired.result, before.result); + assert_eq!( + store + .get_backend_job_attempt("resource-workspace", "reclaim", &attempt.attempt_id) + .unwrap() + .unwrap(), + attempt + ); + drop(store); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let replay = reopened + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + assert!(replay.replayed); + assert_eq!(replay.job, retired); + assert_eq!( + replay.outcome().input_digest, + original.input_digest().unwrap() + ); + let mut changed = original.clone(); + changed.input = serde_json::json!({"changed":true}); + assert!( + reopened + .reserve_backend_job("resource-workspace", &changed, RESOURCE_JOB_NOW) + .is_err() + ); + assert!( + reopened + .reserve_backend_job_retry("resource-workspace", "reclaim", RESOURCE_JOB_NOW) + .is_err() + ); + assert!( + reopened + .claim_backend_job_attempt_dispatch( + "resource-workspace", + "reclaim", + &attempt.attempt_id, + RESOURCE_JOB_NOW + ) + .is_err() + ); + assert!(reopened.with_conn(|conn| { conn.execute("UPDATE backend_jobs SET input_disposition='available',request_json=?1 WHERE job_id='reclaim'", [serde_json::to_string(&original).unwrap()])?; Ok(()) }).is_err()); + let erased = reopened + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + &retired.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(erased.input_disposition, JobInputDisposition::Erased); + assert_eq!(erased.state, BackendJobState::Completed); + assert_eq!(erased.result, before.result); + } + + #[tokio::test] + async fn backend_job_input_unknown_retention_and_legacy_invalidation_do_not_reinterpret_outcome() + { + use crate::backend_job::{JobInputDisposition, JobInputRetirementReason}; + let store = SqliteWorkspaceStore::in_memory_fixture().unwrap(); + seed_resource_job_workspace(&store).await; + for (id, state) in [ + ("unfinished", None), + ("unknown-input", Some(BackendJobAttemptState::Unknown)), + ("failed-input", Some(BackendJobAttemptState::Failed)), + ] { + let original = resource_job_request(id, id, "candidate"); + let reservation = store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + if let Some(state) = state { + store + .finish_backend_job_attempt( + "resource-workspace", + id, + &reservation.attempt.attempt_id, + state, + "fixture", + "known stored outcome", + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + let before = store + .get_backend_job("resource-workspace", id) + .unwrap() + .unwrap(); + assert!( + store + .reclaim_backend_job_input( + "resource-workspace", + id, + &before.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + assert_eq!(before.load_executable_request().unwrap(), &original); + assert_eq!( + store + .get_backend_job("resource-workspace", id) + .unwrap() + .unwrap(), + before + ); + let retired = store + .reclaim_backend_job_input( + "resource-workspace", + id, + &before.input_digest, + JobInputRetirementReason::LegacyInvalidation, + ) + .unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated); + assert_eq!(retired.state, before.state); + assert_eq!(retired.failure_category, before.failure_category); + assert_eq!(retired.completed_at, before.completed_at); + assert!(retired.load_executable_request().is_err()); + assert!( + store + .reserve_backend_job_retry("resource-workspace", id, RESOURCE_JOB_NOW) + .is_err() + ); + assert!( + store + .claim_backend_job_attempt_dispatch( + "resource-workspace", + id, + &reservation.attempt.attempt_id, + RESOURCE_JOB_NOW + ) + .is_err() + ); + assert_eq!( + store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap() + .job, + retired + ); + } + } + + #[tokio::test] + async fn backend_job_input_available_loader_and_dispatch_refuse_missing_tampered_or_changed_intent() + { + for mutation in [ + "json_set(request_json,'$.input',json('{\"tampered\":true}'))", + "json_remove(request_json,'$.input')", + "json_set(request_json,'$.instruction','changed')", + ] { + let store = SqliteWorkspaceStore::in_memory_fixture().unwrap(); + seed_resource_job_workspace(&store).await; + let original = resource_job_request("tamper", "subject", "candidate"); + let reservation = store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER backend_job_input_retirement")?; + conn.execute_batch(&format!( + "UPDATE backend_jobs SET request_json={mutation}" + ))?; + Ok(()) + }) + .unwrap(); + if let Ok(Some(job)) = store.get_backend_job("resource-workspace", "tamper") { + assert!(job.load_executable_request().is_err()); + } + assert!( + store + .claim_backend_job_attempt_dispatch( + "resource-workspace", + "tamper", + &reservation.attempt.attempt_id, + RESOURCE_JOB_NOW + ) + .is_err() + ); + } + } + + #[tokio::test] + async fn backend_job_input_migration_preserves_original_identity_and_rolls_back_failure() { + use crate::backend_job::{JobInputDisposition, JobInputRetirementReason}; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("migration.sqlite"); + let store = SqliteWorkspaceStore::open_api_fixture(&path).unwrap(); + seed_resource_job_workspace(&store).await; + let original = resource_job_request("upgrade", "subject", "candidate"); + let before = store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + store.with_conn(|conn| { + conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable; + ALTER TABLE backend_jobs DROP COLUMN input_disposition; + UPDATE __yoi_schema_migrations SET version=102 WHERE version=103; + CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(()) + }).unwrap(); + drop(store); + assert!(SqliteWorkspaceStore::open_for_test(&path).is_err()); + let conn = Connection::open(&path).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 102); + assert!(!column_exists(&conn, "backend_jobs", "input_disposition").unwrap()); + conn.execute_batch("DROP TRIGGER backend_job_input_retirement") + .unwrap(); + drop(conn); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + assert_eq!( + reopened + .get_backend_job("resource-workspace", "upgrade") + .unwrap() + .unwrap(), + before.job + ); + let erased = reopened + .reclaim_backend_job_input( + "resource-workspace", + "upgrade", + &before.job.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(erased.input_disposition, JobInputDisposition::Erased); + assert_eq!(erased.state, BackendJobState::Pending); + assert_eq!( + reopened + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap() + .job, + erased + ); + } + #[tokio::test] async fn backend_job_resource_reserve_converges_across_connections_without_changing_input() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/ticket_item_checker.rs b/crates/workspace-server/src/ticket_item_checker.rs index 254312e00..b1f683e77 100644 --- a/crates/workspace-server/src/ticket_item_checker.rs +++ b/crates/workspace-server/src/ticket_item_checker.rs @@ -231,12 +231,10 @@ pub fn result_facts( current_ticket: &ticket::Ticket, ) -> Result { let input = parse_input(&acceptance.job.request.input)?; - let result = validate_result( - &acceptance.job.request.input, - acceptance.job.result.as_ref().ok_or_else(|| { - Error::InvalidInput("completed Ticket item checker Job has no result".to_string()) - })?, - )?; + let result_value = acceptance.job.result.as_ref().ok_or_else(|| { + Error::InvalidInput("completed Ticket item checker Job has no result".to_string()) + })?; + let result = validate_result(&acceptance.job.request.input, result_value)?; if input.content_digest != content_digest(current_ticket) || input.ticket_id != current_ticket.meta.id { @@ -246,21 +244,17 @@ pub fn result_facts( return Ok(TicketItemCheckFacts::NoFindings); } - Ok(TicketItemCheckFacts::Findings(render_result(acceptance)?)) + Ok(TicketItemCheckFacts::Findings(render_result( + &acceptance.job.request.input, + result_value, + )?)) } /// Stable rendering from the original checked snapshot. Used to pin pre-upgrade /// unacknowledged requests without reinterpreting them against newer Ticket text. -pub(crate) fn render_result(acceptance: &BackendJobResultAcceptance) -> Result { - let input = parse_input(&acceptance.job.request.input)?; - let result = validate_result( - &acceptance.job.request.input, - acceptance - .job - .result - .as_ref() - .ok_or_else(|| Error::InvalidInput("completed checker has no result".into()))?, - )?; +pub(crate) fn render_result(input: &Value, result: &Value) -> Result { + let checked = parse_input(input)?; + let result = validate_result(input, result)?; if result.findings.is_empty() { return Err(Error::InvalidInput( "clean checker result has no advisory body".into(), @@ -268,7 +262,7 @@ pub(crate) fn render_result(acceptance: &BackendJobResultAcceptance) -> Result