feat(journal): execute frozen body inputs with durable delivery and erasure fences
This commit is contained in:
Generated
+3
@@ -3990,6 +3990,7 @@ dependencies = [
|
||||
"protocol",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"session-store",
|
||||
"tokio",
|
||||
"workdir",
|
||||
]
|
||||
@@ -4783,12 +4784,14 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"feature-storage",
|
||||
"job",
|
||||
"manifest",
|
||||
"memory",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"server-api",
|
||||
"session-store",
|
||||
"sha2 0.11.0",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
|
||||
@@ -9,6 +9,7 @@ license.workspace = true
|
||||
api-macros = { workspace = true, features = ["reqwest", "axum"] }
|
||||
manifest.workspace = true
|
||||
protocol.workspace = true
|
||||
session-store.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
workdir.workspace = true
|
||||
|
||||
|
||||
@@ -836,6 +836,34 @@ pub struct SessionMaterialPinResponse {
|
||||
pub pin_id: String,
|
||||
}
|
||||
|
||||
/// One body-bearing immutable slice. The byte budget includes slice metadata,
|
||||
/// but not the enclosing availability enum or the caller's durable Job envelope.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SessionRecordingCaptureRequest {
|
||||
pub workspace_id: String,
|
||||
pub source: SessionPublicSource,
|
||||
pub expected_session_id: String,
|
||||
pub start: session_store::capture::RecordingPosition,
|
||||
pub max_entries: usize,
|
||||
pub max_serialized_bytes: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "availability", rename_all = "snake_case")]
|
||||
pub enum SessionRecordingCaptureAvailability {
|
||||
NotMaterialized {
|
||||
session_id: String,
|
||||
},
|
||||
Captured {
|
||||
slice: session_store::capture::RecordingCaptureSlice,
|
||||
},
|
||||
Unavailable {
|
||||
reason: SessionPublicUnavailableReason,
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SessionMaterialCatalogRequest {
|
||||
@@ -1246,6 +1274,16 @@ pub trait RuntimeApi {
|
||||
#[body] request: SessionMaterialPinRequest,
|
||||
) -> Result<SessionMaterialPinResponse, RuntimeApiError>;
|
||||
|
||||
#[post(
|
||||
"/v1/session-public/recording-capture",
|
||||
status = 200,
|
||||
error_status = 400
|
||||
)]
|
||||
async fn session_recording_capture(
|
||||
&self,
|
||||
#[body] request: SessionRecordingCaptureRequest,
|
||||
) -> Result<SessionRecordingCaptureAvailability, RuntimeApiError>;
|
||||
|
||||
#[post(
|
||||
"/v1/session-public/material/catalog",
|
||||
status = 200,
|
||||
@@ -1730,6 +1768,13 @@ mod tests {
|
||||
Err(test_error(501))
|
||||
}
|
||||
|
||||
async fn session_recording_capture(
|
||||
&self,
|
||||
_request: SessionRecordingCaptureRequest,
|
||||
) -> Result<SessionRecordingCaptureAvailability, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
|
||||
async fn session_material_catalog(
|
||||
&self,
|
||||
_request: SessionMaterialCatalogRequest,
|
||||
|
||||
@@ -13,6 +13,14 @@
|
||||
//! observation share only this domain; they do not enable or modify each other's Features.
|
||||
//! The provider owns authorization to recapture a current Worker. A capture itself proves neither
|
||||
//! completion nor approval and grants no business-mutation authority.
|
||||
//!
|
||||
//! Recording Hosts use the common committed replay through the guarded APIs below.
|
||||
//! [`recording_catalog_worker_guarded`] and [`recording_catalog_fs_guarded`] expose
|
||||
//! descriptors only to an admission callback under the caller's existing publication/
|
||||
//! erasure guard and persisted active selector, without allocating a full body DTO.
|
||||
//! [`capture_recording_slice_worker_guarded`] and [`capture_recording_slice_fs_guarded`]
|
||||
//! separately construct bounded body-bearing inputs. Admission descriptors are not
|
||||
//! sufficient Job inputs; saved recording DTOs are read/sliced without live storage.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -21,12 +29,14 @@ use crate::{
|
||||
LoggedSessionHistoryOrigin as HistoryOrigin,
|
||||
};
|
||||
mod body;
|
||||
mod recording;
|
||||
use agen::{HistoryEntry, Item, Role};
|
||||
pub use body::*;
|
||||
use protocol::{
|
||||
SessionContentPart, SessionEntryProvenance, SessionMessageRole, SessionSnapshot,
|
||||
SessionSnapshotEntryData,
|
||||
};
|
||||
pub use recording::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const DEFAULT_SEARCH_LIMIT: usize = 20;
|
||||
@@ -50,7 +60,8 @@ impl SessionEntryRef {
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
let suffix = value.strip_prefix('E')?;
|
||||
if suffix.is_empty()
|
||||
|| suffix.len() > 64
|
||||
// Legacy Host boundary identities contain both Session and Segment UUIDs.
|
||||
|| suffix.len() > 256
|
||||
|| !suffix
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
@@ -822,7 +833,7 @@ impl SessionCapture {
|
||||
}
|
||||
}
|
||||
|
||||
fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance {
|
||||
pub(crate) fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance {
|
||||
match origin {
|
||||
HistoryOrigin::HumanInput { .. } | HistoryOrigin::LocalHumanInput => {
|
||||
SessionEntryProvenance::HumanInput
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
use super::*;
|
||||
use serde::de::Error as _;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Write;
|
||||
|
||||
pub const CAPTURE_VERSION: u32 = 1;
|
||||
@@ -81,6 +81,17 @@ impl CaptureContentPart {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum CaptureBody {
|
||||
/// Canonical recording bytes retain the historical Journal descriptor contract.
|
||||
/// A nonzero offset or short text is an explicit fragment, never a full body.
|
||||
/// History entries retain their original raw segment-history row, including
|
||||
/// hidden rows in the count. A typed Host boundary has no history row: its
|
||||
/// `source.entry_index` is the next-history gap at the original boundary.
|
||||
CanonicalPublic {
|
||||
descriptor: crate::public_material::PublicMaterialDescriptor,
|
||||
byte_offset: u64,
|
||||
text: String,
|
||||
boundary: Option<crate::public_material::PublicRunBoundary>,
|
||||
},
|
||||
Message {
|
||||
role: CaptureRole,
|
||||
content: Vec<CaptureContentPart>,
|
||||
@@ -175,6 +186,7 @@ impl CaptureBody {
|
||||
|
||||
pub(super) fn to_item(&self) -> Item {
|
||||
match self {
|
||||
Self::CanonicalPublic { text, .. } => Item::system_message(text.clone()),
|
||||
Self::Message { role, content } => Item::Message {
|
||||
id: None,
|
||||
status: None,
|
||||
@@ -214,6 +226,7 @@ impl CaptureBody {
|
||||
|
||||
fn fields(&self) -> Vec<(RawField, &str)> {
|
||||
match self {
|
||||
Self::CanonicalPublic { text, .. } => vec![(RawField::CanonicalPublic, text)],
|
||||
Self::Message { content, .. } => content
|
||||
.iter()
|
||||
.enumerate()
|
||||
@@ -235,6 +248,16 @@ impl CaptureBody {
|
||||
|
||||
fn metadata(&self, field: RawField) -> CaptureBodyMetadata {
|
||||
match self {
|
||||
Self::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
boundary,
|
||||
..
|
||||
} => CaptureBodyMetadata::CanonicalPublic {
|
||||
descriptor: descriptor.clone(),
|
||||
byte_offset: *byte_offset,
|
||||
boundary: *boundary,
|
||||
},
|
||||
Self::Message { role, content } => CaptureBodyMetadata::Message {
|
||||
role: *role,
|
||||
refusal: matches!(field, RawField::MessagePart(i) if matches!(content.get(i), Some(CaptureContentPart::Refusal { .. }))),
|
||||
@@ -264,6 +287,9 @@ impl CaptureBody {
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CaptureSource {
|
||||
pub segment_id: String,
|
||||
/// Original raw segment-history position, not captured/public view order.
|
||||
/// For `CanonicalPublic` Host boundaries only, this is the next-history gap;
|
||||
/// several typed Host events may share that gap with the following history row.
|
||||
pub entry_index: u64,
|
||||
pub provenance: SessionEntryProvenance,
|
||||
/// Full original origin when supplied by the committed-history adapter.
|
||||
@@ -375,7 +401,7 @@ impl Write for ByteCounter {
|
||||
}
|
||||
}
|
||||
|
||||
fn json_size<T: Serialize>(value: &T, limit: usize) -> Result<usize, CaptureError> {
|
||||
pub(super) fn json_size<T: Serialize>(value: &T, limit: usize) -> Result<usize, CaptureError> {
|
||||
let mut counter = ByteCounter { bytes: 0, limit };
|
||||
serde_json::to_writer(&mut counter, value).map_err(|_| CaptureError::ResourceLimit)?;
|
||||
Ok(counter.bytes)
|
||||
@@ -403,6 +429,7 @@ impl CaptureDto {
|
||||
identity(&self.segment_id, limits)?;
|
||||
let mut ids = HashSet::new();
|
||||
let mut sources = HashSet::new();
|
||||
let mut recording_source_floor = HashMap::<&str, u64>::new();
|
||||
let mut previous = None;
|
||||
for entry in &self.entries {
|
||||
identity(&entry.entry_id, limits)?;
|
||||
@@ -419,7 +446,16 @@ impl CaptureDto {
|
||||
}
|
||||
previous = Some(entry.order);
|
||||
identity(&entry.source.segment_id, limits)?;
|
||||
if !sources.insert((&entry.source.segment_id, entry.source.entry_index)) {
|
||||
let host_boundary = matches!(
|
||||
&entry.body,
|
||||
CaptureBody::CanonicalPublic {
|
||||
boundary: Some(_),
|
||||
..
|
||||
}
|
||||
);
|
||||
if !host_boundary
|
||||
&& !sources.insert((&entry.source.segment_id, entry.source.entry_index))
|
||||
{
|
||||
return Err(CaptureError::Invalid("duplicate source position"));
|
||||
}
|
||||
if !entry.policy.any() {
|
||||
@@ -443,6 +479,79 @@ impl CaptureDto {
|
||||
}
|
||||
}
|
||||
match &entry.body {
|
||||
CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
boundary,
|
||||
} => {
|
||||
if descriptor.identity.entry_ref != format!("E{}", entry.entry_id)
|
||||
|| descriptor.identity.segment_id.to_string() != entry.source.segment_id
|
||||
|| descriptor.canonical_order != entry.order
|
||||
|| text.is_empty()
|
||||
|| descriptor.total_bytes == 0
|
||||
|| byte_offset
|
||||
.checked_add(text.len() as u64)
|
||||
.is_none_or(|end| end > descriptor.total_bytes)
|
||||
|| descriptor.sha256.len() != 64
|
||||
|| !descriptor
|
||||
.sha256
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
|| !entry.policy.recording_material
|
||||
|| entry.policy.memory_evidence
|
||||
|| entry.policy.observation_readable
|
||||
{
|
||||
return Err(CaptureError::Invalid("canonical descriptor"));
|
||||
}
|
||||
match (&descriptor.origin, &entry.source.history_origin, boundary) {
|
||||
(
|
||||
crate::public_material::PublicMaterialOrigin::History { origin },
|
||||
Some(stored),
|
||||
None,
|
||||
) if origin == stored => {}
|
||||
(
|
||||
crate::public_material::PublicMaterialOrigin::HostRunBoundary,
|
||||
None,
|
||||
Some(_),
|
||||
) => {}
|
||||
_ => return Err(CaptureError::Invalid("canonical origin")),
|
||||
}
|
||||
// A partial DTO cannot prove the hidden history extent, but can
|
||||
// reject aliases and reversals of original per-Segment positions.
|
||||
// Host events anchor a gap, not a fabricated history row.
|
||||
let floor = recording_source_floor
|
||||
.entry(&entry.source.segment_id)
|
||||
.or_default();
|
||||
if entry.source.entry_index < *floor {
|
||||
return Err(CaptureError::Invalid("recording source position order"));
|
||||
}
|
||||
*floor = if boundary.is_some() {
|
||||
entry.source.entry_index
|
||||
} else {
|
||||
entry
|
||||
.source
|
||||
.entry_index
|
||||
.checked_add(1)
|
||||
.ok_or(CaptureError::Invalid("source position overflow"))?
|
||||
};
|
||||
if *byte_offset == 0 && text.len() as u64 == descriptor.total_bytes {
|
||||
let digest: String = Sha256::digest(text.as_bytes())
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect();
|
||||
if digest != descriptor.sha256 {
|
||||
return Err(CaptureError::Invalid("canonical digest"));
|
||||
}
|
||||
if let Some(boundary) = boundary {
|
||||
let expected =
|
||||
serde_json::json!({"kind":"run_boundary", "boundary":boundary});
|
||||
if serde_json::from_str::<serde_json::Value>(text)? != expected {
|
||||
return Err(CaptureError::Invalid("run boundary body"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
CaptureBody::Message { content, .. } if content.is_empty() => {
|
||||
return Err(CaptureError::Invalid("empty message parts"));
|
||||
}
|
||||
@@ -493,6 +602,7 @@ pub struct ScopedSessionEntryRef {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RawField {
|
||||
CanonicalPublic,
|
||||
MessagePart(usize),
|
||||
PublicSystem,
|
||||
ToolArguments,
|
||||
@@ -515,6 +625,11 @@ pub struct RawRead {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum CaptureBodyMetadata {
|
||||
CanonicalPublic {
|
||||
descriptor: crate::public_material::PublicMaterialDescriptor,
|
||||
byte_offset: u64,
|
||||
boundary: Option<crate::public_material::PublicRunBoundary>,
|
||||
},
|
||||
Message {
|
||||
role: CaptureRole,
|
||||
refusal: bool,
|
||||
|
||||
@@ -0,0 +1,425 @@
|
||||
//! Host capture and pure canonical slicing. No Job state or live-reader fallback lives here.
|
||||
|
||||
use super::*;
|
||||
use crate::committed_replay::CommittedPublicReplay;
|
||||
use crate::public_material::{
|
||||
PublicMaterialDescriptor, PublicMaterialError, PublicMaterialLimits, PublicMaterialUse,
|
||||
};
|
||||
use crate::retention::SessionRetentionStore;
|
||||
use crate::{SegmentId, SessionId};
|
||||
use std::path::Path;
|
||||
|
||||
/// Continuation in canonical Journal order, not an observation/Memory view cursor.
|
||||
/// A Host must reconcile the descriptor at this position with its saved checkpoint.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RecordingPosition {
|
||||
pub canonical_order: u64,
|
||||
pub byte_offset: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RecordingCaptureSlice {
|
||||
pub capture: CaptureDto,
|
||||
pub next: Option<RecordingPosition>,
|
||||
}
|
||||
|
||||
/// Inspect recording admission descriptors without constructing a body-bearing DTO.
|
||||
///
|
||||
/// The Host must authorize the source and hold the **same** Session publication/erasure
|
||||
/// guard from before reading its persisted active pointer until this function returns
|
||||
/// (including the callback). Pass that pointer's Segment, not a filename/mtime-derived
|
||||
/// selector. This function verifies the guard and stored Session identities but does
|
||||
/// not authorize the caller or establish that the supplied guard fences this root.
|
||||
/// No lock is recursively acquired, no active pointer is reread, and no replay handle
|
||||
/// or body-read capability is exposed to `inspect`.
|
||||
///
|
||||
/// The common committed recording replay fixes per-file ends before decoding. It
|
||||
/// retains only descriptors/locations, decoding one record at a time and streaming
|
||||
/// canonical bodies into their digests. `replay_limits` bound record allocation,
|
||||
/// aggregate storage extent, entries and Segments; there is no full-capture text/JSON
|
||||
/// allocation or `CaptureLimits` ceiling. The callback receives only descriptors in
|
||||
/// canonical order and may return a Host-specific mapping (including a `Result`). It
|
||||
/// runs once on success and not at all on replay/identity/resource errors.
|
||||
///
|
||||
/// This catalog is for Host admission, **not** a descriptor-only Job input. Construct
|
||||
/// the Job's bounded body input with the guarded slice API at its own authorized
|
||||
/// boundary; after saving it, execute only that immutable value. Neither descriptors
|
||||
/// nor a completed callback authorize delayed adoption after erasure.
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use std::path::Path;
|
||||
/// # use session_store::{SessionId, SegmentId};
|
||||
/// # use session_store::retention::SessionRetentionGuard;
|
||||
/// # use session_store::public_material::PublicMaterialError;
|
||||
/// # fn admit(root: &Path, session: SessionId, active: SegmentId,
|
||||
/// # guard: &SessionRetentionGuard) -> Result<Vec<u64>, PublicMaterialError> {
|
||||
/// // `active` was resolved from persisted Host metadata while holding `guard`.
|
||||
/// session_store::capture::recording_catalog_worker_guarded(
|
||||
/// root, session, active, Default::default(), guard,
|
||||
/// |descriptors| descriptors.iter().map(|d| d.total_bytes).collect(),
|
||||
/// )
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn recording_catalog_worker_guarded<T>(
|
||||
session_root: &Path,
|
||||
expected_session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T,
|
||||
) -> Result<T, PublicMaterialError> {
|
||||
if guard.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
let replay = CommittedPublicReplay::open_worker_guarded(
|
||||
session_root,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
None,
|
||||
PublicMaterialUse::Recording,
|
||||
)?;
|
||||
if replay.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
Ok(inspect(&replay.into_recording_descriptors()))
|
||||
}
|
||||
|
||||
/// FsStore-layout counterpart of [`recording_catalog_worker_guarded`].
|
||||
///
|
||||
/// `sessions_root` is the directory containing per-Session directories. Invoke this
|
||||
/// inside the Host's `with_recording_boundary` callback, passing **its existing guard
|
||||
/// and persisted active Segment**. That same external retention control store must
|
||||
/// fence publishers, pointer changes, archive moves and erasure. Do not reacquire the
|
||||
/// guard or use a separately opened legacy public reader inside the boundary.
|
||||
/// All descriptor-only, allocation and callback guarantees of the Worker variant
|
||||
/// apply; this function does not construct an unbounded transient `CaptureDto`.
|
||||
pub fn recording_catalog_fs_guarded<T>(
|
||||
sessions_root: &Path,
|
||||
session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T,
|
||||
) -> Result<T, PublicMaterialError> {
|
||||
if guard.session_id() != session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
let replay = CommittedPublicReplay::open_fs_guarded(
|
||||
sessions_root,
|
||||
session,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
)?;
|
||||
Ok(inspect(&replay.into_recording_descriptors()))
|
||||
}
|
||||
|
||||
/// Capture the selected original lineage, including typed Host run boundaries.
|
||||
/// The caller must hold the Session publication/erasure guard while selecting the
|
||||
/// active pointer and through this call. Runtime publishers use this same lock
|
||||
/// for appends and Segment creation. No lock is recursively acquired here.
|
||||
pub fn capture_recording_worker_guarded(
|
||||
session_root: &Path,
|
||||
expected_session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
capture_limits: CaptureLimits,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
) -> Result<CaptureDto, PublicMaterialError> {
|
||||
if guard.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
let replay = CommittedPublicReplay::open_worker_guarded(
|
||||
session_root,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
None,
|
||||
PublicMaterialUse::Recording,
|
||||
)?;
|
||||
if replay.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
replay.into_recording_dto(capture_limits)
|
||||
}
|
||||
|
||||
/// Standalone convenience entry; Hosts resolving a mutable active pointer should
|
||||
/// instead hold the fence while resolving it and call the guarded entry.
|
||||
pub fn capture_recording_worker(
|
||||
session_root: &Path,
|
||||
expected_session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
retention: &SessionRetentionStore,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
capture_limits: CaptureLimits,
|
||||
) -> Result<CaptureDto, PublicMaterialError> {
|
||||
let guard = retention
|
||||
.lock_live(expected_session)
|
||||
.map_err(|error| match error {
|
||||
crate::StoreError::SessionErased(_) => PublicMaterialError::Erased,
|
||||
_ => PublicMaterialError::Unavailable,
|
||||
})?;
|
||||
capture_recording_worker_guarded(
|
||||
session_root,
|
||||
expected_session,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
capture_limits,
|
||||
&guard,
|
||||
)
|
||||
}
|
||||
|
||||
/// Bounded construction for one Job input: descriptor replay is bounded separately;
|
||||
/// only at most the requested body budget is materialized from the fixed files.
|
||||
pub fn capture_recording_slice_worker_guarded(
|
||||
session_root: &Path,
|
||||
expected_session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
start: RecordingPosition,
|
||||
max_entries: usize,
|
||||
max_serialized_bytes: usize,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
) -> Result<RecordingCaptureSlice, PublicMaterialError> {
|
||||
if guard.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
let replay = CommittedPublicReplay::open_worker_guarded(
|
||||
session_root,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
None,
|
||||
PublicMaterialUse::Recording,
|
||||
)?;
|
||||
if replay.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
replay.recording_slice(start, max_entries, max_serialized_bytes)
|
||||
}
|
||||
|
||||
/// The standalone FsStore layout uses the same committed replay and fixed values.
|
||||
pub fn capture_recording_fs_guarded(
|
||||
sessions_root: &Path,
|
||||
session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
capture_limits: CaptureLimits,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
) -> Result<CaptureDto, PublicMaterialError> {
|
||||
if guard.session_id() != session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)?
|
||||
.into_recording_dto(capture_limits)
|
||||
}
|
||||
|
||||
/// Bounded standalone capture; the caller holds its FsStore retention fence.
|
||||
pub fn capture_recording_slice_fs_guarded(
|
||||
sessions_root: &Path,
|
||||
session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
start: RecordingPosition,
|
||||
max_entries: usize,
|
||||
max_serialized_bytes: usize,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
) -> Result<RecordingCaptureSlice, PublicMaterialError> {
|
||||
if guard.session_id() != session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)?
|
||||
.recording_slice(start, max_entries, max_serialized_bytes)
|
||||
}
|
||||
|
||||
impl CaptureDto {
|
||||
/// Select the oldest canonical prefix fitting both budgets, including JSON
|
||||
/// escaping, metadata and continuation. Only immutable owned bytes are read.
|
||||
/// A later Job can recapture at `next`; never combine live pages into this value.
|
||||
pub fn recording_slice(
|
||||
&self,
|
||||
start: RecordingPosition,
|
||||
max_entries: usize,
|
||||
max_serialized_bytes: usize,
|
||||
) -> Result<RecordingCaptureSlice, CaptureError> {
|
||||
if max_entries == 0 {
|
||||
return Err(CaptureError::BudgetTooSmall);
|
||||
}
|
||||
if start.canonical_order > self.view_entry_count {
|
||||
return Err(CaptureError::InvalidRange);
|
||||
}
|
||||
let mut result = RecordingCaptureSlice {
|
||||
capture: CaptureDto {
|
||||
version: self.version,
|
||||
session_id: self.session_id.clone(),
|
||||
segment_id: self.segment_id.clone(),
|
||||
view_entry_count: self.view_entry_count,
|
||||
entries: Vec::new(),
|
||||
},
|
||||
next: (start.canonical_order < self.view_entry_count).then_some(start),
|
||||
};
|
||||
if body::json_size(&result, max_serialized_bytes).is_err() {
|
||||
return Err(CaptureError::BudgetTooSmall);
|
||||
}
|
||||
if start.canonical_order == self.view_entry_count {
|
||||
if start.byte_offset != 0 {
|
||||
return Err(CaptureError::InvalidRange);
|
||||
}
|
||||
result.capture.validate(CaptureLimits {
|
||||
max_bytes: max_serialized_bytes,
|
||||
max_text_bytes: max_serialized_bytes,
|
||||
max_entries,
|
||||
..Default::default()
|
||||
})?;
|
||||
return Ok(result);
|
||||
}
|
||||
let first = self
|
||||
.entries
|
||||
.iter()
|
||||
.position(|entry| entry.order == start.canonical_order)
|
||||
.ok_or(CaptureError::InvalidRange)?;
|
||||
let mut expected_order = start.canonical_order;
|
||||
for entry in self.entries.iter().skip(first).take(max_entries) {
|
||||
if entry.order != expected_order {
|
||||
return Err(CaptureError::InvalidRange);
|
||||
}
|
||||
let CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
boundary,
|
||||
} = &entry.body
|
||||
else {
|
||||
return Err(CaptureError::Invalid("not a recording capture"));
|
||||
};
|
||||
if !entry.policy.recording_material {
|
||||
return Err(CaptureError::PolicyDenied);
|
||||
}
|
||||
let offset = if entry.order == start.canonical_order {
|
||||
start.byte_offset
|
||||
} else {
|
||||
*byte_offset
|
||||
};
|
||||
let local = offset
|
||||
.checked_sub(*byte_offset)
|
||||
.and_then(|o| usize::try_from(o).ok())
|
||||
.ok_or(CaptureError::InvalidRange)?;
|
||||
if local > text.len()
|
||||
|| !text.is_char_boundary(local)
|
||||
|| offset >= descriptor.total_bytes
|
||||
{
|
||||
return Err(CaptureError::InvalidRange);
|
||||
}
|
||||
let item = CaptureEntryDto {
|
||||
entry_id: entry.entry_id.clone(),
|
||||
order: entry.order,
|
||||
source: entry.source.clone(),
|
||||
policy: entry.policy,
|
||||
body: CaptureBody::CanonicalPublic {
|
||||
descriptor: descriptor.clone(),
|
||||
byte_offset: offset,
|
||||
text: String::new(),
|
||||
boundary: *boundary,
|
||||
},
|
||||
};
|
||||
result.capture.entries.push(item);
|
||||
// At most budget bytes are considered before allocating a trial fragment.
|
||||
let mut low = 0;
|
||||
let mut high = text.len().saturating_sub(local).min(max_serialized_bytes);
|
||||
let prior_next = result.next;
|
||||
while low < high {
|
||||
let mid = low + (high - low).div_ceil(2);
|
||||
let mut end = local + mid;
|
||||
while !text.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
if let CaptureBody::CanonicalPublic { text: fragment, .. } =
|
||||
&mut result.capture.entries.last_mut().unwrap().body
|
||||
{
|
||||
*fragment = text[local..end].to_owned();
|
||||
}
|
||||
let next_offset = offset + (end - local) as u64;
|
||||
result.next = if next_offset < descriptor.total_bytes {
|
||||
Some(RecordingPosition {
|
||||
canonical_order: entry.order,
|
||||
byte_offset: next_offset,
|
||||
})
|
||||
} else {
|
||||
(entry.order + 1 < self.view_entry_count).then_some(RecordingPosition {
|
||||
canonical_order: entry.order + 1,
|
||||
byte_offset: 0,
|
||||
})
|
||||
};
|
||||
if body::json_size(&result, max_serialized_bytes).is_ok() {
|
||||
low = mid;
|
||||
} else {
|
||||
high = mid - 1;
|
||||
}
|
||||
}
|
||||
let mut end = local + low;
|
||||
while !text.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
if end == local {
|
||||
result.capture.entries.pop();
|
||||
result.next = prior_next;
|
||||
if result.capture.entries.is_empty() {
|
||||
return Err(CaptureError::BudgetTooSmall);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if let CaptureBody::CanonicalPublic { text: fragment, .. } =
|
||||
&mut result.capture.entries.last_mut().unwrap().body
|
||||
{
|
||||
*fragment = text[local..end].to_owned();
|
||||
}
|
||||
let next_offset = offset + (end - local) as u64;
|
||||
result.next = if next_offset < descriptor.total_bytes {
|
||||
Some(RecordingPosition {
|
||||
canonical_order: entry.order,
|
||||
byte_offset: next_offset,
|
||||
})
|
||||
} else {
|
||||
(entry.order + 1 < self.view_entry_count).then_some(RecordingPosition {
|
||||
canonical_order: entry.order + 1,
|
||||
byte_offset: 0,
|
||||
})
|
||||
};
|
||||
if next_offset < descriptor.total_bytes {
|
||||
break;
|
||||
}
|
||||
expected_order = entry.order + 1;
|
||||
}
|
||||
body::json_size(&result, max_serialized_bytes).map_err(|_| CaptureError::BudgetTooSmall)?;
|
||||
result.capture.validate(CaptureLimits {
|
||||
max_bytes: max_serialized_bytes,
|
||||
max_text_bytes: max_serialized_bytes,
|
||||
max_entries,
|
||||
..Default::default()
|
||||
})?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Resolve canonical text already in the value. No original storage is consulted.
|
||||
pub fn recording_text(
|
||||
&self,
|
||||
order: u64,
|
||||
) -> Result<(&crate::public_material::PublicMaterialDescriptor, u64, &str), CaptureError> {
|
||||
let entry = self
|
||||
.entries
|
||||
.iter()
|
||||
.find(|entry| entry.order == order)
|
||||
.ok_or(CaptureError::UnknownReference)?;
|
||||
if !entry.policy.recording_material {
|
||||
return Err(CaptureError::PolicyDenied);
|
||||
}
|
||||
match &entry.body {
|
||||
CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
..
|
||||
} => Ok((descriptor, *byte_offset, text)),
|
||||
_ => Err(CaptureError::Invalid("not canonical recording text")),
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -39,6 +39,7 @@ pub(crate) fn read_without_atime_bounded(path: &Path, max_bytes: u64) -> io::Res
|
||||
}
|
||||
|
||||
pub mod capture;
|
||||
mod committed_replay;
|
||||
pub mod event_trace;
|
||||
pub mod fs_store;
|
||||
pub mod history;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -46,6 +46,13 @@ pub struct SessionRetentionStore {
|
||||
/// Do not recursively acquire the same Session lock inside a guarded callback.
|
||||
pub struct SessionRetentionGuard {
|
||||
_file: File,
|
||||
session_id: SessionId,
|
||||
}
|
||||
|
||||
impl SessionRetentionGuard {
|
||||
pub fn session_id(&self) -> SessionId {
|
||||
self.session_id
|
||||
}
|
||||
}
|
||||
|
||||
impl SessionRetentionStore {
|
||||
@@ -71,7 +78,10 @@ impl SessionRetentionStore {
|
||||
}
|
||||
let file = options.open(self.root.join(format!("{session}.lock")))?;
|
||||
FileExt::lock_exclusive(&file)?;
|
||||
Ok(SessionRetentionGuard { _file: file })
|
||||
Ok(SessionRetentionGuard {
|
||||
_file: file,
|
||||
session_id: session,
|
||||
})
|
||||
}
|
||||
|
||||
fn read(&self, session: SessionId) -> Result<RetentionState, StoreError> {
|
||||
|
||||
@@ -25,7 +25,46 @@ use thiserror::Error;
|
||||
#[path = "job_store/legacy_migrations.rs"]
|
||||
mod legacy_migrations;
|
||||
|
||||
const SCHEMA_VERSION: i64 = 4;
|
||||
const SCHEMA_VERSION: i64 = 5;
|
||||
|
||||
/// Availability of the immutable input body, independent of execution outcome.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobInputDisposition {
|
||||
Available,
|
||||
Reclaimed,
|
||||
Erased,
|
||||
Invalidated,
|
||||
}
|
||||
|
||||
impl JobInputDisposition {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Available => "available",
|
||||
Self::Reclaimed => "reclaimed",
|
||||
Self::Erased => "erased",
|
||||
Self::Invalidated => "invalidated",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum JobInputRetirementReason {
|
||||
Completed,
|
||||
ExplicitErasure,
|
||||
LegacyInvalidation,
|
||||
}
|
||||
|
||||
impl JobInputRetirementReason {
|
||||
pub fn disposition(self) -> JobInputDisposition {
|
||||
match self {
|
||||
Self::Completed => JobInputDisposition::Reclaimed,
|
||||
Self::ExplicitErasure => JobInputDisposition::Erased,
|
||||
Self::LegacyInvalidation => JobInputDisposition::Invalidated,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -67,7 +106,11 @@ impl JobAttemptState {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct JobSnapshot {
|
||||
/// Receipt metadata. After retirement `input` is null; use the verified loader to execute.
|
||||
pub request: JobRequest,
|
||||
pub input_digest: String,
|
||||
pub intent_fingerprint: String,
|
||||
pub input_disposition: JobInputDisposition,
|
||||
/// Durable Host-issued domain delegation, not model input or Profile identity.
|
||||
pub domain_grant: Option<Value>,
|
||||
pub state: JobState,
|
||||
@@ -77,14 +120,27 @@ pub struct JobSnapshot {
|
||||
}
|
||||
|
||||
impl JobSnapshot {
|
||||
/// Load only the original available body, checking both content and immutable intent.
|
||||
pub fn load_executable_request(&self) -> Result<&JobRequest, JobStoreError> {
|
||||
if self.input_disposition != JobInputDisposition::Available {
|
||||
return Err(JobStoreError::InvalidTransition("Job input is retired"));
|
||||
}
|
||||
self.request.validate()?;
|
||||
if self.request.input_digest()? != self.input_digest
|
||||
|| self.request.fingerprint()? != self.intent_fingerprint
|
||||
{
|
||||
return Err(JobStoreError::Corrupt(
|
||||
"immutable input or intent fingerprint mismatch".into(),
|
||||
));
|
||||
}
|
||||
Ok(&self.request)
|
||||
}
|
||||
|
||||
/// Shared Feature-facing read contract; no Workspace or execution identity leaks.
|
||||
pub fn outcome(&self) -> job::JobOutcome {
|
||||
job::JobOutcome {
|
||||
job_id: self.request.job_id.clone(),
|
||||
input_digest: self
|
||||
.request
|
||||
.input_digest()
|
||||
.expect("validated immutable Job input"),
|
||||
input_digest: self.input_digest.clone(),
|
||||
attempt_id: self.attempt.attempt_id.clone(),
|
||||
attempt: self.attempt.number,
|
||||
state: match self.state {
|
||||
@@ -204,9 +260,23 @@ impl JobStore {
|
||||
tx.execute_batch(RECONCILIATION_SCHEMA)?;
|
||||
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
|
||||
}
|
||||
SCHEMA_VERSION => {}
|
||||
4 | SCHEMA_VERSION => {}
|
||||
other => return Err(JobStoreError::UnsupportedSchema(other)),
|
||||
}
|
||||
if version < 5 {
|
||||
tx.execute_batch(INPUT_RETENTION_SCHEMA)?;
|
||||
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
|
||||
if has_legacy_archive {
|
||||
tx.execute_batch("DROP TRIGGER legacy_job_archive_immutable;
|
||||
CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END;
|
||||
CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive
|
||||
WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available')
|
||||
OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;")?;
|
||||
}
|
||||
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
|
||||
}
|
||||
// An accepted result is the only evidence of success. Reserved attempts
|
||||
// were never started and remain pending; cancelled attempts never resume.
|
||||
tx.execute_batch(
|
||||
@@ -249,10 +319,7 @@ impl JobStore {
|
||||
[&request.job_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if stored != fingerprint
|
||||
|| snapshot.request != request
|
||||
|| snapshot.domain_grant != grant
|
||||
{
|
||||
if stored != fingerprint || snapshot.domain_grant != grant {
|
||||
return Err(JobStoreError::IntentConflict(request.job_id));
|
||||
}
|
||||
tx.commit()?;
|
||||
@@ -334,6 +401,7 @@ impl JobStore {
|
||||
.conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let previous = require_snapshot(&tx, job_id)?;
|
||||
previous.load_executable_request()?;
|
||||
if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed {
|
||||
return Err(JobStoreError::InvalidTransition(
|
||||
"retry requires known failure",
|
||||
@@ -354,6 +422,81 @@ impl JobStore {
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
/// Historical snapshots for one domain Session, including terminal and retired Jobs.
|
||||
pub(crate) fn list_session_snapshots(
|
||||
&self,
|
||||
session_id: &str,
|
||||
after: Option<&str>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<JobSnapshot>, JobStoreError> {
|
||||
let mut stmt = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_domain_grants g ON g.job_id=i.job_id WHERE json_extract(g.grant_json,'$.session_id')=?1 AND (?2 IS NULL OR i.job_id>?2) ORDER BY i.job_id LIMIT ?3")?;
|
||||
let ids = stmt
|
||||
.query_map(
|
||||
params![session_id, after, limit.clamp(1, 100) as i64],
|
||||
|r| r.get::<_, String>(0),
|
||||
)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
ids.into_iter()
|
||||
.map(|id| require_snapshot(&self.conn, &id))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The Host must fence domain mutation and delivery before erasure/invalidation.
|
||||
/// This atomically prevents dispatch/retry/result acceptance, without rewriting outcome.
|
||||
pub(crate) fn reclaim_input(
|
||||
&mut self,
|
||||
job_id: &str,
|
||||
expected_input_digest: &str,
|
||||
reason: JobInputRetirementReason,
|
||||
) -> Result<JobSnapshot, JobStoreError> {
|
||||
let tx = self
|
||||
.conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let snapshot = require_snapshot(&tx, job_id)?;
|
||||
if snapshot.input_digest != expected_input_digest {
|
||||
return Err(JobStoreError::DigestMismatch);
|
||||
}
|
||||
let desired = reason.disposition();
|
||||
if snapshot.input_disposition == desired {
|
||||
tx.commit()?;
|
||||
reclaim_retired_input_storage(&self.conn)?;
|
||||
return Ok(snapshot);
|
||||
}
|
||||
if reason == JobInputRetirementReason::Completed
|
||||
&& (snapshot.state != JobState::Completed
|
||||
|| snapshot.attempt.state != JobAttemptState::Completed
|
||||
|| snapshot.result.is_none())
|
||||
{
|
||||
return Err(JobStoreError::InvalidTransition(
|
||||
"reclaim requires known completed result",
|
||||
));
|
||||
}
|
||||
if snapshot.input_disposition != JobInputDisposition::Available
|
||||
&& reason != JobInputRetirementReason::ExplicitErasure
|
||||
{
|
||||
return Err(JobStoreError::InvalidTransition(
|
||||
"retired input cannot be restored or reclassified",
|
||||
));
|
||||
}
|
||||
if reason == JobInputRetirementReason::Completed {
|
||||
snapshot.load_executable_request()?;
|
||||
}
|
||||
let mut metadata = snapshot.request.clone();
|
||||
metadata.input = Value::Null;
|
||||
tx.execute(
|
||||
"UPDATE job_intents SET request_json=?2, input_disposition=?3 WHERE job_id=?1",
|
||||
params![job_id, serde_json::to_string(&metadata)?, desired.as_str()],
|
||||
)?;
|
||||
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
|
||||
if has_legacy_archive {
|
||||
tx.execute("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input',json('null')) WHERE job_id=?1", [job_id])?;
|
||||
}
|
||||
let snapshot = require_snapshot(&tx, job_id)?;
|
||||
tx.commit()?;
|
||||
reclaim_retired_input_storage(&self.conn)?;
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
pub(crate) fn get(&self, job_id: &str) -> Result<JobSnapshot, JobStoreError> {
|
||||
require_snapshot(&self.conn, job_id)
|
||||
}
|
||||
@@ -383,7 +526,7 @@ impl JobStore {
|
||||
let mut statement = self.conn.prepare(
|
||||
"SELECT i.job_id FROM job_intents i JOIN job_attempts a
|
||||
ON a.job_id = i.job_id AND a.number = i.current_attempt
|
||||
WHERE i.state = 'pending' AND a.state = 'reserved' ORDER BY i.rowid",
|
||||
WHERE i.state = 'pending' AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid",
|
||||
)?;
|
||||
let ids = statement
|
||||
.query_map([], |row| row.get::<_, String>(0))?
|
||||
@@ -406,6 +549,7 @@ impl JobStore {
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let snapshot = require_snapshot(&tx, job_id)?;
|
||||
check_attempt(&snapshot, attempt_id)?;
|
||||
snapshot.load_executable_request()?;
|
||||
if snapshot.state != JobState::Pending
|
||||
|| snapshot.attempt.state != JobAttemptState::Reserved
|
||||
{
|
||||
@@ -447,7 +591,7 @@ impl JobStore {
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let snapshot = require_snapshot(&tx, &bound.job_id)?;
|
||||
check_attempt(&snapshot, &bound.attempt_id)?;
|
||||
if snapshot.request.input_digest()? != bound.input_digest {
|
||||
if snapshot.input_digest != bound.input_digest {
|
||||
return Err(JobStoreError::DigestMismatch);
|
||||
}
|
||||
let (digest, encoded) =
|
||||
@@ -466,6 +610,7 @@ impl JobStore {
|
||||
tx.commit()?;
|
||||
return Ok(snapshot);
|
||||
}
|
||||
snapshot.load_executable_request()?;
|
||||
if snapshot.state != JobState::Pending
|
||||
|| snapshot.attempt.state != JobAttemptState::Dispatched
|
||||
{
|
||||
@@ -649,7 +794,7 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
let row = conn
|
||||
.query_row(
|
||||
"SELECT i.request_json, i.state, i.acknowledged, a.attempt_id, a.number,
|
||||
a.state, a.failure, a.result_json FROM job_intents i JOIN job_attempts a
|
||||
a.state, a.failure, a.result_json, a.input_digest, i.fingerprint, i.input_disposition FROM job_intents i JOIN job_attempts a
|
||||
ON a.job_id = i.job_id AND a.number = i.current_attempt WHERE i.job_id = ?1",
|
||||
[job_id],
|
||||
|row| {
|
||||
@@ -662,12 +807,27 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
row.get::<_, String>(5)?,
|
||||
row.get::<_, Option<String>>(6)?,
|
||||
row.get::<_, Option<String>>(7)?,
|
||||
row.get::<_, String>(8)?,
|
||||
row.get::<_, String>(9)?,
|
||||
row.get::<_, String>(10)?,
|
||||
))
|
||||
},
|
||||
)
|
||||
.optional()?;
|
||||
row.map(
|
||||
|(request, state, acknowledged, attempt_id, number, attempt_state, failure, result)| {
|
||||
|(
|
||||
request,
|
||||
state,
|
||||
acknowledged,
|
||||
attempt_id,
|
||||
number,
|
||||
attempt_state,
|
||||
failure,
|
||||
result,
|
||||
input_digest,
|
||||
intent_fingerprint,
|
||||
disposition,
|
||||
)| {
|
||||
let state = match state.as_str() {
|
||||
"pending" => JobState::Pending,
|
||||
"completed" => JobState::Completed,
|
||||
@@ -693,8 +853,17 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
)));
|
||||
}
|
||||
};
|
||||
let request_value: Value = serde_json::from_str(&request)?;
|
||||
if request_value.get("input").is_none() {
|
||||
return Err(JobStoreError::Corrupt(
|
||||
"stored Job input field is missing".into(),
|
||||
));
|
||||
}
|
||||
Ok(JobSnapshot {
|
||||
request: serde_json::from_str(&request)?,
|
||||
request: serde_json::from_value(request_value)?,
|
||||
input_digest,
|
||||
intent_fingerprint,
|
||||
input_disposition: serde_json::from_value(Value::String(disposition))?,
|
||||
domain_grant: conn
|
||||
.query_row(
|
||||
"SELECT grant_json FROM job_domain_grants WHERE job_id = ?1",
|
||||
@@ -719,6 +888,37 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
.transpose()
|
||||
}
|
||||
|
||||
// Retirement is already durable before this maintenance step. Failure must be
|
||||
// retried with the same Job/digest/reason; it must never restore input. VACUUM
|
||||
// removes historical free-page copies, including stores written before secure_delete.
|
||||
fn reclaim_retired_input_storage(conn: &Connection) -> Result<(), JobStoreError> {
|
||||
conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?;
|
||||
let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?;
|
||||
if busy != 0 {
|
||||
return Err(JobStoreError::InvalidTransition(
|
||||
"input retired; SQLite body cleanup blocked by a reader, retry retirement",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const INPUT_RETENTION_SCHEMA: &str = "
|
||||
ALTER TABLE job_intents ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available'
|
||||
CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));
|
||||
DROP TRIGGER job_intent_immutable;
|
||||
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF
|
||||
job_id, fingerprint, serialization_key, max_concurrent ON job_intents
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
|
||||
CREATE TRIGGER job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON job_intents
|
||||
WHEN (
|
||||
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
|
||||
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
|
||||
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
|
||||
AND json_type(NEW.request_json,'$.input') = 'null'
|
||||
) IS NOT TRUE
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Job input retirement'); END;
|
||||
";
|
||||
|
||||
const RECONCILIATION_SCHEMA: &str = "
|
||||
DROP INDEX job_serialization_singleflight;
|
||||
CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key)
|
||||
@@ -885,7 +1085,11 @@ mod tests {
|
||||
Some("unconfirmed"),
|
||||
)
|
||||
.unwrap();
|
||||
store.conn.execute_batch("DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
|
||||
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
|
||||
ALTER TABLE job_intents DROP COLUMN input_disposition;
|
||||
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
|
||||
DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
|
||||
drop(store);
|
||||
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
||||
assert_eq!(reopened.get("first").unwrap(), unknown);
|
||||
@@ -911,10 +1115,7 @@ mod tests {
|
||||
JobResultSubmission {
|
||||
job_id: snapshot.request.job_id.clone(),
|
||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||
input_digest: snapshot
|
||||
.request
|
||||
.input_digest()
|
||||
.expect("validated immutable Job input"),
|
||||
input_digest: snapshot.input_digest.clone(),
|
||||
result,
|
||||
}
|
||||
}
|
||||
@@ -926,6 +1127,314 @@ mod tests {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_reclaim_reopens_as_receipt_and_exact_replay_never_rehydrates() {
|
||||
let (dir, mut store) = fixture();
|
||||
let original = request("retained");
|
||||
let grant = Some(json!({"session_id":"session-a", "consumer":"fixture"}));
|
||||
let reserved = store
|
||||
.reserve_granted(original.clone(), grant.clone())
|
||||
.unwrap();
|
||||
let running = store
|
||||
.dispatch("retained", &reserved.attempt.attempt_id, 8)
|
||||
.unwrap();
|
||||
let result = submission(&running, json!({"saved":true}));
|
||||
let completed = store.accept(&result).unwrap();
|
||||
store.ack("retained").unwrap();
|
||||
let retired = store
|
||||
.reclaim_input(
|
||||
"retained",
|
||||
&completed.input_digest,
|
||||
JobInputRetirementReason::Completed,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed);
|
||||
assert!(retired.request.input.is_null());
|
||||
assert!(retired.load_executable_request().is_err());
|
||||
assert_eq!(
|
||||
retired.outcome().input_digest,
|
||||
original.input_digest().unwrap()
|
||||
);
|
||||
assert_eq!(retired.result, completed.result);
|
||||
drop(store);
|
||||
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
.reserve_granted(original.clone(), grant.clone())
|
||||
.unwrap(),
|
||||
retired
|
||||
);
|
||||
assert_eq!(reopened.accept(&result).unwrap(), retired);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.list_session_snapshots("session-a", None, 1)
|
||||
.unwrap(),
|
||||
vec![retired.clone()]
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.list_session_snapshots("session-b", None, 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.list_session_snapshots("session-a", Some("retained"), 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
let mut changed = original.clone();
|
||||
changed.input = json!({"changed":true});
|
||||
assert!(matches!(
|
||||
reopened.reserve_granted(changed, grant),
|
||||
Err(JobStoreError::IntentConflict(_))
|
||||
));
|
||||
assert!(reopened.retry("retained").is_err());
|
||||
assert!(
|
||||
reopened
|
||||
.dispatch("retained", &retired.attempt.attempt_id, 8)
|
||||
.is_err()
|
||||
);
|
||||
assert!(reopened.conn.execute("UPDATE job_intents SET request_json=?1,input_disposition='available' WHERE job_id='retained'", [serde_json::to_string(&original).unwrap()]).is_err());
|
||||
let erased = reopened
|
||||
.reclaim_input(
|
||||
"retained",
|
||||
&retired.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
|
||||
assert_eq!(erased.state, JobState::Completed);
|
||||
assert!(erased.acknowledged);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.reserve_granted(original, erased.domain_grant.clone())
|
||||
.unwrap(),
|
||||
erased
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unfinished_unknown_and_failed_inputs_are_kept_until_explicit_invalidation() {
|
||||
let (_dir, mut store) = fixture();
|
||||
for (id, state) in [
|
||||
("pending", None),
|
||||
("unknown", Some(JobAttemptState::Unknown)),
|
||||
("failed", Some(JobAttemptState::Failed)),
|
||||
] {
|
||||
let original = request(id);
|
||||
let snapshot = store.reserve(original.clone()).unwrap();
|
||||
if let Some(state) = state {
|
||||
store.dispatch(id, &snapshot.attempt.attempt_id, 8).unwrap();
|
||||
store
|
||||
.finish(
|
||||
id,
|
||||
&snapshot.attempt.attempt_id,
|
||||
state,
|
||||
Some("not complete"),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let before = store.get(id).unwrap();
|
||||
assert!(
|
||||
store
|
||||
.reclaim_input(
|
||||
id,
|
||||
&before.input_digest,
|
||||
JobInputRetirementReason::Completed
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(store.get(id).unwrap(), before);
|
||||
assert_eq!(before.load_executable_request().unwrap(), &original);
|
||||
assert!(
|
||||
store
|
||||
.reclaim_input(id, "wrong", JobInputRetirementReason::ExplicitErasure)
|
||||
.is_err()
|
||||
);
|
||||
let retired = store
|
||||
.reclaim_input(
|
||||
id,
|
||||
&before.input_digest,
|
||||
JobInputRetirementReason::LegacyInvalidation,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(retired.state, before.state);
|
||||
assert_eq!(retired.attempt, before.attempt);
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
|
||||
assert!(store.retry(id).is_err());
|
||||
assert!(store.dispatch(id, &before.attempt.attempt_id, 8).is_err());
|
||||
assert_eq!(store.reserve(original).unwrap(), retired);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn erased_dispatched_input_refuses_delayed_result_without_rewriting_unknown() {
|
||||
let (dir, mut store) = fixture();
|
||||
let running = dispatched(&mut store, request("erase-running"));
|
||||
let result = submission(&running, json!({"late":true}));
|
||||
let erased = store
|
||||
.reclaim_input(
|
||||
"erase-running",
|
||||
&running.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(store.accept(&result).is_err());
|
||||
assert_eq!(erased.state, JobState::Pending);
|
||||
drop(store);
|
||||
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
||||
let recovered = reopened.get("erase-running").unwrap();
|
||||
assert_eq!(recovered.state, JobState::Unknown);
|
||||
assert_eq!(recovered.input_disposition, JobInputDisposition::Erased);
|
||||
assert_eq!(recovered.input_digest, running.input_digest);
|
||||
assert!(reopened.accept(&result).is_err());
|
||||
assert!(reopened.retry("erase-running").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn available_input_loader_refuses_tamper_missing_body_and_envelope_changes() {
|
||||
for mutation in [
|
||||
"json_set(request_json,'$.input',json('{\"tampered\":true}'))",
|
||||
"json_remove(request_json,'$.input')",
|
||||
"json_set(request_json,'$.instruction','changed')",
|
||||
] {
|
||||
let (_dir, mut store) = fixture();
|
||||
store.reserve(request("tamper")).unwrap();
|
||||
// Simulate external corruption bypassing SQLite's normal write guard.
|
||||
store
|
||||
.conn
|
||||
.execute_batch("DROP TRIGGER job_input_retirement")
|
||||
.unwrap();
|
||||
store
|
||||
.conn
|
||||
.execute_batch(&format!("UPDATE job_intents SET request_json={mutation}"))
|
||||
.unwrap();
|
||||
match store.get("tamper") {
|
||||
Ok(snapshot) => {
|
||||
assert!(snapshot.load_executable_request().is_err());
|
||||
assert!(
|
||||
store
|
||||
.dispatch("tamper", &snapshot.attempt.attempt_id, 8)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
Err(JobStoreError::Json(_) | JobStoreError::Corrupt(_)) => {}
|
||||
other => panic!("unexpected corrupt input read: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_retirement_cleanup_failure_is_durable_and_exact_retry_finishes_without_rehydration() {
|
||||
let (dir, mut store) = fixture();
|
||||
let mut original = request("physical-erasure");
|
||||
let private_body = "private-input-retirement-original-bytes".repeat(1000);
|
||||
original.input = json!({"body":private_body});
|
||||
let reserved = store.reserve(original.clone()).unwrap();
|
||||
// A reader retaining the old SQLite snapshot prevents truncation of the old WAL.
|
||||
let reader = Connection::open(dir.path().join("jobs.sqlite3")).unwrap();
|
||||
reader
|
||||
.execute_batch("BEGIN; SELECT request_json FROM job_intents;")
|
||||
.unwrap();
|
||||
store.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
|
||||
assert!(
|
||||
store
|
||||
.reclaim_input(
|
||||
"physical-erasure",
|
||||
&reserved.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let retired = store.get("physical-erasure").unwrap();
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Erased);
|
||||
assert!(retired.load_executable_request().is_err());
|
||||
reader.execute_batch("ROLLBACK").unwrap();
|
||||
drop(reader);
|
||||
let replay = store
|
||||
.reclaim_input(
|
||||
"physical-erasure",
|
||||
&reserved.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(replay, retired);
|
||||
assert_eq!(store.reserve(original).unwrap(), retired);
|
||||
let needle = b"private-input-retirement-original-bytes";
|
||||
for entry in std::fs::read_dir(dir.path()).unwrap() {
|
||||
let path = entry.unwrap().path();
|
||||
if path.is_file() {
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
assert!(
|
||||
!bytes.windows(needle.len()).any(|bytes| bytes == needle),
|
||||
"retired body remained in {}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_original_null_input_is_not_treated_as_available_null() {
|
||||
let (_dir, mut store) = fixture();
|
||||
let mut original = request("null-original");
|
||||
original.input = Value::Null;
|
||||
let reserved = store.reserve(original).unwrap();
|
||||
assert!(reserved.load_executable_request().is_ok());
|
||||
store.conn.execute_batch("DROP TRIGGER job_input_retirement; UPDATE job_intents SET request_json=json_remove(request_json,'$.input')").unwrap();
|
||||
assert!(store.get("null-original").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_retirement_migration_preserves_receipts_and_rolls_back_failed_ddl() {
|
||||
let (dir, mut store) = fixture();
|
||||
let original = store.reserve(request("upgrade")).unwrap();
|
||||
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
|
||||
ALTER TABLE job_intents DROP COLUMN input_disposition;
|
||||
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
|
||||
PRAGMA user_version=4;
|
||||
CREATE TRIGGER job_input_retirement BEFORE UPDATE ON job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;").unwrap();
|
||||
drop(store);
|
||||
let path = dir.path().join("jobs.sqlite3");
|
||||
assert!(JobStore::open(&path).is_err());
|
||||
let conn = Connection::open(&path).unwrap();
|
||||
let version: i64 = conn
|
||||
.pragma_query_value(None, "user_version", |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(version, 4);
|
||||
let present: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info('job_intents') WHERE name='input_disposition')",[],|r|r.get(0)).unwrap();
|
||||
assert!(!present);
|
||||
conn.execute_batch("DROP TRIGGER job_input_retirement")
|
||||
.unwrap();
|
||||
drop(conn);
|
||||
let mut reopened = JobStore::open(&path).unwrap();
|
||||
assert_eq!(reopened.get("upgrade").unwrap(), original);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.get("upgrade")
|
||||
.unwrap()
|
||||
.load_executable_request()
|
||||
.unwrap(),
|
||||
&request("upgrade")
|
||||
);
|
||||
let retired = reopened
|
||||
.reclaim_input(
|
||||
"upgrade",
|
||||
&original.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(retired.state, JobState::Pending);
|
||||
assert!(
|
||||
reopened
|
||||
.conn
|
||||
.execute_batch("UPDATE job_intents SET input_disposition='available'")
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_configuration_schema_and_identity_constraints() {
|
||||
let (_dir, mut store) = fixture();
|
||||
|
||||
@@ -143,6 +143,42 @@ mod tests {
|
||||
(original, receipt, grant)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_archive_input_erasure_preserves_fingerprints_and_attempt_receipts() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("jobs.sqlite3");
|
||||
seed(&path, 2);
|
||||
let mut store = JobStore::open(&path).unwrap();
|
||||
let before: (String, String) = store
|
||||
.conn
|
||||
.query_row(
|
||||
"SELECT fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
let snapshot = store.get("done").unwrap();
|
||||
store
|
||||
.reclaim_input(
|
||||
"done",
|
||||
&snapshot.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
let (body, fingerprint, attempts): (String,String,String) = store.conn.query_row("SELECT request_json,fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||
assert!(serde_json::from_str::<Value>(&body).unwrap()["input"].is_null());
|
||||
assert_eq!((fingerprint, attempts), before);
|
||||
assert!(store.conn.execute_batch("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input','restored') WHERE job_id='done'").is_err());
|
||||
drop(store);
|
||||
let mut reopened = JobStore::open(&path).unwrap();
|
||||
let receipt = reopened
|
||||
.reserve_granted(request("done"), snapshot.domain_grant)
|
||||
.unwrap();
|
||||
assert_eq!(receipt.state, JobState::Completed);
|
||||
assert_eq!(receipt.input_disposition, JobInputDisposition::Erased);
|
||||
assert!(receipt.load_executable_request().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_versions_preserve_intent_receipts_attempts_grants_and_replay() {
|
||||
for version in [1, 2] {
|
||||
|
||||
+113
-48
@@ -26,7 +26,10 @@ use job::{JobRequest, JobResultSubmission};
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::job_store::JobStore;
|
||||
pub use crate::job_store::{JobAttempt, JobAttemptState, JobSnapshot, JobState, JobStoreError};
|
||||
pub use crate::job_store::{
|
||||
JobAttempt, JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot,
|
||||
JobState, JobStoreError,
|
||||
};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum JobServiceError {
|
||||
@@ -40,6 +43,8 @@ pub enum JobServiceError {
|
||||
Capacity,
|
||||
#[error("standalone Job cleanup failed: {0}")]
|
||||
Cleanup(String),
|
||||
#[error("standalone Job shutdown is incomplete; execution resources are still owned")]
|
||||
ShutdownIncomplete,
|
||||
}
|
||||
|
||||
pub(crate) trait JobAttemptFence: Send + Sync {
|
||||
@@ -173,6 +178,36 @@ impl StandaloneJobs {
|
||||
Ok(self.with_store(|store| store.reserve(request))?)
|
||||
}
|
||||
|
||||
/// Enumerate historical domain snapshots for a Session, including retired input receipts.
|
||||
pub fn list_session_snapshots(
|
||||
&self,
|
||||
session_id: &str,
|
||||
after: Option<&str>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<JobSnapshot>, JobServiceError> {
|
||||
self.with_store(|store| store.list_session_snapshots(session_id, after, limit))
|
||||
}
|
||||
|
||||
/// Retire body only. The caller fences domain effects/delivered snapshots before erasure.
|
||||
/// Normal reclamation also requires that execution cleanup has finished.
|
||||
pub fn reclaim_input(
|
||||
&self,
|
||||
job_id: &str,
|
||||
expected_input_digest: &str,
|
||||
reason: JobInputRetirementReason,
|
||||
) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if reason == JobInputRetirementReason::Completed
|
||||
&& executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|execution| !execution.task.is_finished())
|
||||
{
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
self.with_store(|store| store.reclaim_input(job_id, expected_input_digest, reason))
|
||||
}
|
||||
|
||||
pub fn get(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
Ok(self.with_store(|store| store.get(job_id))?)
|
||||
}
|
||||
@@ -274,6 +309,12 @@ impl StandaloneJobs {
|
||||
job_id: &str,
|
||||
client: Option<Box<dyn LlmClient>>,
|
||||
) -> Result<(), JobServiceError> {
|
||||
// Preparation owns copies of the immutable input and domain adapter too. Exclude
|
||||
// owner shutdown before creating them, not only before publishing the task.
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
let client = client.or_else(|| {
|
||||
self.model_client
|
||||
.lock()
|
||||
@@ -282,16 +323,14 @@ impl StandaloneJobs {
|
||||
.map(|c| c.clone_boxed())
|
||||
});
|
||||
let snapshot = self.get(job_id)?;
|
||||
snapshot.load_executable_request()?;
|
||||
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
let fence: Arc<dyn JobAttemptFence> = Arc::new(BoundAttemptFence {
|
||||
store: self.store.clone(),
|
||||
job_id: job_id.into(),
|
||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||
input_digest: snapshot
|
||||
.request
|
||||
.input_digest()
|
||||
.map_err(JobStoreError::from)?,
|
||||
input_digest: snapshot.input_digest.clone(),
|
||||
deadline,
|
||||
});
|
||||
let grant = match (
|
||||
@@ -311,10 +350,6 @@ impl StandaloneJobs {
|
||||
let prepared =
|
||||
worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant)
|
||||
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
executions.jobs.retain(|_, run| {
|
||||
!(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(()))))
|
||||
});
|
||||
@@ -367,10 +402,7 @@ impl StandaloneJobs {
|
||||
store: store.clone(),
|
||||
job_id: id.clone(),
|
||||
attempt_id: attempt_id.clone(),
|
||||
input_digest: snapshot
|
||||
.request
|
||||
.input_digest()
|
||||
.map_err(JobStoreError::from)?,
|
||||
input_digest: snapshot.input_digest.clone(),
|
||||
deadline,
|
||||
});
|
||||
let (cancel, mut cancellation) = watch::channel(false);
|
||||
@@ -446,23 +478,25 @@ impl StandaloneJobs {
|
||||
.clone()
|
||||
.expect("completion");
|
||||
result.map_err(JobServiceError::Cleanup)?;
|
||||
let task = {
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| run.attempt_id == attempt)
|
||||
{
|
||||
executions.jobs.remove(job_id)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(execution) = task {
|
||||
execution
|
||||
.task
|
||||
.await
|
||||
.map_err(|error| JobServiceError::Cleanup(error.to_string()))??;
|
||||
// Completion is published immediately before the task returns. Do not remove
|
||||
// its ownership record until the runtime confirms that return (and future drop).
|
||||
while self
|
||||
.executions
|
||||
.lock()
|
||||
.expect("Job executions poisoned")
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| run.attempt_id == attempt && !run.task.is_finished())
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| run.attempt_id == attempt)
|
||||
{
|
||||
executions.jobs.remove(job_id);
|
||||
}
|
||||
}
|
||||
self.get(job_id)
|
||||
@@ -476,33 +510,63 @@ impl StandaloneJobs {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
|
||||
/// Initiate owner shutdown without blocking a synchronous caller. Success confirms all
|
||||
/// local executions have returned, including request/history and Feature cleanup. An
|
||||
/// incomplete/error result retains the service resources and must retain the Host lease.
|
||||
pub(crate) fn try_shutdown(&self) -> Result<(), JobServiceError> {
|
||||
self.close();
|
||||
let tasks = {
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
std::mem::take(&mut executions.jobs)
|
||||
};
|
||||
let mut error = None;
|
||||
for execution in tasks.into_values() {
|
||||
match execution.task.await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(failure)) => {
|
||||
error = Some(failure);
|
||||
}
|
||||
Err(failure) => {
|
||||
error = Some(JobServiceError::Cleanup(failure.to_string()));
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
for execution in executions.jobs.values() {
|
||||
if !execution.task.is_finished() {
|
||||
return Err(JobServiceError::ShutdownIncomplete);
|
||||
}
|
||||
match execution.complete.borrow().as_ref() {
|
||||
Some(Ok(())) => {}
|
||||
Some(Err(error)) => return Err(JobServiceError::Cleanup(error.clone())),
|
||||
None => {
|
||||
return Err(JobServiceError::Cleanup(
|
||||
"Internal Worker ended without cleanup confirmation".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Close SQLite before the Host releases its lease. Cloned handles may
|
||||
// outlive the Host but cannot mutate or retain the live connection.
|
||||
executions.jobs.clear();
|
||||
// Retained external handles share these slots; they cannot retain a live store,
|
||||
// provider or model client after successful owner shutdown.
|
||||
self.store.lock().expect("Job store poisoned").take();
|
||||
self.domain.lock().expect("Job domain poisoned").take();
|
||||
self.model_client
|
||||
.lock()
|
||||
.expect("Job model transport poisoned")
|
||||
.take();
|
||||
error.map_or(Ok(()), Err)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
|
||||
self.close();
|
||||
let completions: Vec<_> = self
|
||||
.executions
|
||||
.lock()
|
||||
.expect("Job executions poisoned")
|
||||
.jobs
|
||||
.values()
|
||||
.map(|execution| execution.complete.clone())
|
||||
.collect();
|
||||
// Never take execution ownership out of the shared service across an await: a
|
||||
// cancelled shutdown future or a concurrent close must still see the barrier.
|
||||
for mut completion in completions {
|
||||
completion
|
||||
.wait_for(Option::is_some)
|
||||
.await
|
||||
.map_err(|error| JobServiceError::Cleanup(error.to_string()))?;
|
||||
}
|
||||
loop {
|
||||
match self.try_shutdown() {
|
||||
// Only the task's final return follows its completion notification.
|
||||
Err(JobServiceError::ShutdownIncomplete) => tokio::task::yield_now().await,
|
||||
result => return result,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -574,7 +638,8 @@ impl JobAttemptFence for BoundAttemptFence {
|
||||
if snapshot.state != JobState::Pending
|
||||
|| snapshot.attempt.state != JobAttemptState::Dispatched
|
||||
|| snapshot.attempt.attempt_id != self.attempt_id
|
||||
|| snapshot.request.input_digest().map_err(|e| e.to_string())? != self.input_digest
|
||||
|| snapshot.input_digest != self.input_digest
|
||||
|| snapshot.input_disposition != JobInputDisposition::Available
|
||||
{
|
||||
return Err("Job domain grant is not bound to the current live attempt".into());
|
||||
}
|
||||
|
||||
@@ -468,3 +468,104 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han
|
||||
assert!(matches!(operation, Err(JobServiceError::Closed)));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() {
|
||||
// Actual generic runner + SQLite; only the provider stream is scripted/blocked.
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("jobs.sqlite3");
|
||||
let jobs = StandaloneJobs::open(&path, temp.path().into()).unwrap();
|
||||
let reserved = jobs.request(request("cancel-shutdown-wait")).unwrap();
|
||||
let model = DomainTestClient {
|
||||
stall: true,
|
||||
..Default::default()
|
||||
};
|
||||
jobs.start_with_model_client(&reserved.request.job_id, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
await_model(&model).await;
|
||||
{
|
||||
let shutdown = jobs.shutdown();
|
||||
tokio::pin!(shutdown);
|
||||
assert!(futures::poll!(shutdown.as_mut()).is_pending());
|
||||
// Drop the caller's waiter while cancellation/cleanup is still pending.
|
||||
}
|
||||
assert!(matches!(
|
||||
jobs.try_shutdown(),
|
||||
Err(JobServiceError::ShutdownIncomplete)
|
||||
));
|
||||
assert!(jobs.store.lock().unwrap().is_some());
|
||||
assert!(matches!(
|
||||
jobs.request(request("after-shutdown")),
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
tokio::time::timeout(Duration::from_secs(5), jobs.shutdown())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(jobs.store.lock().unwrap().is_none());
|
||||
let reopened = StandaloneJobs::open(&path, temp.path().into()).unwrap();
|
||||
let unknown = reopened.get(&reserved.request.job_id).unwrap();
|
||||
assert_eq!(unknown.attempt.state, JobAttemptState::Unknown);
|
||||
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
|
||||
assert_eq!(unknown.request, reserved.request);
|
||||
assert_eq!(unknown.input_digest, reserved.input_digest);
|
||||
assert!(reopened.retry(&unknown.request.job_id).is_err());
|
||||
reopened.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn input_retirement_service_fences_live_attempt_and_preserves_session_receipts() {
|
||||
// Production service + SQLite + attempt/result fences, without a model or Worker fixture.
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("jobs.sqlite3");
|
||||
let jobs = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap();
|
||||
let original = request("input-retirement");
|
||||
let grant = json!({"session_id":"session-a"});
|
||||
let reserved = jobs
|
||||
.request_granted(original.clone(), grant.clone())
|
||||
.unwrap();
|
||||
jobs.with_store(|store| store.dispatch("input-retirement", &reserved.attempt.attempt_id, 8))
|
||||
.unwrap();
|
||||
let fence = BoundAttemptFence {
|
||||
store: jobs.store.clone(),
|
||||
job_id: "input-retirement".into(),
|
||||
attempt_id: reserved.attempt.attempt_id.clone(),
|
||||
input_digest: reserved.input_digest.clone(),
|
||||
deadline: tokio::time::Instant::now() + Duration::from_secs(10),
|
||||
};
|
||||
assert!(fence.ensure_live().is_ok());
|
||||
assert!(
|
||||
jobs.reclaim_input(
|
||||
"input-retirement",
|
||||
&reserved.input_digest,
|
||||
JobInputRetirementReason::Completed
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let erased = jobs
|
||||
.reclaim_input(
|
||||
"input-retirement",
|
||||
&reserved.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(fence.ensure_live().is_err());
|
||||
assert!(
|
||||
sink(&jobs, "input-retirement", &reserved.attempt.attempt_id)
|
||||
.submit(result("input-retirement", &reserved.attempt.attempt_id))
|
||||
.is_err()
|
||||
);
|
||||
assert!(jobs.start("input-retirement").await.is_err());
|
||||
assert_eq!(
|
||||
jobs.list_session_snapshots("session-a", None, 100).unwrap(),
|
||||
vec![erased.clone()]
|
||||
);
|
||||
assert!(
|
||||
jobs.list_session_snapshots("session-b", None, 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(jobs.request_granted(original, grant).unwrap(), erased);
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
@@ -54,6 +54,8 @@ pub enum SubjectError {
|
||||
Journal(#[from] subjektiv::journal::JournalError),
|
||||
#[error(transparent)]
|
||||
Storage(#[from] feature_storage::FeatureStorageError),
|
||||
#[error("local Subject Job shutdown: {0}")]
|
||||
JobShutdown(#[from] crate::jobs::JobServiceError),
|
||||
#[error("local Subject storage metadata: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("local Subject storage permission or identity is invalid: {0}")]
|
||||
@@ -338,8 +340,16 @@ impl SubjectConnection {
|
||||
}
|
||||
}
|
||||
pub fn close(&mut self) -> Result<(), SubjectError> {
|
||||
self.host.jobs.lock().expect("Subject Jobs poisoned").take();
|
||||
self.host.live.store(false, Ordering::Release);
|
||||
{
|
||||
let mut jobs = self.host.jobs.lock().expect("Subject Jobs poisoned");
|
||||
if let Some(service) = jobs.as_ref() {
|
||||
// Signal cancellation, but retain the handle and exclusive lease until
|
||||
// the generic owner confirms local request/history/resource teardown.
|
||||
service.try_shutdown()?;
|
||||
}
|
||||
jobs.take();
|
||||
}
|
||||
self.host.catalog.manager.shutdown()?;
|
||||
if let Some(file) = &self.lease {
|
||||
file.unlock()?;
|
||||
|
||||
@@ -5,9 +5,12 @@
|
||||
//! recovery authority; shutdown does not delete either original Sessions or pending requests.
|
||||
|
||||
use super::*;
|
||||
use crate::jobs::{JobAttemptState, JobSnapshot, JobState};
|
||||
use crate::jobs::{
|
||||
JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, JobState,
|
||||
};
|
||||
use session_store::capture::{CaptureDto, CaptureLimits, RecordingPosition};
|
||||
use subjektiv::api::JobAttemptBinding;
|
||||
use subjektiv::journal::recording::JournalMaterial;
|
||||
use subjektiv::journal::recording::JournalInput;
|
||||
use subjektiv::journal::transport::*;
|
||||
use subjektiv::journal::*;
|
||||
use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature};
|
||||
@@ -16,7 +19,6 @@ use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeat
|
||||
mod tests;
|
||||
|
||||
pub(super) const PURPOSE: &str = "subjektiv_journal";
|
||||
const INPUT_BUDGET: usize = 48 * 1024;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -27,6 +29,7 @@ struct LocalJournalGrant {
|
||||
session_id: String,
|
||||
journal_id: String,
|
||||
ranges: Vec<JournalSourceRange>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
origins: Vec<SourceOrigin>,
|
||||
}
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -35,11 +38,6 @@ struct SourceOrigin {
|
||||
pointer: JournalSourcePointer,
|
||||
origin: session_store::public_material::PublicMaterialOrigin,
|
||||
}
|
||||
impl LocalJournalGrant {
|
||||
fn input(&self) -> serde_json::Value {
|
||||
serde_json::json!({"selected_ranges": self.ranges, "origins": self.origins})
|
||||
}
|
||||
}
|
||||
|
||||
impl StandaloneSubjects {
|
||||
/// Explicit operator erasure, separate from ordinary Worker removal. Requires the
|
||||
@@ -84,32 +82,29 @@ impl StandaloneSubjects {
|
||||
self.state_root.clone(),
|
||||
)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
for snapshot in jobs
|
||||
.pending()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
.into_iter()
|
||||
.chain(
|
||||
jobs.unknown_granted()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?,
|
||||
)
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
let mut after = None;
|
||||
loop {
|
||||
let page = jobs
|
||||
.list_session_snapshots(session_id, after.as_deref(), 100)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
if page.is_empty() {
|
||||
break;
|
||||
}
|
||||
for snapshot in &page {
|
||||
jobs.cancel(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
jobs.reclaim_input(
|
||||
&snapshot.request.job_id,
|
||||
&snapshot.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
if snapshot.state == JobState::Completed && !snapshot.acknowledged {
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
for snapshot in jobs
|
||||
.unacknowledged_results()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
after = page.last().map(|job| job.request.job_id.clone());
|
||||
}
|
||||
session_store::FsStore::new_with_retention(
|
||||
path,
|
||||
@@ -590,17 +585,28 @@ impl LocalSubjectHost {
|
||||
self.ensure_live()?;
|
||||
let jobs = self.jobs()?;
|
||||
for saved in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if self.journal_grant(&saved).is_ok() {
|
||||
if self.journal_grant_scope(&saved).is_ok() {
|
||||
self.complete_journal_job(&jobs, &saved)?;
|
||||
}
|
||||
}
|
||||
for pending in jobs.pending().map_err(service_error)? {
|
||||
if self.journal_grant(&pending).is_ok() {
|
||||
if pending.request.purpose != PURPOSE {
|
||||
continue;
|
||||
}
|
||||
self.journal_grant_scope(&pending)?;
|
||||
if pending.request.input.get("capture").is_some() {
|
||||
self.journal_grant(&pending)?;
|
||||
return Ok(Some((jobs, pending.request.job_id)));
|
||||
}
|
||||
// Old descriptor jobs never acquire new execution authority. Their immutable
|
||||
// identity/input is retired, not upgraded in place; saved checkpoint wins.
|
||||
let replacement = self.replace_legacy_journal_job(&jobs, &pending)?;
|
||||
if let Some(id) = replacement {
|
||||
return Ok(Some((jobs, id)));
|
||||
}
|
||||
}
|
||||
for unknown in jobs.unknown_granted().map_err(service_error)? {
|
||||
let Ok(grant) = self.journal_grant(&unknown) else {
|
||||
let Ok(grant) = self.journal_grant_scope(&unknown) else {
|
||||
continue;
|
||||
};
|
||||
let context = authority(self, &grant.session_id)?;
|
||||
@@ -685,32 +691,96 @@ impl LocalSubjectHost {
|
||||
}
|
||||
}
|
||||
}
|
||||
let materials = self.select_journal_materials(&context, &journal.id)?;
|
||||
if materials.is_empty() {
|
||||
let mut retained = None;
|
||||
// Saved body snapshots always win over descriptor migration. In particular,
|
||||
// a failed continuation must not recapture its original text even if an
|
||||
// invalidated legacy grant still bounds a larger unprocessed suffix.
|
||||
for legacy_pass in [false, true] {
|
||||
let mut after_job = None;
|
||||
loop {
|
||||
let page = jobs
|
||||
.list_session_snapshots(
|
||||
&journal.source_session_id,
|
||||
after_job.as_deref(),
|
||||
100,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
if page.is_empty() {
|
||||
break;
|
||||
}
|
||||
for saved in &page {
|
||||
if saved.request.purpose != PURPOSE {
|
||||
continue;
|
||||
}
|
||||
let legacy = saved.input_disposition
|
||||
== JobInputDisposition::Invalidated
|
||||
|| (saved.input_disposition == JobInputDisposition::Available
|
||||
&& saved.request.input.get("capture").is_none());
|
||||
if legacy != legacy_pass
|
||||
|| (!legacy
|
||||
&& saved.input_disposition != JobInputDisposition::Available)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let saved_grant = self.journal_grant_scope(saved)?;
|
||||
if saved_grant.journal_id != journal.id {
|
||||
continue;
|
||||
}
|
||||
if legacy {
|
||||
if let Some(id) = self.replace_legacy_journal_job(&jobs, saved)? {
|
||||
return Ok(Some((jobs, id)));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
self.journal_grant(saved)?;
|
||||
let original = JournalInput::decode(&saved.request.input)
|
||||
.map_err(journal_error)?;
|
||||
let continuation = self.build_journal_input(
|
||||
&context,
|
||||
&journal.id,
|
||||
Some(&original.capture),
|
||||
)?;
|
||||
if !continuation.selected_ranges.is_empty() {
|
||||
retained = Some(continuation);
|
||||
break;
|
||||
}
|
||||
if saved.state == JobState::Completed && saved.acknowledged {
|
||||
// Recover the acknowledgement-before-reclamation gap without
|
||||
// turning a retained receipt into an executable empty input.
|
||||
jobs.reclaim_input(
|
||||
&saved.request.job_id,
|
||||
&saved.input_digest,
|
||||
JobInputRetirementReason::Completed,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
}
|
||||
}
|
||||
if retained.is_some() {
|
||||
break;
|
||||
}
|
||||
after_job = page.last().map(|job| job.request.job_id.clone());
|
||||
}
|
||||
if retained.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let input = match retained {
|
||||
Some(input) => input,
|
||||
None => self.build_journal_input(&context, &journal.id, None)?,
|
||||
};
|
||||
|
||||
if input.selected_ranges.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let ranges = materials
|
||||
.into_iter()
|
||||
.map(|material| material.range)
|
||||
.collect::<Vec<_>>();
|
||||
let origins = self.journal_origins(&ranges)?;
|
||||
let grant = LocalJournalGrant {
|
||||
writer_binding_id: Some(self.writer.binding_id.clone()),
|
||||
binding: self.binding.clone(),
|
||||
session_id: journal.source_session_id,
|
||||
journal_id: journal.id.clone(),
|
||||
ranges,
|
||||
origins,
|
||||
};
|
||||
let request = job::JobRequest {
|
||||
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()),
|
||||
purpose: PURPOSE.into(), input_ref: format!("subjektiv://{}/journal/{}", self.binding.subject_id, journal.id),
|
||||
input: grant.input(),
|
||||
instruction: "ReadJournalState first. Input descriptors are the frozen canonical public ranges authorized for this Job, not already read material. Read every selected source byte with bounded ReadJournalText before recording; then read initial/current position and open text in bounded pages. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
|
||||
profile: "builtin:subjektiv-journal-recording".into(),
|
||||
serialization_key: Some(format!("subjektiv-journal:{}", journal.id)),
|
||||
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
|
||||
ranges: input.selected_ranges.clone(),
|
||||
origins: vec![],
|
||||
};
|
||||
let request = self.journal_request(&journal.id, &input)?;
|
||||
let saved = jobs
|
||||
.request_granted(
|
||||
request,
|
||||
@@ -727,6 +797,138 @@ impl LocalSubjectHost {
|
||||
}
|
||||
}
|
||||
|
||||
fn replace_legacy_journal_job(
|
||||
&self,
|
||||
jobs: &crate::jobs::StandaloneJobs,
|
||||
old: &JobSnapshot,
|
||||
) -> Result<Option<String>, SubjektivHostError> {
|
||||
let grant = self.journal_grant_scope(old)?;
|
||||
if old.input_disposition == JobInputDisposition::Available {
|
||||
old.load_executable_request()
|
||||
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
|
||||
if old.request.input
|
||||
!= serde_json::json!({"selected_ranges":grant.ranges,"origins":grant.origins})
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"invalid legacy Journal descriptor input".into(),
|
||||
));
|
||||
}
|
||||
} else if old.input_disposition != JobInputDisposition::Invalidated {
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"legacy Journal input is unavailable".into(),
|
||||
));
|
||||
}
|
||||
let context = authority(self, &grant.session_id)?;
|
||||
if let Some(active) = self
|
||||
.catalog
|
||||
.store
|
||||
.active_journal_generation(&context, &grant.journal_id)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
if active.job.job_id != old.request.job_id {
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"another generation owns legacy migration".into(),
|
||||
));
|
||||
}
|
||||
self.catalog
|
||||
.store
|
||||
.close_journal_generation(&context, &active.id, &active.job)
|
||||
.map_err(journal_error)?;
|
||||
}
|
||||
jobs.cancel(&old.request.job_id).map_err(service_error)?;
|
||||
let current = self
|
||||
.catalog
|
||||
.store
|
||||
.read_journal(&context, &grant.journal_id)
|
||||
.map_err(journal_error)?;
|
||||
let end = grant
|
||||
.ranges
|
||||
.last()
|
||||
.ok_or_else(|| SubjektivHostError::InvalidArgument("empty legacy selection".into()))?;
|
||||
let end_position = if end.bytes.end == end.source.total_bytes {
|
||||
(end.source.order + 1, 0)
|
||||
} else {
|
||||
(end.source.order, end.bytes.end)
|
||||
};
|
||||
let replacement = if (current.process_order, current.process_byte) >= end_position {
|
||||
None
|
||||
} else {
|
||||
let mut capture = self.recording_capture(&grant.session_id)?;
|
||||
capture
|
||||
.entries
|
||||
.retain(|entry| grant.ranges.iter().any(|r| r.source.order == entry.order));
|
||||
for entry in &mut capture.entries {
|
||||
let range = grant
|
||||
.ranges
|
||||
.iter()
|
||||
.find(|r| r.source.order == entry.order)
|
||||
.unwrap();
|
||||
let session_store::capture::CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
..
|
||||
} = &mut entry.body
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
if source_descriptor(&grant.session_id, descriptor) != range.source
|
||||
|| !grant.origins.iter().any(|origin| {
|
||||
origin.pointer == range.source.pointer && origin.origin == descriptor.origin
|
||||
})
|
||||
|| !text.is_char_boundary(range.bytes.start as usize)
|
||||
|| !text.is_char_boundary(range.bytes.end as usize)
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"legacy original source identity/range changed".into(),
|
||||
));
|
||||
}
|
||||
*text = text[range.bytes.start as usize..range.bytes.end as usize].to_owned();
|
||||
*byte_offset = range.bytes.start;
|
||||
}
|
||||
let input = self.build_journal_input(&context, &grant.journal_id, Some(&capture))?;
|
||||
let request = self.journal_request(&grant.journal_id, &input)?;
|
||||
let next_grant = LocalJournalGrant {
|
||||
ranges: input.selected_ranges.clone(),
|
||||
origins: vec![],
|
||||
..grant
|
||||
};
|
||||
Some(
|
||||
jobs.request_granted(
|
||||
request,
|
||||
serde_json::to_value(next_grant)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
|
||||
)
|
||||
.map_err(service_error)?
|
||||
.request
|
||||
.job_id,
|
||||
)
|
||||
};
|
||||
jobs.reclaim_input(
|
||||
&old.request.job_id,
|
||||
&old.input_digest,
|
||||
JobInputRetirementReason::LegacyInvalidation,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
Ok(replacement)
|
||||
}
|
||||
|
||||
fn journal_request(
|
||||
&self,
|
||||
journal_id: &str,
|
||||
input: &JournalInput,
|
||||
) -> Result<job::JobRequest, SubjektivHostError> {
|
||||
Ok(job::JobRequest {
|
||||
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()), purpose: PURPOSE.into(),
|
||||
input_ref: format!("subjektiv://{}/journal/{journal_id}", self.binding.subject_id),
|
||||
input: serde_json::to_value(input).map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
|
||||
instruction: "Input contains the frozen canonical public source bodies, provenance and byte ranges. Use only this saved input, never re-fetch Source. ReadJournalState first and read initial/current Journal position/open text in bounded pages if needed. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
|
||||
profile: "builtin:subjektiv-journal-recording".into(),
|
||||
serialization_key: Some(format!("subjektiv-journal:{journal_id}")),
|
||||
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
|
||||
})
|
||||
}
|
||||
|
||||
fn pin_journal_session(&self, session: &str) -> Result<(), SubjektivHostError> {
|
||||
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&self.catalog.state_root,
|
||||
@@ -869,11 +1071,16 @@ impl LocalSubjectHost {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn material_reader(
|
||||
fn with_recording_boundary<T>(
|
||||
&self,
|
||||
session: &str,
|
||||
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
|
||||
{
|
||||
read: impl FnOnce(
|
||||
&Path,
|
||||
session_store::SessionId,
|
||||
session_store::SegmentId,
|
||||
&session_store::retention::SessionRetentionGuard,
|
||||
) -> Result<T, SubjektivHostError>,
|
||||
) -> Result<T, SubjektivHostError> {
|
||||
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&self.catalog.state_root,
|
||||
&self.binding.scope_id,
|
||||
@@ -906,6 +1113,11 @@ impl LocalSubjectHost {
|
||||
.ok_or_else(|| SubjektivHostError::InvalidArgument("invalid Session storage".into()))?
|
||||
.join("worker")
|
||||
.join(&record.storage_key);
|
||||
let session_id = session
|
||||
.parse()
|
||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?;
|
||||
let retention = crate::store::session_retention(&self.catalog.state_root);
|
||||
let guard = retention.lock_live(session_id).map_err(source_error)?;
|
||||
// This is the persisted active selector, not the newest filename/mtime or a model claim.
|
||||
let metadata =
|
||||
session_store::WorkerAggregateStore::open_read_only(metadata_root, &record.storage_key)
|
||||
@@ -933,154 +1145,203 @@ impl LocalSubjectHost {
|
||||
"no attested active lineage for the retained Session".into(),
|
||||
)
|
||||
})?;
|
||||
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
|
||||
&sessions,
|
||||
session
|
||||
.parse()
|
||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
|
||||
active,
|
||||
session_store::public_material::PublicMaterialLimits {
|
||||
max_segments: 256,
|
||||
max_entries: 100_000,
|
||||
max_record_bytes: 64 * 1024 * 1024,
|
||||
},
|
||||
crate::store::session_retention(&self.catalog.state_root),
|
||||
)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
|
||||
read(&sessions, session_id, active, &guard)
|
||||
}
|
||||
|
||||
fn recording_capture(&self, session: &str) -> Result<CaptureDto, SubjektivHostError> {
|
||||
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
|
||||
session_store::capture::capture_recording_fs_guarded(
|
||||
sessions,
|
||||
session_id,
|
||||
active,
|
||||
session_store::public_material::PublicMaterialLimits::default(),
|
||||
CaptureLimits {
|
||||
max_bytes: 128 * 1024 * 1024,
|
||||
max_entries: 100_000,
|
||||
max_text_bytes: 64 * 1024 * 1024,
|
||||
..Default::default()
|
||||
},
|
||||
guard,
|
||||
)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
|
||||
})
|
||||
}
|
||||
|
||||
fn journal_sources(
|
||||
&self,
|
||||
session: &str,
|
||||
) -> Result<Vec<JournalSourceRange>, SubjektivHostError> {
|
||||
let reader = self.material_reader(session)?;
|
||||
Ok(reader
|
||||
.descriptors()
|
||||
.map(|descriptor| JournalSourceRange {
|
||||
source: source_descriptor(session, descriptor),
|
||||
bytes: JournalByteRange {
|
||||
start: 0,
|
||||
end: descriptor.total_bytes,
|
||||
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
|
||||
session_store::capture::recording_catalog_fs_guarded(
|
||||
sessions,
|
||||
session_id,
|
||||
active,
|
||||
session_store::public_material::PublicMaterialLimits::default(),
|
||||
guard,
|
||||
|descriptors| {
|
||||
descriptors
|
||||
.iter()
|
||||
.map(|descriptor| JournalSourceRange {
|
||||
source: source_descriptor(session, descriptor),
|
||||
bytes: JournalByteRange {
|
||||
start: 0,
|
||||
end: descriptor.total_bytes,
|
||||
},
|
||||
})
|
||||
.collect()
|
||||
},
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn source_reader(
|
||||
&self,
|
||||
source: &JournalSource,
|
||||
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
|
||||
{
|
||||
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&self.catalog.state_root,
|
||||
&self.binding.scope_id,
|
||||
&self.binding.subject_id,
|
||||
&self.catalog.store,
|
||||
&source.session_id,
|
||||
)?;
|
||||
// Frozen, already admitted original pointers do not follow the Worker's later active
|
||||
// Session/Segment. This reads that original lineage and still checks descriptor identity,
|
||||
// canonical order and digest below; the shared service enforces the frozen byte range.
|
||||
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
|
||||
&sessions,
|
||||
source
|
||||
.session_id
|
||||
.parse()
|
||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
|
||||
source.pointer.segment_id,
|
||||
session_store::public_material::PublicMaterialLimits {
|
||||
max_segments: 256,
|
||||
max_entries: 100_000,
|
||||
max_record_bytes: 64 * 1024 * 1024,
|
||||
},
|
||||
crate::store::session_retention(&self.catalog.state_root),
|
||||
)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
|
||||
}
|
||||
|
||||
fn journal_origins(
|
||||
&self,
|
||||
ranges: &[JournalSourceRange],
|
||||
) -> Result<Vec<SourceOrigin>, SubjektivHostError> {
|
||||
if ranges.is_empty() {
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"empty material selection".into(),
|
||||
));
|
||||
}
|
||||
ranges
|
||||
.iter()
|
||||
.map(|range| {
|
||||
let reader = self.source_reader(&range.source)?;
|
||||
reader
|
||||
.descriptors()
|
||||
.find(|descriptor| {
|
||||
source_descriptor(&range.source.session_id, descriptor) == range.source
|
||||
})
|
||||
.map(|descriptor| SourceOrigin {
|
||||
pointer: range.source.pointer.clone(),
|
||||
origin: descriptor.origin.clone(),
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
SubjektivHostError::InvalidArgument(
|
||||
"source provenance/identity changed".into(),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn journal_source_window(
|
||||
&self,
|
||||
source: &JournalSource,
|
||||
offset: u64,
|
||||
max_bytes: usize,
|
||||
) -> Result<String, SubjektivHostError> {
|
||||
let reader = self.source_reader(source)?;
|
||||
let descriptor = reader.descriptors().find(|descriptor| source_descriptor(&source.session_id, descriptor) == *source)
|
||||
.ok_or_else(|| SubjektivHostError::InvalidArgument("canonical source identity/order/digest changed or is outside the authorized Session".into()))?;
|
||||
reader
|
||||
.read_content(descriptor, offset, max_bytes)
|
||||
.map(|window| window.content)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
|
||||
}
|
||||
|
||||
fn journal_source_material(
|
||||
&self,
|
||||
source: &JournalSource,
|
||||
start: u64,
|
||||
limit: usize,
|
||||
) -> JournalResult<JournalMaterial> {
|
||||
let text = self
|
||||
.journal_source_window(source, start, limit.max(4))
|
||||
.map_err(|error| JournalError::Conflict(error.to_string()))?;
|
||||
Ok(JournalMaterial {
|
||||
range: JournalSourceRange {
|
||||
source: source.clone(),
|
||||
bytes: JournalByteRange {
|
||||
start,
|
||||
end: start + text.len() as u64,
|
||||
},
|
||||
},
|
||||
text,
|
||||
)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
|
||||
})
|
||||
}
|
||||
|
||||
fn select_journal_materials(
|
||||
fn build_journal_input(
|
||||
&self,
|
||||
context: &JournalHostContext,
|
||||
id: &str,
|
||||
) -> Result<Vec<JournalMaterial>, SubjektivHostError> {
|
||||
subjektiv::journal::recording::select_materials_bounded(
|
||||
&self.catalog.store,
|
||||
context,
|
||||
id,
|
||||
INPUT_BUDGET,
|
||||
|source, start, limit| self.journal_source_material(source, start, limit),
|
||||
)
|
||||
saved: Option<&CaptureDto>,
|
||||
) -> Result<JournalInput, SubjektivHostError> {
|
||||
let session = self
|
||||
.catalog
|
||||
.store
|
||||
.read_journal(context, id)
|
||||
.map_err(journal_error)?
|
||||
.source_session_id;
|
||||
JournalInput::build(&self.catalog.store, context, id, |order, offset, budget| {
|
||||
let capture = match saved {
|
||||
Some(capture) => capture.clone(),
|
||||
None => {
|
||||
let capture = self
|
||||
.with_recording_boundary(&session, |sessions, session_id, active, guard| {
|
||||
session_store::capture::capture_recording_slice_fs_guarded(
|
||||
sessions,
|
||||
session_id,
|
||||
active,
|
||||
session_store::public_material::PublicMaterialLimits::default(),
|
||||
RecordingPosition {
|
||||
canonical_order: order,
|
||||
byte_offset: offset,
|
||||
},
|
||||
MAX_JOURNAL_RANGES,
|
||||
budget,
|
||||
guard,
|
||||
)
|
||||
.map(|slice| slice.capture)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
|
||||
})
|
||||
.map_err(|e| JournalError::Conflict(e.to_string()))?;
|
||||
// Admission and capture are separate saves. An append between them
|
||||
// may be in replay but has not yet been accepted for this Journal.
|
||||
return self.accepted_capture_prefix(context, id, capture);
|
||||
}
|
||||
};
|
||||
let end = capture
|
||||
.entries
|
||||
.last()
|
||||
.map(|entry| {
|
||||
capture
|
||||
.recording_text(entry.order)
|
||||
.map(|(descriptor, start, text)| {
|
||||
if start + text.len() as u64 == descriptor.total_bytes {
|
||||
RecordingPosition {
|
||||
canonical_order: entry.order + 1,
|
||||
byte_offset: 0,
|
||||
}
|
||||
} else {
|
||||
RecordingPosition {
|
||||
canonical_order: entry.order,
|
||||
byte_offset: start + text.len() as u64,
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
.map_err(|e| JournalError::Conflict(e.to_string()))?;
|
||||
if end.is_none_or(|end| (order, offset) >= (end.canonical_order, end.byte_offset)) {
|
||||
return Ok(CaptureDto {
|
||||
entries: vec![],
|
||||
..capture
|
||||
});
|
||||
}
|
||||
capture
|
||||
.recording_slice(
|
||||
RecordingPosition {
|
||||
canonical_order: order,
|
||||
byte_offset: offset,
|
||||
},
|
||||
MAX_JOURNAL_RANGES,
|
||||
budget,
|
||||
)
|
||||
.map(|slice| slice.capture)
|
||||
.map_err(|e| JournalError::Conflict(e.to_string()))
|
||||
})
|
||||
.map_err(journal_error)
|
||||
}
|
||||
|
||||
fn journal_grant(
|
||||
fn accepted_capture_prefix(
|
||||
&self,
|
||||
context: &JournalHostContext,
|
||||
id: &str,
|
||||
mut capture: CaptureDto,
|
||||
) -> JournalResult<CaptureDto> {
|
||||
let mut selected = Vec::new();
|
||||
for mut entry in capture.entries {
|
||||
let status = match self
|
||||
.catalog
|
||||
.store
|
||||
.read_journal_source(context, id, entry.order)
|
||||
{
|
||||
Ok(status) => status,
|
||||
Err(JournalError::NotFound(_)) => break,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let session_store::capture::CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
..
|
||||
} = &mut entry.body
|
||||
else {
|
||||
return Err(JournalError::InvalidInput("non-recording capture".into()));
|
||||
};
|
||||
if source_descriptor(&capture.session_id, descriptor) != status.source {
|
||||
return Err(JournalError::Conflict(
|
||||
"captured source differs from admitted identity".into(),
|
||||
));
|
||||
}
|
||||
let Some(accepted) = status
|
||||
.accepted
|
||||
.iter()
|
||||
.find(|range| range.start <= *byte_offset && *byte_offset < range.end)
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let end = status
|
||||
.unavailable
|
||||
.iter()
|
||||
.filter(|range| range.end > *byte_offset)
|
||||
.fold(accepted.end, |end, range| {
|
||||
end.min(range.start.max(*byte_offset))
|
||||
});
|
||||
let mut length = text.len().min(end.saturating_sub(*byte_offset) as usize);
|
||||
while !text.is_char_boundary(length) {
|
||||
length -= 1;
|
||||
}
|
||||
if length == 0 {
|
||||
break;
|
||||
}
|
||||
text.truncate(length);
|
||||
let partial = *byte_offset + (text.len() as u64) < descriptor.total_bytes;
|
||||
selected.push(entry);
|
||||
if partial {
|
||||
break;
|
||||
}
|
||||
}
|
||||
capture.entries = selected;
|
||||
Ok(capture)
|
||||
}
|
||||
|
||||
fn journal_grant_scope(
|
||||
&self,
|
||||
snapshot: &JobSnapshot,
|
||||
) -> Result<LocalJournalGrant, SubjektivHostError> {
|
||||
@@ -1097,7 +1358,6 @@ impl LocalSubjectHost {
|
||||
|| snapshot.request.purpose != PURPOSE
|
||||
|| snapshot.request.serialization_key.as_deref()
|
||||
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
|
||||
|| snapshot.request.input != grant.input()
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"Job input/grant does not match this Journal capability".into(),
|
||||
@@ -1135,12 +1395,33 @@ impl LocalSubjectHost {
|
||||
Ok(grant)
|
||||
}
|
||||
|
||||
fn journal_grant(
|
||||
&self,
|
||||
snapshot: &JobSnapshot,
|
||||
) -> Result<LocalJournalGrant, SubjektivHostError> {
|
||||
snapshot
|
||||
.load_executable_request()
|
||||
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
|
||||
let grant = self.journal_grant_scope(snapshot)?;
|
||||
let input = JournalInput::decode(&snapshot.request.input).map_err(journal_error)?;
|
||||
if input.subject_id != self.binding.subject_id
|
||||
|| input.journal_id != grant.journal_id
|
||||
|| input.capture.session_id != grant.session_id
|
||||
|| input.selected_ranges != grant.ranges
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"saved Journal body identity/ranges differ from grant".into(),
|
||||
));
|
||||
}
|
||||
Ok(grant)
|
||||
}
|
||||
|
||||
fn complete_journal_job(
|
||||
&self,
|
||||
jobs: &crate::jobs::StandaloneJobs,
|
||||
snapshot: &JobSnapshot,
|
||||
) -> Result<(), SubjektivHostError> {
|
||||
let grant = self.journal_grant(snapshot)?;
|
||||
let grant = self.journal_grant_scope(snapshot)?;
|
||||
let context = authority(self, &grant.session_id)?;
|
||||
let completion: JournalJobCompletion =
|
||||
serde_json::from_value(snapshot.result.clone().ok_or_else(|| {
|
||||
@@ -1174,6 +1455,12 @@ impl LocalSubjectHost {
|
||||
}
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
jobs.reclaim_input(
|
||||
&snapshot.request.job_id,
|
||||
&snapshot.input_digest,
|
||||
JobInputRetirementReason::Completed,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
if let Err(error) = self.prune_retained_workers() {
|
||||
// Pruning is independent of a successful checkpoint/result. Keep the originals
|
||||
// and continue draining other saved prefixes; a retention error is not model failure.
|
||||
@@ -1192,47 +1479,18 @@ pub(super) fn grant(
|
||||
.journal_grant(snapshot)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let context = authority(&host, &grant.session_id).map_err(|error| error.to_string())?;
|
||||
// Re-fetch selected identities before opening mutation authority. Only descriptors are
|
||||
// retained in Job storage; the model must fetch actual source bytes with bounded tools.
|
||||
for range in &grant.ranges {
|
||||
let length = usize::try_from(
|
||||
range
|
||||
.bytes
|
||||
.end
|
||||
.checked_sub(range.bytes.start)
|
||||
.ok_or("invalid source range")?,
|
||||
)
|
||||
.map_err(|_| "source range overflow")?;
|
||||
if length > INPUT_BUDGET {
|
||||
return Err("source range exceeds frozen input budget".into());
|
||||
}
|
||||
let text = host
|
||||
.journal_source_window(&range.source, range.bytes.start, length.max(4))
|
||||
.map_err(|error| error.to_string())?;
|
||||
if text.len() != length {
|
||||
return Err("frozen Journal source byte range changed".into());
|
||||
}
|
||||
}
|
||||
if host
|
||||
.journal_origins(&grant.ranges)
|
||||
.map_err(|error| error.to_string())?
|
||||
!= grant.origins
|
||||
{
|
||||
return Err(
|
||||
"frozen Journal provenance does not match the authorized original Session".into(),
|
||||
);
|
||||
}
|
||||
let input = JournalInput::decode(&snapshot.request.input).map_err(|e| e.to_string())?;
|
||||
let input_digest = snapshot.input_digest.clone();
|
||||
let generation = host
|
||||
.catalog
|
||||
.store
|
||||
.prepare_journal_generation(
|
||||
.prepare_journal_input(
|
||||
&context,
|
||||
&grant.journal_id,
|
||||
&input,
|
||||
JobAttemptBinding {
|
||||
job_id: snapshot.request.job_id.clone(),
|
||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||
},
|
||||
grant.ranges.clone(),
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let job_context = JournalJobContext::validated(
|
||||
@@ -1242,12 +1500,17 @@ pub(super) fn grant(
|
||||
generation.job.clone(),
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
input
|
||||
.validate_generation(&generation)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let adapter = Arc::new(LocalJournalHost {
|
||||
host,
|
||||
grant,
|
||||
context,
|
||||
job_context,
|
||||
#[cfg(test)]
|
||||
generation,
|
||||
input_digest,
|
||||
fence,
|
||||
});
|
||||
Ok(worker::job::JobFeatureGrant {
|
||||
@@ -1263,7 +1526,9 @@ struct LocalJournalHost {
|
||||
grant: LocalJournalGrant,
|
||||
context: JournalHostContext,
|
||||
job_context: JournalJobContext,
|
||||
#[cfg(test)]
|
||||
generation: JournalGeneration,
|
||||
input_digest: String,
|
||||
fence: Arc<dyn crate::jobs::JobAttemptFence>,
|
||||
}
|
||||
impl LocalJournalHost {
|
||||
@@ -1296,18 +1561,20 @@ impl JournalJobHost for LocalJournalHost {
|
||||
request: JournalJobRequest,
|
||||
) -> Result<JournalJobResponse, SubjektivHostError> {
|
||||
let context = self.live()?;
|
||||
subjektiv::journal::service::execute_bounded(
|
||||
// The generic runner has supplied this exact immutable input before it can
|
||||
// invoke this bound tool. Preparation alone is not evidence of delivery.
|
||||
subjektiv::journal::service::attest_input_delivery(
|
||||
&self.host.catalog.store,
|
||||
&context,
|
||||
&self.input_digest,
|
||||
&self.grant.ranges,
|
||||
)
|
||||
.map_err(journal_error)?;
|
||||
subjektiv::journal::service::execute_snapshot(
|
||||
&self.host.catalog.store,
|
||||
&context,
|
||||
&self.input_digest,
|
||||
request,
|
||||
|source, start, limit| {
|
||||
if source.session_id != self.generation.initial_journal.source_session_id {
|
||||
return Err(JournalError::PermissionDenied(
|
||||
"foreign original Session".into(),
|
||||
));
|
||||
}
|
||||
self.host.journal_source_material(source, start, limit)
|
||||
},
|
||||
)
|
||||
.map_err(journal_error)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Integration of standalone attribution, original FsStore material, shared Journal storage
|
||||
//! and the local Job adapter. Only the outer model/execution lease is replaced in these cases.
|
||||
//! The generic Job runner is exercised separately by the standalone Jobs integration suite.
|
||||
//! The scripted model also exercises saved direct input through the actual generic Job runner.
|
||||
|
||||
use super::*;
|
||||
use crate::store::{StaleLeasePolicy, StandaloneWorkerAllocation, StandaloneWorkerStore};
|
||||
@@ -144,8 +144,8 @@ impl Fixture {
|
||||
.unwrap()
|
||||
.contains(sentinel)
|
||||
);
|
||||
// Include SQLite/WAL/journal files: immutable request/grant storage must never hold
|
||||
// original text, even in freed pages left behind by explicit source erasure.
|
||||
// Include SQLite/WAL/journal files: erasure must scrub retired original bodies,
|
||||
// including any freed pages or journal left by the immutable request store.
|
||||
for entry in fs::read_dir(self.connection.jobs_path.parent().unwrap()).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
if entry.file_type().unwrap().is_file() {
|
||||
@@ -194,9 +194,9 @@ impl Fixture {
|
||||
context,
|
||||
job_context,
|
||||
generation,
|
||||
input_digest: snapshot.input_digest.clone(),
|
||||
fence: fence.clone(),
|
||||
};
|
||||
adapter.read_all_sources();
|
||||
(adapter, fence, snapshot)
|
||||
}
|
||||
}
|
||||
@@ -220,37 +220,6 @@ fn message(id: &str, text: &str) -> session_store::LoggedHistoryEntry {
|
||||
}
|
||||
}
|
||||
impl LocalJournalHost {
|
||||
fn read_all_sources(&self) {
|
||||
for range in &self.generation.selected_ranges {
|
||||
let mut offset = range.bytes.start as usize;
|
||||
let mut content_token = None;
|
||||
loop {
|
||||
let response = self
|
||||
.execute(JournalJobRequest::ReadText {
|
||||
read: JournalTextRead {
|
||||
target: JournalTextTarget::Source {
|
||||
pointer: range.source.pointer.clone(),
|
||||
},
|
||||
offset,
|
||||
limit: MAX_JOURNAL_READ_BYTES,
|
||||
content_token: content_token.clone(),
|
||||
},
|
||||
})
|
||||
.unwrap();
|
||||
let JournalJobResponse::Text(text) = response else {
|
||||
panic!()
|
||||
};
|
||||
match text.next_byte {
|
||||
Some(next) => {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
content_token = Some(text.content_token);
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fn state(&self) -> Result<JournalJobState, SubjektivHostError> {
|
||||
match self.execute(JournalJobRequest::ReadState)? {
|
||||
JournalJobResponse::State(state) => Ok(state),
|
||||
@@ -362,8 +331,8 @@ fn duplicate_admission_reserves_one_job_and_excludes_private_prompt() {
|
||||
assert_eq!(jobs.pending().unwrap().len(), 1);
|
||||
let input = jobs.get(&id).unwrap().request.input.to_string();
|
||||
assert!(
|
||||
!input.contains("short request"),
|
||||
"Job storage retains descriptors, never a second copy of original text"
|
||||
input.contains("short request"),
|
||||
"Job storage supplies the canonical body directly"
|
||||
);
|
||||
assert!(!input.contains("PRIVATE PROMPT"));
|
||||
let foreign = SubjektivHostContext {
|
||||
@@ -498,14 +467,16 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() {
|
||||
&adapter.generation.job,
|
||||
)
|
||||
.unwrap();
|
||||
let materials = fixture
|
||||
let next = fixture
|
||||
.host()
|
||||
.select_journal_materials(&adapter.context, &adapter.grant.journal_id)
|
||||
.build_journal_input(&adapter.context, &adapter.grant.journal_id, None)
|
||||
.unwrap();
|
||||
assert_eq!(materials[0].range.source.pointer.entry_ref, "Elast");
|
||||
assert_eq!(next.selected_ranges[0].source.pointer.entry_ref, "Elast");
|
||||
assert!(
|
||||
materials[0]
|
||||
.text
|
||||
next.capture
|
||||
.recording_text(next.selected_ranges[0].source.order)
|
||||
.unwrap()
|
||||
.2
|
||||
.contains("last input while generation runs")
|
||||
);
|
||||
}
|
||||
@@ -547,7 +518,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
|
||||
let (adapter, _, _) = fixture.adapter();
|
||||
let range = &adapter.grant.ranges[0];
|
||||
assert!(range.bytes.end < range.source.total_bytes);
|
||||
assert!(range.bytes.end - range.bytes.start < INPUT_BUDGET as u64);
|
||||
assert!(range.bytes.end - range.bytes.start < job::MAX_JOB_INPUT_BYTES as u64);
|
||||
let snapshot = fixture
|
||||
.host()
|
||||
.jobs()
|
||||
@@ -557,7 +528,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
|
||||
.pop()
|
||||
.unwrap();
|
||||
assert!(serde_json::to_vec(&snapshot.request.input).unwrap().len() < 128 * 1024);
|
||||
assert!(!snapshot.request.input.to_string().contains("漢字"));
|
||||
assert!(snapshot.request.input.to_string().contains("漢字"));
|
||||
assert!(adapter.finish().is_err());
|
||||
}
|
||||
|
||||
@@ -628,43 +599,35 @@ impl JournalScript {
|
||||
)
|
||||
} else if let Some(state) = result("state") {
|
||||
let state = &state["value"];
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
let input: serde_json::Value = request
|
||||
.items
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("source-{index}-{offset}");
|
||||
if let Some(page) = result(&id) {
|
||||
let page = &page["value"];
|
||||
let key = format!("{}:{id}", state["generation_id"].as_str().unwrap());
|
||||
let text = page["text"].as_str().unwrap().to_owned();
|
||||
if let Some(previous) =
|
||||
self.source_reads.lock().unwrap().insert(key, text.clone())
|
||||
{
|
||||
assert_eq!(
|
||||
previous, text,
|
||||
"frozen source read changed within a generation"
|
||||
);
|
||||
}
|
||||
if let Some(next) = page["next_byte"].as_u64() {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["content_token"].clone();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
let input = json!({"target":{"target":"source","pointer":range["source"]["pointer"]},"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token});
|
||||
return vec![
|
||||
Event::tool_use_start(0, &id, "ReadJournalText"),
|
||||
Event::tool_input_delta(0, input.to_string()),
|
||||
Event::tool_use_stop(0),
|
||||
];
|
||||
}
|
||||
.filter_map(Item::as_text)
|
||||
.find_map(|text| {
|
||||
text.lines()
|
||||
.find_map(|line| line.strip_prefix("input_json: "))
|
||||
})
|
||||
.map(|json| serde_json::from_str(json).unwrap())
|
||||
.expect("direct saved Journal input");
|
||||
let input = JournalInput::decode(&input).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(&input.selected_ranges).unwrap(),
|
||||
state["selected_ranges"]
|
||||
);
|
||||
for entry in &input.capture.entries {
|
||||
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
|
||||
let key = format!(
|
||||
"{}:{}:{offset}",
|
||||
state["generation_id"].as_str().unwrap(),
|
||||
entry.order
|
||||
);
|
||||
if let Some(previous) = self
|
||||
.source_reads
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(key, text.to_owned())
|
||||
{
|
||||
assert_eq!(previous, text, "saved source changed within generation");
|
||||
}
|
||||
}
|
||||
let (name, action) = if state["current_open"].is_null() {
|
||||
@@ -1166,6 +1129,11 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
||||
.items;
|
||||
assert_eq!(entries.len(), 1);
|
||||
let saved_entry = entries[0].id.clone();
|
||||
let original_input = jobs.get(&id).unwrap().request.input;
|
||||
fixture.append(
|
||||
"newer-than-snapshot",
|
||||
"NEW INPUT MUST NOT MIX INTO RESUMED SNAPSHOT",
|
||||
);
|
||||
jobs.shutdown().await.unwrap();
|
||||
fixture.connection.close().unwrap();
|
||||
let mut reopened = StandaloneSubjects::open_existing(&state)
|
||||
@@ -1176,8 +1144,29 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
||||
.unwrap();
|
||||
reopened.host.bind_jobs(jobs.clone()).unwrap();
|
||||
assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown);
|
||||
// Domain reconciliation inspects the now-fenced prior attempt and preserves its receipts.
|
||||
// Deny every original-source read while preparing and running the replacement.
|
||||
let source_path = fixture
|
||||
.sessions
|
||||
.root_dir()
|
||||
.join(&fixture.context.session_id);
|
||||
let denied = fixture
|
||||
._temp
|
||||
.path()
|
||||
.join("temporarily-unavailable-original");
|
||||
fs::rename(&source_path, &denied).unwrap();
|
||||
let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap();
|
||||
let replacement_input = jobs.get(&replacement).unwrap().request.input;
|
||||
assert_eq!(replacement_input["capture"], original_input["capture"]);
|
||||
assert!(
|
||||
!replacement_input
|
||||
.to_string()
|
||||
.contains("NEW INPUT MUST NOT MIX")
|
||||
);
|
||||
assert_eq!(jobs.get(&id).unwrap().request.input, original_input);
|
||||
assert_eq!(
|
||||
jobs.get(&id).unwrap().input_disposition,
|
||||
JobInputDisposition::Available
|
||||
);
|
||||
assert_ne!(
|
||||
replacement, id,
|
||||
"unknown external execution is not blindly replayed"
|
||||
@@ -1205,8 +1194,26 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
||||
.items;
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].id, saved_entry);
|
||||
fs::rename(&denied, &source_path).unwrap();
|
||||
let journal = reopened
|
||||
.host
|
||||
.catalog
|
||||
.store
|
||||
.read_journal(&context, &journal_id)
|
||||
.unwrap();
|
||||
let original = JournalInput::decode(&original_input).unwrap();
|
||||
let end = original.selected_ranges.last().unwrap();
|
||||
assert_eq!(
|
||||
(journal.process_order, journal.process_byte),
|
||||
if end.bytes.end == end.source.total_bytes {
|
||||
(end.source.order + 1, 0)
|
||||
} else {
|
||||
(end.source.order, end.bytes.end)
|
||||
}
|
||||
);
|
||||
assert!(reopened.host.admit_journal(&fixture.context).unwrap());
|
||||
assert!(
|
||||
reopened
|
||||
!reopened
|
||||
.host
|
||||
.session_fully_reflected(&fixture.context.session_id)
|
||||
.unwrap()
|
||||
@@ -1302,9 +1309,9 @@ fn missing_original_material_blocks_worker_removal_without_erasing_state() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
|
||||
fn frozen_input_survives_active_session_change_and_source_tool_remains_refused() {
|
||||
let fixture = Fixture::new("old Session original");
|
||||
let (adapter, _, _) = fixture.adapter();
|
||||
let (adapter, _, snapshot) = fixture.adapter();
|
||||
let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap();
|
||||
let worker = fixture._allocation.as_ref().unwrap().worker_id();
|
||||
let record = workers.load(worker).unwrap();
|
||||
@@ -1318,22 +1325,30 @@ fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
|
||||
)),
|
||||
))
|
||||
.unwrap();
|
||||
let response = adapter
|
||||
.execute(JournalJobRequest::ReadText {
|
||||
read: JournalTextRead {
|
||||
target: JournalTextTarget::Source {
|
||||
pointer: adapter.generation.selected_ranges[0].source.pointer.clone(),
|
||||
},
|
||||
offset: 0,
|
||||
limit: MAX_JOURNAL_READ_BYTES,
|
||||
content_token: None,
|
||||
},
|
||||
})
|
||||
.unwrap();
|
||||
let JournalJobResponse::Text(page) = response else {
|
||||
panic!()
|
||||
};
|
||||
assert!(page.text.contains("old Session original"));
|
||||
assert!(
|
||||
adapter
|
||||
.execute(JournalJobRequest::ReadText {
|
||||
read: JournalTextRead {
|
||||
target: JournalTextTarget::Source {
|
||||
pointer: adapter.generation.selected_ranges[0].source.pointer.clone()
|
||||
},
|
||||
offset: 0,
|
||||
limit: MAX_JOURNAL_READ_BYTES,
|
||||
content_token: None
|
||||
}
|
||||
})
|
||||
.is_err(),
|
||||
"original Source lookup is explicitly unavailable"
|
||||
);
|
||||
let input = JournalInput::decode(&snapshot.request.input).unwrap();
|
||||
assert!(
|
||||
input
|
||||
.capture
|
||||
.recording_text(0)
|
||||
.unwrap()
|
||||
.2
|
||||
.contains("old Session original")
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.host()
|
||||
@@ -1512,14 +1527,18 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins()
|
||||
const SENTINEL: &str = "original-plaintext-sentinel-must-never-enter-immutable-job-storage";
|
||||
let mut fixture = Fixture::new(SENTINEL);
|
||||
let (_, _, snapshot) = fixture.adapter();
|
||||
fixture.assert_job_storage_excludes(&snapshot, SENTINEL);
|
||||
assert!(snapshot.request.input.to_string().contains(SENTINEL));
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let binding = fixture.connection.binding.clone();
|
||||
let session: session_store::SessionId = fixture.context.session_id.parse().unwrap();
|
||||
let old_reader = fixture
|
||||
.host()
|
||||
.material_reader(&fixture.context.session_id)
|
||||
.unwrap();
|
||||
let old_reader = SessionPublicMaterialReader::open_fs_with_retention(
|
||||
fixture.sessions.root_dir(),
|
||||
session,
|
||||
fixture.segment,
|
||||
PublicMaterialLimits::default(),
|
||||
crate::store::session_retention(&state),
|
||||
)
|
||||
.unwrap();
|
||||
let descriptor = old_reader.descriptors().next().unwrap().clone();
|
||||
let (worker, workers) = fixture.stop_worker();
|
||||
assert!(workers.delete(worker).is_err());
|
||||
@@ -1759,14 +1778,12 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
|
||||
let snapshot = jobs.get(&next).unwrap();
|
||||
let grant = fixture.host().journal_grant(&snapshot).unwrap();
|
||||
assert_eq!(grant.ranges.len(), 1);
|
||||
assert_eq!(
|
||||
grant.origins[0].origin,
|
||||
PublicMaterialOrigin::HostRunBoundary
|
||||
);
|
||||
let content = fixture
|
||||
.host()
|
||||
.journal_source_window(&grant.ranges[0].source, 0, 4096)
|
||||
let input = JournalInput::decode(&snapshot.request.input).unwrap();
|
||||
let (descriptor, _, content) = input
|
||||
.capture
|
||||
.recording_text(grant.ranges[0].source.order)
|
||||
.unwrap();
|
||||
assert_eq!(descriptor.origin, PublicMaterialOrigin::HostRunBoundary);
|
||||
let value: serde_json::Value = serde_json::from_str(&content).unwrap();
|
||||
assert_eq!(value["kind"], "run_boundary");
|
||||
assert_eq!(value["boundary"]["kind"], "errored");
|
||||
@@ -1791,3 +1808,621 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
|
||||
assert!(fixture.host().next_journal_job().unwrap().is_none());
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
/// Original FsStore -> bounded common capture -> persisted Job -> actual generic runner ->
|
||||
/// sequential Journal tools. Only the LLM transport is scripted. Source files are deliberately
|
||||
/// unavailable after persistence, and input bodies are inspected in the provider request.
|
||||
#[tokio::test]
|
||||
async fn saved_body_jobs_process_over_a_hundred_entries_and_utf8_without_live_source_reads() {
|
||||
let fixture = Fixture::new("oldest original");
|
||||
for index in 0..115 {
|
||||
fixture.append(
|
||||
&format!("entry-{index}"),
|
||||
&format!("public original {index}"),
|
||||
);
|
||||
}
|
||||
fixture.append("long-utf8", &"漢字🙂\n\"".repeat(24_000));
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let expected = fixture
|
||||
.host()
|
||||
.recording_capture(&fixture.context.session_id)
|
||||
.unwrap();
|
||||
let source_path = fixture
|
||||
.sessions
|
||||
.root_dir()
|
||||
.join(&fixture.context.session_id);
|
||||
let denied = fixture._temp.path().join("denied-original-source");
|
||||
let mut seen: std::collections::BTreeMap<u64, String> = Default::default();
|
||||
let mut jobs_processed = 0;
|
||||
loop {
|
||||
let Some((jobs, id)) = fixture.host().next_journal_job().unwrap() else {
|
||||
break;
|
||||
};
|
||||
let saved = jobs.get(&id).unwrap();
|
||||
let input = JournalInput::decode(&saved.request.input).unwrap();
|
||||
assert!(
|
||||
serde_json::to_vec(&saved.request.input).unwrap().len() <= job::MAX_JOB_INPUT_BYTES
|
||||
);
|
||||
assert!(input.capture.entries.len() <= MAX_JOURNAL_RANGES);
|
||||
for entry in &input.capture.entries {
|
||||
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
|
||||
let previous = seen.entry(entry.order).or_default();
|
||||
assert_eq!(
|
||||
previous.len() as u64,
|
||||
offset,
|
||||
"no gap, preview or duplicate prefix"
|
||||
);
|
||||
previous.push_str(text);
|
||||
}
|
||||
// Deny all catalogs/reads, including grant preparation. The model still receives
|
||||
// exactly the persisted body, and the Host must not follow the active pointer.
|
||||
fs::rename(&source_path, &denied).unwrap();
|
||||
let model = JournalScript::default();
|
||||
jobs.start_with_model_client(&id, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(result.state, JobState::Completed);
|
||||
assert!(
|
||||
!model.source_reads.lock().unwrap().is_empty(),
|
||||
"provider saw saved source bodies"
|
||||
);
|
||||
fixture.host().complete_journal_job(&jobs, &result).unwrap();
|
||||
assert_eq!(
|
||||
jobs.get(&id).unwrap().input_disposition,
|
||||
JobInputDisposition::Reclaimed
|
||||
);
|
||||
fs::rename(&denied, &source_path).unwrap();
|
||||
jobs_processed += 1;
|
||||
assert!(jobs_processed < 20, "bounded prefix drain must progress");
|
||||
}
|
||||
assert!(
|
||||
jobs_processed > 2,
|
||||
"entry count and long text both require continuation"
|
||||
);
|
||||
assert_eq!(seen.len(), expected.entries.len());
|
||||
for entry in &expected.entries {
|
||||
assert_eq!(
|
||||
seen[&entry.order],
|
||||
expected.recording_text(entry.order).unwrap().2
|
||||
);
|
||||
}
|
||||
fixture.host().jobs().unwrap().shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful_records() {
|
||||
let fixture = Fixture::new("original descriptor-only input");
|
||||
let (adapter, _, modern) = fixture.adapter();
|
||||
let jobs = fixture.host().jobs().unwrap();
|
||||
fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.close_journal_generation(
|
||||
&adapter.context,
|
||||
&adapter.generation.id,
|
||||
&adapter.generation.job,
|
||||
)
|
||||
.unwrap();
|
||||
jobs.cancel(&modern.request.job_id).unwrap();
|
||||
let input = JournalInput::decode(&modern.request.input).unwrap();
|
||||
let origins: Vec<_> = input
|
||||
.capture
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let (descriptor, _, _) = input.capture.recording_text(entry.order).unwrap();
|
||||
SourceOrigin {
|
||||
pointer: source_descriptor(&fixture.context.session_id, descriptor).pointer,
|
||||
origin: descriptor.origin.clone(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let mut old_grant = adapter.grant.clone();
|
||||
old_grant.origins = origins;
|
||||
old_grant.writer_binding_id = None;
|
||||
let mut request = modern.request.clone();
|
||||
request.job_id = format!("legacy-descriptor:{}", uuid::Uuid::now_v7());
|
||||
request.input =
|
||||
serde_json::json!({"selected_ranges":old_grant.ranges,"origins":old_grant.origins});
|
||||
let old = jobs
|
||||
.request_granted(request, serde_json::to_value(&old_grant).unwrap())
|
||||
.unwrap();
|
||||
assert!(
|
||||
fixture.host().journal_grant(&old).is_err(),
|
||||
"descriptor jobs must not execute as snapshots"
|
||||
);
|
||||
let generation = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.prepare_journal_generation(
|
||||
&adapter.context,
|
||||
&old_grant.journal_id,
|
||||
JobAttemptBinding {
|
||||
job_id: old.request.job_id.clone(),
|
||||
attempt_id: old.attempt.attempt_id.clone(),
|
||||
},
|
||||
old_grant.ranges.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let context = JournalJobContext::validated(
|
||||
&fixture.host().catalog.store,
|
||||
&adapter.context,
|
||||
&generation.id,
|
||||
generation.job.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let saved = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.execute_journal(
|
||||
&context,
|
||||
JournalOperation {
|
||||
request_id: "legacy-record-success-response-lost".into(),
|
||||
expected_state: generation.initial_journal.state_token.clone(),
|
||||
action: JournalAction::Create {
|
||||
title: "already saved".into(),
|
||||
body: "legacy successful operation".into(),
|
||||
ongoing: "unfinished".into(),
|
||||
state: JournalEntryState::Open,
|
||||
source_end: None,
|
||||
correction_refs: vec![],
|
||||
},
|
||||
covered_ranges: generation.selected_ranges.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
fixture.append(
|
||||
"new-not-selected",
|
||||
"LATEST SOURCE MUST WAIT UNTIL OLD INPUT FINISHES",
|
||||
);
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
assert_ne!(next, old.request.job_id);
|
||||
let replacement = jobs.get(&next).unwrap();
|
||||
let replacement_input = JournalInput::decode(&replacement.request.input).unwrap();
|
||||
assert!(
|
||||
!replacement
|
||||
.request
|
||||
.input
|
||||
.to_string()
|
||||
.contains("LATEST SOURCE MUST WAIT")
|
||||
);
|
||||
assert_eq!(replacement_input.selected_ranges, old_grant.ranges);
|
||||
let retired = jobs.get(&old.request.job_id).unwrap();
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
|
||||
assert_eq!(retired.input_digest, old.input_digest);
|
||||
assert_eq!(retired.intent_fingerprint, old.intent_fingerprint);
|
||||
assert!(retired.request.input.is_null());
|
||||
assert!(
|
||||
fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.execute_journal(
|
||||
&context,
|
||||
JournalOperation {
|
||||
request_id: "legacy-delayed".into(),
|
||||
expected_state: saved.state_token,
|
||||
action: JournalAction::ConfirmUnchanged,
|
||||
covered_ranges: old_grant.ranges,
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
jobs.start_with_model_client(&next, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&next))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
fixture
|
||||
.host()
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
let entries = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32)
|
||||
.unwrap();
|
||||
assert_eq!(entries.items.len(), 1);
|
||||
assert_eq!(entries.items[0].id, saved.entry_id.unwrap());
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_erase_retires_every_historical_body_across_job_history_pages() {
|
||||
const SENTINEL: &str = "historical-input-erasure-sentinel-781";
|
||||
let mut fixture = Fixture::new("ordinary original");
|
||||
let (adapter, _, first) = fixture.adapter();
|
||||
let jobs = fixture.host().jobs().unwrap();
|
||||
for index in 0..105 {
|
||||
let mut request = first.request.clone();
|
||||
request.job_id = format!("historical-{index:03}");
|
||||
request.purpose = "historical-other-domain-job".into();
|
||||
request.serialization_key = None;
|
||||
request.input = serde_json::json!({"body":SENTINEL, "index":index});
|
||||
let saved = jobs
|
||||
.request_granted(
|
||||
request,
|
||||
serde_json::json!({"session_id":fixture.context.session_id}),
|
||||
)
|
||||
.unwrap();
|
||||
jobs.cancel(&saved.request.job_id).unwrap();
|
||||
}
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let subject = fixture.connection.binding.subject_id.clone();
|
||||
fixture.connection.close().unwrap();
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.erase_session_completely(&subject, &fixture.context.session_id, "erase-all-history")
|
||||
.unwrap();
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&fixture.connection.jobs_path,
|
||||
fixture._temp.path().into(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut after = None;
|
||||
let mut count = 0;
|
||||
loop {
|
||||
let page = jobs
|
||||
.list_session_snapshots(&fixture.context.session_id, after.as_deref(), 100)
|
||||
.unwrap();
|
||||
if page.is_empty() {
|
||||
break;
|
||||
}
|
||||
for saved in &page {
|
||||
assert_eq!(saved.input_disposition, JobInputDisposition::Erased);
|
||||
assert!(saved.request.input.is_null());
|
||||
assert!(saved.load_executable_request().is_err());
|
||||
}
|
||||
count += page.len();
|
||||
after = page.last().map(|job| job.request.job_id.clone());
|
||||
}
|
||||
assert_eq!(count, 106);
|
||||
assert!(
|
||||
adapter.execute(JournalJobRequest::ReadState).is_err(),
|
||||
"prepared local adapter is invalidated"
|
||||
);
|
||||
fixture.assert_job_storage_excludes(&jobs.get(&first.request.job_id).unwrap(), SENTINEL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn saved_snapshot_missing_tampered_foreign_or_changed_initial_state_is_not_recaptured() {
|
||||
let fixture = Fixture::new("saved immutable original");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let saved = jobs.get(&id).unwrap();
|
||||
for changed in [serde_json::Value::Null, serde_json::json!({"capture":{}})] {
|
||||
let mut malformed = saved.clone();
|
||||
malformed.request.input = changed;
|
||||
assert!(fixture.host().journal_grant(&malformed).is_err());
|
||||
}
|
||||
let original = JournalInput::decode(&saved.request.input).unwrap();
|
||||
for field in ["subject_id", "journal_id", "initial_state_token"] {
|
||||
let mut changed = saved.request.clone();
|
||||
changed.job_id = format!("changed-{field}");
|
||||
changed.serialization_key = None;
|
||||
changed.input[field] = serde_json::json!("foreign-or-altered");
|
||||
// Even a separately reserved immutable Job may not adopt the original capability.
|
||||
let changed = jobs
|
||||
.request_granted(changed, saved.domain_grant.clone().unwrap())
|
||||
.unwrap();
|
||||
assert!(
|
||||
grant(
|
||||
fixture.host().clone(),
|
||||
&changed,
|
||||
Arc::new(TestFence(AtomicBool::new(true)))
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let mut generation = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.prepare_journal_generation(
|
||||
&authority(fixture.host(), &fixture.context.session_id).unwrap(),
|
||||
&original.journal_id,
|
||||
JobAttemptBinding {
|
||||
job_id: id,
|
||||
attempt_id: saved.attempt.attempt_id.clone(),
|
||||
},
|
||||
original.selected_ranges.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
generation.initial_journal.state_token = "changed-after-snapshot".into();
|
||||
assert!(original.validate_generation(&generation).is_err());
|
||||
assert_eq!(
|
||||
saved.request.input,
|
||||
jobs.get(&saved.request.job_id).unwrap().request.input
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn append_between_admission_and_body_capture_does_not_publish_unaccepted_input() {
|
||||
let fixture = Fixture::new("accepted oldest prefix");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
fixture.append("not-yet-admitted", "newly saved but not accepted");
|
||||
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let input = jobs.get(&id).unwrap().request.input;
|
||||
assert!(input.to_string().contains("accepted oldest prefix"));
|
||||
assert!(!input.to_string().contains("newly saved but not accepted"));
|
||||
jobs.start_with_model_client(&id, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
fixture
|
||||
.host()
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let remaining = jobs.get(&next).unwrap().request.input;
|
||||
assert!(
|
||||
remaining
|
||||
.to_string()
|
||||
.contains("newly saved but not accepted")
|
||||
);
|
||||
assert!(!remaining.to_string().contains("accepted oldest prefix"));
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
/// Generic Job preparation must not attest delivery. The saved body is supplied by the
|
||||
/// runner before a bound Journal tool may execute; no model-provided digest is trusted.
|
||||
#[tokio::test]
|
||||
async fn actual_bound_tool_attests_delivery_but_prepared_blocked_job_does_not() {
|
||||
let fixture = Fixture::new("body delivered through generic runner");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let saved = jobs.get(&id).unwrap();
|
||||
let model = JournalScript {
|
||||
gate_first: true,
|
||||
pause_after_record: true,
|
||||
..Default::default()
|
||||
};
|
||||
jobs.start_with_model_client(&id, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
let host_context = authority(fixture.host(), &fixture.context.session_id).unwrap();
|
||||
let journal_id = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.journal_for_session(&host_context)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let generation = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.active_journal_generation(&host_context, &journal_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let context = JournalJobContext::validated(
|
||||
&fixture.host().catalog.store,
|
||||
&host_context,
|
||||
&generation.id,
|
||||
generation.job.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let undelivered = subjektiv::journal::service::execute_snapshot(
|
||||
&fixture.host().catalog.store,
|
||||
&context,
|
||||
&saved.input_digest,
|
||||
JournalJobRequest::ReadState,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
undelivered.to_string().contains("delivery"),
|
||||
"prepared grant must not satisfy delivery gate: {undelivered}"
|
||||
);
|
||||
let before = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.read_journal(&host_context, &journal_id)
|
||||
.unwrap();
|
||||
assert_eq!((before.process_order, before.process_byte), (0, 0));
|
||||
model.release.notify_one();
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(20),
|
||||
model.record_saved.notified(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
subjektiv::journal::service::execute_snapshot(
|
||||
&fixture.host().catalog.store,
|
||||
&context,
|
||||
&saved.input_digest,
|
||||
JournalJobRequest::ReadState
|
||||
)
|
||||
.is_ok(),
|
||||
"actual bound tool execution attests exact saved input"
|
||||
);
|
||||
model.release.notify_one();
|
||||
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
fixture
|
||||
.host()
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct BlockedJournalRequest {
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
dropped: Arc<tokio::sync::Notify>,
|
||||
request_dropped: Arc<AtomicBool>,
|
||||
}
|
||||
struct OwnedBlockedJournalRequest {
|
||||
// A local request copy, including the saved input, must be dropped before erasure
|
||||
// can acquire the lease. This does not claim remote/provider bytes are retractable.
|
||||
_request: agen::llm_client::Request,
|
||||
model: BlockedJournalRequest,
|
||||
}
|
||||
impl Drop for OwnedBlockedJournalRequest {
|
||||
fn drop(&mut self) {
|
||||
self.model.request_dropped.store(true, Ordering::Release);
|
||||
self.model.dropped.notify_one();
|
||||
}
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl agen::llm_client::LlmClient for BlockedJournalRequest {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: agen::llm_client::Request,
|
||||
) -> Result<
|
||||
std::pin::Pin<
|
||||
Box<
|
||||
dyn futures::Stream<
|
||||
Item = Result<
|
||||
agen::llm_client::event::Event,
|
||||
agen::llm_client::error::ClientError,
|
||||
>,
|
||||
> + Send,
|
||||
>,
|
||||
>,
|
||||
agen::llm_client::error::ClientError,
|
||||
> {
|
||||
let owned = OwnedBlockedJournalRequest {
|
||||
_request: request,
|
||||
model: self.clone(),
|
||||
};
|
||||
assert!(
|
||||
owned
|
||||
._request
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(agen::llm_client::types::Item::as_text)
|
||||
.any(|text| text.contains("local blocked source body"))
|
||||
);
|
||||
self.entered.notify_one();
|
||||
std::future::pending::<()>().await;
|
||||
drop(owned);
|
||||
unreachable!()
|
||||
}
|
||||
fn clone_boxed(&self) -> Box<dyn agen::llm_client::LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Real generic Jobs and a retained public handle, with only the provider transport blocked.
|
||||
/// Connection close must initiate cancellation and preserve the exclusive lease until local
|
||||
/// request/history ownership is gone (or return an explicit incomplete barrier).
|
||||
#[tokio::test]
|
||||
async fn close_cancels_retained_local_job_before_erase_can_acquire_exclusive_lease() {
|
||||
let mut fixture = Fixture::new("local blocked source body");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let retained = jobs.clone();
|
||||
let model = BlockedJournalRequest::default();
|
||||
retained
|
||||
.start_with_model_client(&id, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let subject = fixture.connection.binding.subject_id.clone();
|
||||
let close_error = fixture.connection.close().unwrap_err();
|
||||
assert!(matches!(
|
||||
close_error,
|
||||
SubjectError::JobShutdown(crate::jobs::JobServiceError::ShutdownIncomplete)
|
||||
));
|
||||
assert!(fixture.connection.lease.is_some());
|
||||
assert!(fixture.host().jobs.lock().unwrap().is_some());
|
||||
assert!(matches!(
|
||||
retained.start(&id).await,
|
||||
Err(crate::jobs::JobServiceError::Closed)
|
||||
));
|
||||
let locally_dropped_before_erase = model.request_dropped.load(Ordering::Acquire);
|
||||
let early_erasure = StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.erase_session_completely(
|
||||
&subject,
|
||||
&fixture.context.session_id,
|
||||
"erase-blocked-local-run",
|
||||
);
|
||||
let early_lease_was_retained = matches!(early_erasure, Err(SubjectError::Active(_)));
|
||||
let teardown_from_close = model.request_dropped.load(Ordering::Acquire)
|
||||
|| tokio::time::timeout(std::time::Duration::from_secs(5), model.dropped.notified())
|
||||
.await
|
||||
.is_ok();
|
||||
// Always tear down before asserting the observed violation, so a failed regression
|
||||
// cannot leak the blocked model future into another test.
|
||||
let _ = retained.shutdown().await;
|
||||
assert!(model.request_dropped.load(Ordering::Acquire));
|
||||
fixture.connection.close().unwrap();
|
||||
assert!(fixture.connection.lease.is_none());
|
||||
assert!(fixture.host().jobs.lock().unwrap().is_none());
|
||||
// Cancellation is not completion/erasure. Reopening retains the same unknown
|
||||
// attempt's immutable body until the explicit erasure transaction retires it.
|
||||
let reopened = crate::jobs::StandaloneJobs::open(
|
||||
&fixture.connection.jobs_path,
|
||||
fixture._temp.path().into(),
|
||||
)
|
||||
.unwrap();
|
||||
let unknown = reopened.get(&id).unwrap();
|
||||
assert_eq!(unknown.attempt.state, crate::jobs::JobAttemptState::Unknown);
|
||||
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
|
||||
assert!(
|
||||
unknown
|
||||
.request
|
||||
.input
|
||||
.to_string()
|
||||
.contains("local blocked source body")
|
||||
);
|
||||
reopened.shutdown().await.unwrap();
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.erase_session_completely(
|
||||
&subject,
|
||||
&fixture.context.session_id,
|
||||
"erase-blocked-local-run",
|
||||
)
|
||||
.unwrap();
|
||||
let erased_store = crate::jobs::StandaloneJobs::open(
|
||||
&fixture.connection.jobs_path,
|
||||
fixture._temp.path().into(),
|
||||
)
|
||||
.unwrap();
|
||||
let erased = erased_store.get(&id).unwrap();
|
||||
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
|
||||
fixture.assert_job_storage_excludes(&erased, "local blocked source body");
|
||||
erased_store.shutdown().await.unwrap();
|
||||
assert!(
|
||||
locally_dropped_before_erase || early_lease_was_retained,
|
||||
"close released exclusive lease while a retained generic Job still owned local input/history"
|
||||
);
|
||||
assert!(
|
||||
teardown_from_close,
|
||||
"connection close did not initiate cancellation of the externally retained local Job"
|
||||
);
|
||||
assert!(
|
||||
retained.get(&id).is_err(),
|
||||
"retained handles are closed after teardown"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -615,10 +615,14 @@ async fn headless_unix_client_uses_production_history_lease_and_local_authority(
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let temp = tempfile::tempdir()?;
|
||||
// Only the transport endpoint needs a short private pathname. Persistence remains the
|
||||
// same production store exercised by the in-process Host tests above.
|
||||
std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o700))?;
|
||||
let endpoint_path = temp.path().join("operator.sock");
|
||||
// Shorten only the owned endpoint's components, preserving the harness TMPDIR
|
||||
// even when it is checkout-local. No process-global cwd or external /tmp fallback.
|
||||
let socket_dir = tempfile::Builder::new()
|
||||
.prefix("")
|
||||
.rand_bytes(4)
|
||||
.tempdir()?;
|
||||
std::fs::set_permissions(socket_dir.path(), std::fs::Permissions::from_mode(0o700))?;
|
||||
let endpoint_path = socket_dir.path().join("s");
|
||||
let endpoint = standalone::UnixEndpoint::bind(&endpoint_path)?;
|
||||
let state_dir = temp.path().join("state");
|
||||
let launch = StandaloneLaunchConfig::new(
|
||||
|
||||
@@ -479,38 +479,28 @@ fn journal_response(request: &Request) -> Vec<LlmEvent> {
|
||||
}),
|
||||
);
|
||||
}
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
let input: Value = request
|
||||
.items
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("journal-source-{index}-{offset}");
|
||||
match result(&id) {
|
||||
Some(page) => match page["value"]["next_byte"].as_u64() {
|
||||
Some(next) => {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["value"]["content_token"].clone();
|
||||
}
|
||||
None => break,
|
||||
},
|
||||
None => {
|
||||
return tool(
|
||||
&id,
|
||||
"ReadJournalText",
|
||||
json!({
|
||||
"target":{"target":"source","pointer":range["source"]["pointer"]},
|
||||
"offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.filter_map(Item::as_text)
|
||||
.find_map(|text| {
|
||||
text.lines()
|
||||
.find_map(|line| line.strip_prefix("input_json: "))
|
||||
})
|
||||
.map(|json| serde_json::from_str(json).unwrap())
|
||||
.expect("saved direct Journal input");
|
||||
let input = subjektiv::journal::recording::JournalInput::decode(&input).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(&input.selected_ranges).unwrap(),
|
||||
state["selected_ranges"]
|
||||
);
|
||||
assert!(
|
||||
input
|
||||
.capture
|
||||
.entries
|
||||
.iter()
|
||||
.all(|entry| input.capture.recording_text(entry.order).is_ok())
|
||||
);
|
||||
let (name, action) = if state["current_open"].is_null() {
|
||||
(
|
||||
"CreateJournalEntry",
|
||||
|
||||
@@ -15,6 +15,8 @@ serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
server-api.workspace = true
|
||||
session-store.workspace = true
|
||||
job.workspace = true
|
||||
thiserror.workspace = true
|
||||
uuid = { workspace = true, features = ["v7"] }
|
||||
|
||||
|
||||
@@ -5,8 +5,221 @@
|
||||
//! original canonical identity while streaming and return only the requested bounded fragment.
|
||||
|
||||
use super::*;
|
||||
use session_store::capture::{CaptureDto, CaptureLimits};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// A Job owns only this bounded original-body prefix. The initial content condition binds
|
||||
/// the separate generation snapshot (whose potentially large Journal text remains paged).
|
||||
/// Neither possessing these bytes nor delivering them is a processing checkpoint.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JournalInput {
|
||||
pub version: u32,
|
||||
pub subject_id: String,
|
||||
pub journal_id: String,
|
||||
pub initial_state_token: String,
|
||||
pub initial_open_id: Option<String>,
|
||||
pub previous_fixed_end: Option<JournalSourcePointer>,
|
||||
pub selected_ranges: Vec<JournalSourceRange>,
|
||||
pub capture: CaptureDto,
|
||||
}
|
||||
|
||||
impl JournalInput {
|
||||
/// Capture callback runs once for this immutable Job, never once per page. Fitting uses
|
||||
/// only that captured value and measures the complete serialized input, including the
|
||||
/// initial Journal condition and ranges duplicated beside the capture's descriptors.
|
||||
pub fn build(
|
||||
store: &SubjektivStore,
|
||||
host: &JournalHostContext,
|
||||
journal_id: &str,
|
||||
capture_once: impl FnOnce(u64, u64, usize) -> JournalResult<CaptureDto>,
|
||||
) -> JournalResult<Self> {
|
||||
let (journal, open, previous_fixed_end) = store.database.try_with_connection(|tx| {
|
||||
super::store::check_host(store, tx, host, true)?;
|
||||
Ok::<_, JournalError>((
|
||||
super::store::journal(tx, host, journal_id, true)?,
|
||||
super::store::open(tx, journal_id)?,
|
||||
super::store::previous_fixed(tx, journal_id)?.map(|p| p.0),
|
||||
))
|
||||
})?;
|
||||
let mut input = Self {
|
||||
version: 1,
|
||||
subject_id: journal.subject_id.clone(),
|
||||
journal_id: journal_id.into(),
|
||||
initial_state_token: journal.state_token,
|
||||
initial_open_id: open.map(|e| e.id),
|
||||
previous_fixed_end,
|
||||
selected_ranges: Vec::new(),
|
||||
capture: CaptureDto {
|
||||
version: 1,
|
||||
session_id: journal.source_session_id,
|
||||
segment_id: "pending".into(),
|
||||
view_entry_count: 0,
|
||||
entries: Vec::new(),
|
||||
},
|
||||
};
|
||||
let envelope = serde_json::to_vec(&input)?.len();
|
||||
// Capture once at a fixed storage boundary. Any adjustment for the Journal envelope
|
||||
// below operates on this value only, never by making another live capture request.
|
||||
let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope);
|
||||
input.capture = capture_once(journal.process_order, journal.process_byte, budget)?;
|
||||
input.selected_ranges = capture_ranges(&input.capture)?;
|
||||
if input.selected_ranges.is_empty() {
|
||||
return Ok(input);
|
||||
}
|
||||
input.fit()?;
|
||||
let value = serde_json::to_value(&input)?;
|
||||
Self::decode(&value)
|
||||
}
|
||||
|
||||
fn fit(&mut self) -> JournalResult<()> {
|
||||
let original = self.capture.clone();
|
||||
let first = self
|
||||
.selected_ranges
|
||||
.first()
|
||||
.ok_or_else(|| invalid("empty recording selection"))?;
|
||||
let start = session_store::capture::RecordingPosition {
|
||||
canonical_order: first.source.order,
|
||||
byte_offset: first.bytes.start,
|
||||
};
|
||||
let mut budget = serde_json::to_vec(&original)?.len();
|
||||
loop {
|
||||
let bytes = serde_json::to_vec(self)?.len();
|
||||
if bytes <= job::MAX_JOB_INPUT_BYTES {
|
||||
return Ok(());
|
||||
}
|
||||
budget = budget
|
||||
.checked_sub(bytes - job::MAX_JOB_INPUT_BYTES)
|
||||
.ok_or_else(|| invalid("Journal envelope cannot fit source bytes"))?;
|
||||
self.capture = original
|
||||
.recording_slice(start, MAX_JOURNAL_RANGES, budget)
|
||||
.map_err(|e| invalid(&e.to_string()))?
|
||||
.capture;
|
||||
self.selected_ranges = capture_ranges(&self.capture)?;
|
||||
if self.selected_ranges.is_empty() {
|
||||
return Err(invalid("Journal envelope cannot fit one source scalar"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode(value: &serde_json::Value) -> JournalResult<Self> {
|
||||
if serde_json::to_vec(value)?.len() > job::MAX_JOB_INPUT_BYTES {
|
||||
return Err(invalid(
|
||||
"Journal input exceeds whole serialized Job input limit",
|
||||
));
|
||||
}
|
||||
let input: Self = serde_json::from_value(value.clone())?;
|
||||
input
|
||||
.capture
|
||||
.validate(CaptureLimits {
|
||||
max_bytes: job::MAX_JOB_INPUT_BYTES,
|
||||
max_entries: MAX_JOURNAL_RANGES,
|
||||
max_text_bytes: job::MAX_JOB_INPUT_BYTES,
|
||||
..CaptureLimits::default()
|
||||
})
|
||||
.map_err(|e| invalid(&e.to_string()))?;
|
||||
if input.version != 1
|
||||
|| input.subject_id.is_empty()
|
||||
|| input.journal_id.is_empty()
|
||||
|| input.initial_state_token.is_empty()
|
||||
|| input.selected_ranges.is_empty()
|
||||
|| input.selected_ranges.len() > MAX_JOURNAL_RANGES
|
||||
|| input
|
||||
.selected_ranges
|
||||
.iter()
|
||||
.any(|r| r.source.session_id != input.capture.session_id)
|
||||
{
|
||||
return Err(invalid("invalid Journal input identity or selection"));
|
||||
}
|
||||
if capture_ranges(&input.capture)? != input.selected_ranges {
|
||||
return Err(conflict(
|
||||
"Journal body snapshot does not match selected canonical ranges",
|
||||
));
|
||||
}
|
||||
Ok(input)
|
||||
}
|
||||
|
||||
pub fn validate_generation(&self, generation: &JournalGeneration) -> JournalResult<()> {
|
||||
if generation.journal_id != self.journal_id
|
||||
|| generation.initial_journal.subject_id != self.subject_id
|
||||
|| generation.initial_journal.source_session_id != self.capture.session_id
|
||||
|| generation.initial_journal.state_token != self.initial_state_token
|
||||
|| generation.initial_open.as_ref().map(|e| &e.id) != self.initial_open_id.as_ref()
|
||||
|| generation.previous_fixed_end != self.previous_fixed_end
|
||||
|| generation.selected_ranges != self.selected_ranges
|
||||
{
|
||||
return Err(conflict(
|
||||
"saved Journal input differs from its initial generation snapshot",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts the shared recording projection into Journal's coverage language without
|
||||
/// interpreting model text or constructing a second capture representation.
|
||||
pub fn capture_ranges(capture: &CaptureDto) -> JournalResult<Vec<JournalSourceRange>> {
|
||||
use session_store::capture::CaptureBody;
|
||||
let mut ranges = Vec::new();
|
||||
for entry in &capture.entries {
|
||||
let CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
..
|
||||
} = &entry.body
|
||||
else {
|
||||
return Err(invalid(
|
||||
"Journal requires canonical original recording capture",
|
||||
));
|
||||
};
|
||||
if !entry.policy.recording_material
|
||||
|| text.is_empty()
|
||||
|| descriptor.identity.segment_id.to_string() != entry.source.segment_id
|
||||
|| descriptor.identity.entry_ref != format!("E{}", entry.entry_id)
|
||||
|| descriptor.canonical_order != entry.order
|
||||
|| byte_offset
|
||||
.checked_add(text.len() as u64)
|
||||
.is_none_or(|end| end > descriptor.total_bytes)
|
||||
{
|
||||
return Err(conflict(
|
||||
"recording body identity, policy or range mismatch",
|
||||
));
|
||||
}
|
||||
let source = JournalSource {
|
||||
session_id: capture.session_id.clone(),
|
||||
pointer: JournalSourcePointer {
|
||||
segment_id: descriptor.identity.segment_id,
|
||||
entry_ref: descriptor.identity.entry_ref.clone(),
|
||||
},
|
||||
order: descriptor.canonical_order,
|
||||
content_sha256: descriptor.sha256.clone(),
|
||||
total_bytes: descriptor.total_bytes,
|
||||
};
|
||||
if *byte_offset == 0 && text.len() as u64 == descriptor.total_bytes {
|
||||
verify_canonical_text(&source, text)?;
|
||||
}
|
||||
ranges.push(JournalSourceRange {
|
||||
source,
|
||||
bytes: JournalByteRange {
|
||||
start: *byte_offset,
|
||||
end: *byte_offset + text.len() as u64,
|
||||
},
|
||||
});
|
||||
}
|
||||
for pair in ranges.windows(2) {
|
||||
if pair[0].source.order.checked_add(1) != Some(pair[1].source.order)
|
||||
|| pair[0].bytes.end != pair[0].source.total_bytes
|
||||
|| pair[1].bytes.start != 0
|
||||
{
|
||||
return Err(conflict(
|
||||
"recording capture is not an oldest contiguous prefix",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(ranges)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JournalMaterial {
|
||||
|
||||
@@ -66,8 +66,8 @@ fn text_page(
|
||||
})
|
||||
}
|
||||
|
||||
/// Adapter for Host captures bounded by their storage contract. Production source reads should
|
||||
/// use [`execute_bounded`] to avoid allocating an entire source for one small read.
|
||||
/// Legacy adapter for Host captures bounded by their storage contract.
|
||||
/// Production Hosts use [`attest_input_delivery`] and [`execute_snapshot`] instead.
|
||||
pub fn execute(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
@@ -101,23 +101,149 @@ pub fn execute(
|
||||
})
|
||||
}
|
||||
|
||||
/// Reads are delivered through this service before mutation authority is enabled. The durable
|
||||
/// delivery ledger stores byte ranges only: Jobs may retain immutable descriptors without
|
||||
/// accumulating a permanent second copy of the original Session in their request ledgers.
|
||||
/// The Host verifies original provenance/digest/length and returns a bounded UTF-8 fragment.
|
||||
pub(crate) fn migrate_input_delivery(
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
) -> feature_storage::Result<()> {
|
||||
tx.execute_batch(
|
||||
"CREATE TABLE journal_generation_input_deliveries (
|
||||
generation_id TEXT PRIMARY KEY NOT NULL REFERENCES journal_generations(id) ON DELETE CASCADE,
|
||||
input_digest TEXT NOT NULL CHECK(length(input_digest)=71 AND substr(input_digest,1,7)='sha256:' AND substr(input_digest,8) NOT GLOB '*[^0-9a-f]*')
|
||||
);
|
||||
CREATE TRIGGER journal_generation_input_delivery_no_update
|
||||
BEFORE UPDATE ON journal_generation_input_deliveries
|
||||
BEGIN SELECT RAISE(ABORT, 'Journal input delivery bindings are immutable'); END;",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_input_digest(input_digest: &str) -> JournalResult<()> {
|
||||
// Preserve JobRequest::input_digest() verbatim, including its algorithm prefix.
|
||||
if input_digest.strip_prefix("sha256:").is_none_or(|hex| {
|
||||
hex.len() != 64
|
||||
|| !hex
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
}) {
|
||||
return Err(JournalError::InvalidInput(
|
||||
"immutable Job input digest must be sha256: followed by 64 lowercase hex digits".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Trusted Host-only delivery evidence, never a model tool or a claim of understanding.
|
||||
/// Before calling, the Host must validate the saved, immutable body-bearing input against the
|
||||
/// bound Job's input digest, its Session/provenance/ranges, and the actual dispatch plan that
|
||||
/// supplies that exact input. The service cannot verify bytes it does not own. This transaction
|
||||
/// stores only the digest bound to the generation's immutable selection; it neither copies the
|
||||
/// body nor changes records, position, reflection or checkpoint. Exact replay is allowed only
|
||||
/// while the same generation and writer binding remain active; changed digests/ranges conflict.
|
||||
pub fn attest_input_delivery(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
input_digest: &str,
|
||||
selected_ranges: &[JournalSourceRange],
|
||||
) -> JournalResult<()> {
|
||||
validate_input_digest(input_digest)?;
|
||||
store.database.try_transaction(|tx| {
|
||||
super::lifecycle::reject_erased(tx, &context.host)?;
|
||||
let generation = super::store::job(store, tx, context, true)?;
|
||||
if generation.selected_ranges != selected_ranges {
|
||||
return Err(JournalError::Conflict(
|
||||
"immutable Job input selection differs from generation".into(),
|
||||
));
|
||||
}
|
||||
let saved: Option<String> = tx.query_row(
|
||||
"SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1",
|
||||
[&generation.id], |row| row.get(0),
|
||||
).optional()?;
|
||||
if let Some(saved) = saved {
|
||||
if saved != input_digest {
|
||||
return Err(JournalError::Conflict(
|
||||
"generation is bound to a different immutable Job input digest".into(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
tx.execute(
|
||||
"INSERT INTO journal_generation_input_deliveries(generation_id,input_digest) VALUES(?1,?2)",
|
||||
params![generation.id, input_digest],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn require_input_delivery(
|
||||
tx: &rusqlite::Connection,
|
||||
context: &JournalJobContext,
|
||||
input_digest: &str,
|
||||
) -> JournalResult<()> {
|
||||
validate_input_digest(input_digest)?;
|
||||
let saved: Option<String> = tx
|
||||
.query_row(
|
||||
"SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1",
|
||||
[&context.generation_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
if saved.as_deref() != Some(input_digest) {
|
||||
return Err(JournalError::Conflict(
|
||||
"trusted Host delivery of this immutable Job input is required".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Production sequential execution over Journal state and the Host-attested immutable input.
|
||||
/// The digest is taken from the bound saved Job by the Host, not from model arguments. Every
|
||||
/// request requires the matching delivery binding and an active generation/writer. Source text
|
||||
/// is already in immutable input: Source targets fail explicitly, with no Session reader or
|
||||
/// fallback. Receipt-backed record, position and checkpoint constraints remain unchanged.
|
||||
pub fn execute_snapshot(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
input_digest: &str,
|
||||
request: JournalJobRequest,
|
||||
) -> JournalResult<JournalJobResponse> {
|
||||
execute_inner(store, context, request, Some(input_digest), |_, _, _| {
|
||||
Err(JournalError::PermissionDenied(
|
||||
"Source reads are unavailable for immutable Journal input".into(),
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
/// Legacy/test source-read adapter. Production Hosts must use [`execute_snapshot`].
|
||||
/// This ledger attests bounded Source responses only and cannot satisfy the input-digest gate.
|
||||
pub fn execute_bounded(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
request: JournalJobRequest,
|
||||
read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult<recording::JournalMaterial>,
|
||||
) -> JournalResult<JournalJobResponse> {
|
||||
execute_inner(store, context, request, None, read_source)
|
||||
}
|
||||
|
||||
fn execute_inner(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
request: JournalJobRequest,
|
||||
input_digest: Option<&str>,
|
||||
mut read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult<recording::JournalMaterial>,
|
||||
) -> JournalResult<JournalJobResponse> {
|
||||
let generation = store
|
||||
.database
|
||||
.try_with_connection(|tx| super::store::job(store, tx, context, true))?;
|
||||
let generation = store.database.try_with_connection(|tx| {
|
||||
super::lifecycle::reject_erased(tx, &context.host)?;
|
||||
let generation = super::store::job(store, tx, context, true)?;
|
||||
if let Some(input_digest) = input_digest {
|
||||
require_input_delivery(tx, context, input_digest)?;
|
||||
}
|
||||
Ok::<_, JournalError>(generation)
|
||||
})?;
|
||||
let mut delivered = None;
|
||||
let response = match request {
|
||||
JournalJobRequest::Execute { operation } => {
|
||||
require_delivery(store, context, &generation.selected_ranges)?;
|
||||
if input_digest.is_none() {
|
||||
require_delivery(store, context, &generation.selected_ranges)?;
|
||||
}
|
||||
let receipt = if matches!(operation.action, JournalAction::Checkpoint { .. }) {
|
||||
crate::api::checkpoint_journal(store, context, operation)?
|
||||
} else {
|
||||
@@ -197,6 +323,11 @@ pub fn execute_bounded(
|
||||
JournalJobRequest::ReadText { read } => {
|
||||
let text = match &read.target {
|
||||
JournalTextTarget::Source { pointer } => {
|
||||
if input_digest.is_some() {
|
||||
return Err(JournalError::PermissionDenied(
|
||||
"Source target is unavailable: raw source bodies are in immutable Job input".into(),
|
||||
));
|
||||
}
|
||||
let range = generation
|
||||
.selected_ranges
|
||||
.iter()
|
||||
|
||||
@@ -202,7 +202,7 @@ pub(super) fn open(tx: &Connection, journal_id: &str) -> JournalResult<Option<Jo
|
||||
.optional()?;
|
||||
id.map(|id| entry(tx, journal_id, &id)).transpose()
|
||||
}
|
||||
fn previous_fixed(
|
||||
pub(super) fn previous_fixed(
|
||||
tx: &Connection,
|
||||
journal_id: &str,
|
||||
) -> JournalResult<Option<(JournalSourcePointer, u64)>> {
|
||||
@@ -569,6 +569,60 @@ impl SubjektivStore {
|
||||
Ok(snapshot)
|
||||
})
|
||||
}
|
||||
/// Rebind a saved body input to a new attempt without resampling its initial state.
|
||||
/// The first binding freezes the full initial Journal snapshot; retries reuse it and
|
||||
/// retain operation receipts. Different Jobs cannot borrow that snapshot as authority.
|
||||
pub fn prepare_journal_input(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
input: &super::recording::JournalInput,
|
||||
binding: JobAttemptBinding,
|
||||
) -> JournalResult<JournalGeneration> {
|
||||
label(&binding.job_id)?;
|
||||
label(&binding.attempt_id)?;
|
||||
self.database.try_transaction(|tx| {
|
||||
check_host(self, tx, host, false)?;
|
||||
check_session_host(self, tx, host)?;
|
||||
let current = journal(tx, host, &input.journal_id, true)?;
|
||||
let existing: Option<String> = tx.query_row(
|
||||
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
|
||||
params![input.journal_id, binding.job_id, binding.attempt_id], |r| r.get(0)).optional()?;
|
||||
if let Some(raw) = existing {
|
||||
let g = serde_json::from_str(&raw)?;
|
||||
input.validate_generation(&g)?;
|
||||
return Ok(g);
|
||||
}
|
||||
// Retained receipt lookup does not reacquire writer authority. Only a new
|
||||
// generation requires the current active Subject/Session writer lifetime.
|
||||
check_host(self, tx, host, true)?;
|
||||
if tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_generations WHERE journal_id=?1 AND active=1)", [&input.journal_id], |r| r.get::<_, bool>(0))? {
|
||||
return Err(conflict("another active generation owns this Journal"));
|
||||
}
|
||||
let prior: Option<String> = tx.query_row(
|
||||
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 ORDER BY rowid LIMIT 1",
|
||||
params![input.journal_id, binding.job_id], |r| r.get(0)).optional()?;
|
||||
let mut g = if let Some(raw) = prior {
|
||||
serde_json::from_str::<JournalGeneration>(&raw)?
|
||||
} else {
|
||||
JournalGeneration { id: String::new(), journal_id: input.journal_id.clone(), job: binding.clone(),
|
||||
selected_ranges: input.selected_ranges.clone(), initial_journal: current.clone(),
|
||||
initial_open: open(tx, &input.journal_id)?, previous_fixed_end: previous_fixed(tx, &input.journal_id)?.map(|p| p.0) }
|
||||
};
|
||||
input.validate_generation(&g)?;
|
||||
check_ranges(tx, ¤t, &input.selected_ranges, true)?;
|
||||
let end = prefix(&g.initial_journal, &input.selected_ranges)?;
|
||||
let cursor = (current.process_order, current.process_byte);
|
||||
if cursor < (g.initial_journal.process_order, g.initial_journal.process_byte) || cursor > end {
|
||||
return Err(conflict("saved input no longer owns the current processing cursor"));
|
||||
}
|
||||
g.id = issued_id("journal-generation");
|
||||
g.job = binding;
|
||||
tx.execute("INSERT INTO journal_generations(id,journal_id,job_id,attempt_id,request_json,snapshot_json,active) VALUES(?1,?2,?3,?4,?5,?6,1)",
|
||||
params![g.id,g.journal_id,g.job.job_id,g.job.attempt_id,serde_json::to_string(&g.selected_ranges)?,serde_json::to_string(&g)?])?;
|
||||
Ok(g)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn validate_journal_job(&self, ctx: &JournalJobContext) -> JournalResult<()> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
job(self, tx, ctx, true)?;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,10 @@
|
||||
//! Bounded, Host-bound Journal recording and sequential Job transport.
|
||||
//!
|
||||
//! Subject, Journal and generation authority never come from these model inputs.
|
||||
//! Hosts validate live execution and public source bytes before selection; reading a
|
||||
//! preview must not select the enclosing entry. Operation retries retain the entire
|
||||
//! Hosts validate live execution and public source bytes before selection, save body-bearing
|
||||
//! immutable Job input, and attest its exact digest and dispatch plan through the service.
|
||||
//! Model requests cannot attest input delivery; Source targets are rejected by snapshot execution.
|
||||
//! Initial/current Journal reads remain separate from input delivery. Operation retries retain the entire
|
||||
//! JournalOperation, including its original expected_state and covered_ranges.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -50,8 +52,8 @@ pub struct JournalJobState {
|
||||
pub current_journal: JournalMetadata,
|
||||
pub current_open: Option<JournalEntrySummary>,
|
||||
pub previous_fixed_end: Option<JournalSourcePointer>,
|
||||
/// Frozen permission to fetch this prefix, not evidence of delivery or reflection.
|
||||
/// The sequential service gates mutations on its durable Source-read delivery ledger.
|
||||
/// Frozen source selection delivered in immutable Job input, not evidence of reflection.
|
||||
/// Production execution requires the trusted Host's matching input-digest attestation.
|
||||
pub selected_ranges: Vec<JournalSourceRange>,
|
||||
}
|
||||
|
||||
@@ -83,7 +85,8 @@ pub enum JournalTextTarget {
|
||||
OpenOngoing {
|
||||
snapshot: JournalSnapshot,
|
||||
},
|
||||
/// The Host restricts this pointer to this Job's frozen public source input.
|
||||
/// Legacy source-read adapter only. Production snapshot execution rejects this target;
|
||||
/// raw source bodies must already be present in the immutable Job input.
|
||||
Source {
|
||||
pointer: JournalSourcePointer,
|
||||
},
|
||||
|
||||
@@ -639,6 +639,11 @@ static MIGRATIONS: &[FeatureMigration] = &[
|
||||
"add explicit Subject permission sources and immutable child execution grants",
|
||||
permissions::migrate,
|
||||
),
|
||||
FeatureMigration::new(
|
||||
13,
|
||||
"bind Journal generation delivery to immutable Job input digest",
|
||||
journal::service::migrate_input_delivery,
|
||||
),
|
||||
];
|
||||
|
||||
pub const REGISTRATION: FeatureRegistration =
|
||||
|
||||
@@ -444,6 +444,31 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
material_pin(&self.state, request, true)
|
||||
}
|
||||
|
||||
async fn session_recording_capture(
|
||||
&self,
|
||||
request: runtime_api::SessionRecordingCaptureRequest,
|
||||
) -> Result<runtime_api::SessionRecordingCaptureAvailability, runtime_api::RuntimeApiError>
|
||||
{
|
||||
let scope = material_scope(&self.state, &request.workspace_id)?;
|
||||
#[cfg(feature = "fs-store")]
|
||||
{
|
||||
Ok(self
|
||||
.state
|
||||
.runtime
|
||||
.session_recording_capture_scoped(&scope, &request))
|
||||
}
|
||||
#[cfg(not(feature = "fs-store"))]
|
||||
{
|
||||
let _ = scope;
|
||||
Ok(
|
||||
runtime_api::SessionRecordingCaptureAvailability::Unavailable {
|
||||
reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable,
|
||||
message: "Committed Session storage unavailable".into(),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async fn session_material_catalog(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialCatalogRequest,
|
||||
|
||||
@@ -150,6 +150,27 @@ impl Fixture {
|
||||
limit: 1,
|
||||
}
|
||||
}
|
||||
fn recording_request(&self) -> runtime_api::SessionRecordingCaptureRequest {
|
||||
runtime_api::SessionRecordingCaptureRequest {
|
||||
workspace_id: self.scope.workspace_id.clone(),
|
||||
source: self.source(),
|
||||
expected_session_id: self.session.to_string(),
|
||||
start: Default::default(),
|
||||
max_entries: 1000,
|
||||
max_serialized_bytes: 128 * 1024,
|
||||
}
|
||||
}
|
||||
fn capture(&self, source: Source) -> session_store::capture::RecordingCaptureSlice {
|
||||
let mut request = self.recording_request();
|
||||
request.source = source;
|
||||
match self
|
||||
.runtime
|
||||
.session_recording_capture_scoped(&self.scope, &request)
|
||||
{
|
||||
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice,
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
fn catalog(&self) -> runtime_api::SessionMaterialCatalogPage {
|
||||
match self
|
||||
.runtime
|
||||
@@ -226,6 +247,9 @@ fn pending_session_proof_is_not_missing_material_and_pin_release_allows_empty_ar
|
||||
.is_err(),
|
||||
"retention intent is not source availability"
|
||||
);
|
||||
assert!(
|
||||
matches!(f.runtime.session_recording_capture_scoped(&f.scope, &f.recording_request()), runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id } if session_id == f.session.to_string())
|
||||
);
|
||||
// A missing committed source is never the same fact as a pending pointer.
|
||||
metadata
|
||||
.write(
|
||||
@@ -479,6 +503,7 @@ fn source_scope_session_and_active_pointer_mismatches_never_return_material() {
|
||||
#[test]
|
||||
fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() {
|
||||
let f = Fixture::new();
|
||||
let frozen = f.capture(f.source());
|
||||
let original: Vec<_> = f.reader(&f.source()).descriptors().cloned().collect();
|
||||
let pin = SessionRetentionPin(uuid::Uuid::now_v7());
|
||||
f.runtime
|
||||
@@ -516,6 +541,7 @@ fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source()
|
||||
let archived = Source::Archived {
|
||||
archive_id: "canonical-archive".into(),
|
||||
};
|
||||
assert_eq!(f.capture(archived.clone()), frozen);
|
||||
let reader = f.reader(&archived);
|
||||
assert_eq!(reader.descriptors().cloned().collect::<Vec<_>>(), original);
|
||||
let body = reader.read_content(&original[0], 0, 16384).unwrap();
|
||||
@@ -951,11 +977,52 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router
|
||||
.authorizer(SignedAuthorizer { identity, trust })
|
||||
.build()
|
||||
.unwrap();
|
||||
let frozen = match client
|
||||
.session_recording_capture(f.recording_request())
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice,
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
frozen.capture.validate(Default::default()).unwrap();
|
||||
assert!(frozen.next.is_none());
|
||||
assert!(
|
||||
serde_json::to_vec(&frozen).unwrap().len()
|
||||
<= f.recording_request().max_serialized_bytes
|
||||
);
|
||||
let snapshot = serde_json::to_vec(&frozen).unwrap();
|
||||
assert!(!String::from_utf8_lossy(&snapshot).contains("private-system-prompt"));
|
||||
f.store
|
||||
.append(
|
||||
f.session,
|
||||
f.segment,
|
||||
&LogEntry::AnnotatedAssistantItem {
|
||||
ts: 6,
|
||||
entry: message("late-answer", LoggedRole::Assistant, "not in frozen input"),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(serde_json::to_vec(&frozen).unwrap(), snapshot);
|
||||
let page = match client.session_material_catalog(f.request()).await.unwrap() {
|
||||
Catalog::Page { page } => page,
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
let descriptor = page.descriptors[0].clone();
|
||||
assert_eq!(
|
||||
frozen
|
||||
.capture
|
||||
.recording_text(0)
|
||||
.unwrap()
|
||||
.0
|
||||
.identity
|
||||
.entry_ref,
|
||||
descriptor.entry_ref
|
||||
);
|
||||
assert_eq!(
|
||||
frozen.capture.recording_text(0).unwrap().0.sha256,
|
||||
descriptor.sha256
|
||||
);
|
||||
let read = client
|
||||
.session_material_read(runtime_api::SessionMaterialReadRequest {
|
||||
workspace_id: "workspace-a".into(),
|
||||
@@ -1019,6 +1086,13 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router
|
||||
);
|
||||
}
|
||||
assert!(!f.root().exists());
|
||||
assert!(matches!(
|
||||
client
|
||||
.session_recording_capture(f.recording_request())
|
||||
.await
|
||||
.unwrap(),
|
||||
runtime_api::SessionRecordingCaptureAvailability::Unavailable { .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
client.session_material_catalog(f.request()).await.unwrap(),
|
||||
Catalog::Unavailable { .. }
|
||||
@@ -1050,41 +1124,54 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() {
|
||||
None,
|
||||
workspace_test_auth(vec![trust.clone()]),
|
||||
);
|
||||
let path = "/v1/session-public/material/catalog";
|
||||
let body = serde_json::to_vec(&f.request()).unwrap();
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
let first = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||
assert_eq!(first.status(), StatusCode::OK);
|
||||
let replay = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||
assert_eq!(replay.status(), StatusCode::UNAUTHORIZED);
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
let changed = signed_request(
|
||||
app.clone(),
|
||||
&identity,
|
||||
&claims,
|
||||
b"{}".to_vec(),
|
||||
"workspace-a",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(changed.status(), StatusCode::UNAUTHORIZED);
|
||||
let mut wrong_scope = f.request();
|
||||
wrong_scope.workspace_id = "other-workspace".into();
|
||||
let wrong_body = serde_json::to_vec(&wrong_scope).unwrap();
|
||||
let claims = signed_claims(&trust, "POST", path, &wrong_body);
|
||||
let response = signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||
let unsigned = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(path)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(body))
|
||||
.unwrap(),
|
||||
for (path, body) in [
|
||||
(
|
||||
"/v1/session-public/material/catalog",
|
||||
serde_json::to_vec(&f.request()).unwrap(),
|
||||
),
|
||||
(
|
||||
"/v1/session-public/recording-capture",
|
||||
serde_json::to_vec(&f.recording_request()).unwrap(),
|
||||
),
|
||||
] {
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
let first =
|
||||
signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||
assert_eq!(first.status(), StatusCode::OK);
|
||||
let replay =
|
||||
signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||
assert_eq!(replay.status(), StatusCode::UNAUTHORIZED);
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
let changed = signed_request(
|
||||
app.clone(),
|
||||
&identity,
|
||||
&claims,
|
||||
b"{}".to_vec(),
|
||||
"workspace-a",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
|
||||
.await;
|
||||
assert_eq!(changed.status(), StatusCode::UNAUTHORIZED);
|
||||
let mut wrong_scope: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
wrong_scope["workspace_id"] = "other-workspace".into();
|
||||
let wrong_body = serde_json::to_vec(&wrong_scope).unwrap();
|
||||
let claims = signed_claims(&trust, "POST", path, &wrong_body);
|
||||
let response =
|
||||
signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||
let unsigned = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(path)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(body))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1170,3 +1257,152 @@ async fn explicit_material_erasure_requires_delete_capability_and_matching_scope
|
||||
);
|
||||
assert!(f.root().exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recording_capture_rejects_scope_session_cursor_and_budget_without_returning_bodies() {
|
||||
use runtime_api::SessionRecordingCaptureAvailability as Capture;
|
||||
let f = Fixture::new();
|
||||
let request = f.recording_request();
|
||||
let frozen = f.capture(f.source());
|
||||
assert_eq!(frozen.capture.entries.len(), 3);
|
||||
let mut request = request;
|
||||
for invalid in 0..5 {
|
||||
let mut bad = request.clone();
|
||||
match invalid {
|
||||
0 => bad.workspace_id = "other".into(),
|
||||
1 => bad.expected_session_id = session_store::new_session_id().to_string(),
|
||||
2 => bad.start.canonical_order = 100,
|
||||
3 => bad.max_entries = 0,
|
||||
4 => bad.max_serialized_bytes = 4 * 1024 * 1024 + 1,
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(matches!(
|
||||
f.runtime.session_recording_capture_scoped(&f.scope, &bad),
|
||||
Capture::Unavailable { .. }
|
||||
));
|
||||
}
|
||||
let text = frozen.capture.recording_text(1).unwrap().2;
|
||||
request.start = session_store::capture::RecordingPosition {
|
||||
canonical_order: 1,
|
||||
byte_offset: text.find('🍎').unwrap() as u64 + 1,
|
||||
};
|
||||
assert!(matches!(
|
||||
f.runtime
|
||||
.session_recording_capture_scoped(&f.scope, &request),
|
||||
Capture::Unavailable {
|
||||
reason: Reason::InvalidCursor,
|
||||
..
|
||||
}
|
||||
));
|
||||
request.start = Default::default();
|
||||
request.max_entries = 1;
|
||||
let Capture::Captured { slice } = f
|
||||
.runtime
|
||||
.session_recording_capture_scoped(&f.scope, &request)
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(slice.capture.entries.len(), 1);
|
||||
assert_eq!(slice.next.unwrap().canonical_order, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recording_capture_compaction_preserves_original_descriptors_and_excludes_summary_and_new_tail() {
|
||||
let f = Fixture::new();
|
||||
let old = f.capture(f.source());
|
||||
let compact = session_store::new_segment_id();
|
||||
let mut summary = message(
|
||||
"summary",
|
||||
LoggedRole::System,
|
||||
"derived summary is not original",
|
||||
);
|
||||
summary.metadata.input_metadata = Some(
|
||||
session_store::HistoryInputMetadata::new(
|
||||
session_store::HistoryInputUsePolicy {
|
||||
recording_material: session_store::InputEvidenceUse::Exclude,
|
||||
memory_evidence: session_store::InputEvidenceUse::Exclude,
|
||||
observation: session_store::InputObservationUse::Hidden,
|
||||
compaction: session_store::InputCompactionUse::OpeningSummary,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
summary.metadata.derivation = Some(session_store::LoggedHistoryDerivation {
|
||||
sources: vec![
|
||||
LoggedSessionHistoryEntryId("user-1".into()),
|
||||
LoggedSessionHistoryEntryId("answer-1".into()),
|
||||
],
|
||||
});
|
||||
f.store
|
||||
.create_segment(
|
||||
f.session,
|
||||
compact,
|
||||
&[
|
||||
LogEntry::AnnotatedSegmentStart {
|
||||
ts: 6,
|
||||
session_id: f.session,
|
||||
system_prompt: Some("private".into()),
|
||||
config: Default::default(),
|
||||
history: vec![summary],
|
||||
forked_from: None,
|
||||
compacted_from: Some(session_store::SegmentOrigin {
|
||||
segment_id: f.segment,
|
||||
at_turn_index: 0,
|
||||
}),
|
||||
},
|
||||
LogEntry::AnnotatedAssistantItem {
|
||||
ts: 7,
|
||||
entry: message(
|
||||
"new-answer",
|
||||
LoggedRole::Assistant,
|
||||
"new original after compaction",
|
||||
),
|
||||
},
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
WorkerAggregateStore::new(
|
||||
f.root().parent().unwrap(),
|
||||
format!("worker-runtime-{}", f.worker.worker_id),
|
||||
)
|
||||
.unwrap()
|
||||
.set_active(
|
||||
&format!("worker-runtime-{}", f.worker.worker_id),
|
||||
Some(session_store::WorkerActiveSegmentRef::active_segment(
|
||||
f.session, compact,
|
||||
)),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let frozen = f.capture(f.source());
|
||||
assert_eq!(frozen.capture.entries.len(), old.capture.entries.len() + 1);
|
||||
for entry in &old.capture.entries {
|
||||
assert_eq!(
|
||||
frozen.capture.recording_text(entry.order).unwrap(),
|
||||
old.capture.recording_text(entry.order).unwrap()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
!serde_json::to_string(&frozen)
|
||||
.unwrap()
|
||||
.contains("derived summary is not original")
|
||||
);
|
||||
let bytes = serde_json::to_vec(&frozen).unwrap();
|
||||
f.store
|
||||
.append(
|
||||
f.session,
|
||||
compact,
|
||||
&LogEntry::AnnotatedAssistantItem {
|
||||
ts: 8,
|
||||
entry: message("tail", LoggedRole::Assistant, "next Job only"),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(serde_json::to_vec(&frozen).unwrap(), bytes);
|
||||
assert_eq!(
|
||||
f.capture(f.source()).capture.entries.len(),
|
||||
frozen.capture.entries.len() + 1
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4949,6 +4949,133 @@ impl Runtime {
|
||||
.map_err(crate::retention::retention_error)
|
||||
}
|
||||
|
||||
/// Freeze committed original public bodies once; no catalog/read pagination
|
||||
/// or restored Worker is involved. Publication and erasure share this fence.
|
||||
#[cfg(feature = "fs-store")]
|
||||
pub fn session_recording_capture_scoped(
|
||||
&self,
|
||||
scope: &RuntimeWorkspaceScope,
|
||||
request: &runtime_api::SessionRecordingCaptureRequest,
|
||||
) -> runtime_api::SessionRecordingCaptureAvailability {
|
||||
use crate::worker_backend::SessionMaterialSourceError as Error;
|
||||
use runtime_api::{
|
||||
SessionPublicUnavailableReason as R, SessionRecordingCaptureAvailability as A,
|
||||
};
|
||||
use session_store::public_material::{PublicMaterialError as E, PublicMaterialLimits};
|
||||
let result = (|| -> Result<_, Error> {
|
||||
if scope.workspace_id != request.workspace_id {
|
||||
return Err("Session Workspace scope mismatch".into());
|
||||
}
|
||||
if !(1..=4096).contains(&request.max_entries)
|
||||
|| !(1024..=4 * 1024 * 1024).contains(&request.max_serialized_bytes)
|
||||
{
|
||||
return Err(E::ResourceLimit.into());
|
||||
}
|
||||
let session = request
|
||||
.expected_session_id
|
||||
.parse::<session_store::SessionId>()
|
||||
.map_err(|_| "Invalid Session identity")?;
|
||||
// Resolve attribution without reading archive bodies. Do not acquire
|
||||
// the Runtime mutex under the Session lock: archive execution takes
|
||||
// those locks in the opposite order.
|
||||
let (root, archive) = self
|
||||
.resolve_session_public_source_with_validation(scope, &request.source, false)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let runtime_dir = {
|
||||
let state = self.lock().map_err(|e| e.to_string())?;
|
||||
state
|
||||
.fs_store()
|
||||
.ok_or("Session storage unavailable")?
|
||||
.runtime_dir()
|
||||
.to_path_buf()
|
||||
};
|
||||
// This is the storage append fence used by live Runtime publishers,
|
||||
// also fencing archive transfer and erase. Resolve the pointer under it.
|
||||
let retention = crate::retention::runtime_session_retention(&runtime_dir);
|
||||
let guard = retention.lock_live(session).map_err(|error| match error {
|
||||
session_store::StoreError::SessionErased(_) => E::Erased,
|
||||
_ => E::Unavailable,
|
||||
})?;
|
||||
let mut archive_to_verify = None;
|
||||
let active = match (&request.source, archive) {
|
||||
(runtime_api::SessionPublicSource::Retained { worker_id }, None) => {
|
||||
let metadata = session_store::worker_metadata::WorkerAggregateStore::open_read_only(
|
||||
root.parent().ok_or("Session root is unavailable")?,
|
||||
crate::worker_backend::ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&WorkerRef::new(*worker_id)),
|
||||
).and_then(|store| store.read_read_only()).map_err(|e| e.to_string())?.ok_or("Session metadata missing")?;
|
||||
if metadata.workspace_id.as_deref() != Some(scope.workspace_id.as_str()) {
|
||||
return Err("Session Workspace mismatch".into());
|
||||
}
|
||||
let active = metadata.active.ok_or("Session active pointer missing")?;
|
||||
if active.session_id != session {
|
||||
return Err("Session identity mismatch".into());
|
||||
}
|
||||
match active.segment_id {
|
||||
Some(segment) => segment,
|
||||
None if request.start
|
||||
== session_store::capture::RecordingPosition::default()
|
||||
&& crate::retention::unmaterialized_session(
|
||||
root.parent().ok_or("Session root unavailable")?,
|
||||
&scope.workspace_id,
|
||||
)
|
||||
.map_err(|e| e.to_string())?
|
||||
== Some(session) =>
|
||||
{
|
||||
return Ok(A::NotMaterialized {
|
||||
session_id: session.to_string(),
|
||||
});
|
||||
}
|
||||
None => return Err("Session active Segment missing".into()),
|
||||
}
|
||||
}
|
||||
(runtime_api::SessionPublicSource::Archived { archive_id }, Some(manifest)) => {
|
||||
archive_to_verify = Some((archive_id, manifest.clone()));
|
||||
if manifest.source_session_id != request.expected_session_id {
|
||||
return Err("Archive Session identity mismatch".into());
|
||||
}
|
||||
manifest
|
||||
.source_active_segment_id
|
||||
.ok_or("Archive active Segment missing")?
|
||||
}
|
||||
_ => return Err("Session source unavailable".into()),
|
||||
};
|
||||
let slice = session_store::capture::capture_recording_slice_worker_guarded(
|
||||
&root,
|
||||
session,
|
||||
active,
|
||||
PublicMaterialLimits::default(),
|
||||
request.start,
|
||||
request.max_entries,
|
||||
request.max_serialized_bytes,
|
||||
&guard,
|
||||
)?;
|
||||
if let Some((archive_id, manifest)) = archive_to_verify {
|
||||
// Validate archive checksums only after capture froze every file
|
||||
// end, still under the same publication/erase fence.
|
||||
let (verified, verified_root) = FsWorkerRetentionProvider::new(&runtime_dir)
|
||||
.session_archive(archive_id)
|
||||
.map_err(|e| e.to_string())?;
|
||||
if verified != manifest || verified_root != root {
|
||||
return Err(E::Changed.into());
|
||||
}
|
||||
}
|
||||
Ok(A::Captured { slice })
|
||||
})();
|
||||
match result {
|
||||
Ok(availability) => availability,
|
||||
Err(error) => A::Unavailable {
|
||||
reason: match error {
|
||||
Error::Material(E::ResourceLimit) => R::ResourceLimit,
|
||||
Error::Material(E::InvalidOffset | E::Changed) => R::InvalidCursor,
|
||||
Error::Material(E::Missing | E::Erased) => R::RetentionMissing,
|
||||
Error::Material(E::Inconsistent | E::Record(_)) => R::CorruptLog,
|
||||
_ => R::StorageUnavailable,
|
||||
},
|
||||
message: "Committed Session recording capture unavailable".into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
pub fn session_material_catalog_scoped(
|
||||
&self,
|
||||
|
||||
@@ -730,7 +730,7 @@ impl ProfileRuntimeWorkerFactory {
|
||||
.map_err(|error| format!("failed to clean restore run artifacts: {error}"))
|
||||
}
|
||||
|
||||
fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String {
|
||||
pub(crate) fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String {
|
||||
format!("worker-runtime-{}", worker_ref.worker_id)
|
||||
}
|
||||
|
||||
|
||||
@@ -179,11 +179,11 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{}})),
|
||||
prose(),
|
||||
]);
|
||||
// This runner/Feature fixture has no selected sources; descriptor delivery gates
|
||||
// for nonempty ranges are owned and tested by the Subjektiv service.
|
||||
// This runner/Feature fixture has no selected sources; actual body-bearing input,
|
||||
// trusted delivery and checkpoint gates are covered by the Host integration suites.
|
||||
let mut journal_request = request("builtin:subjektiv-journal-recording");
|
||||
journal_request.input = json!({"selected_ranges":[],"origins":[]});
|
||||
journal_request.instruction = "Read immutable selected_ranges through ReadJournalText Source before recording; descriptors are not source bodies.".into();
|
||||
journal_request.input = json!({"selected_ranges":[],"capture":{"entries":[]}});
|
||||
journal_request.instruction = "Record only the immutable canonical bodies in this Job input. Do not read original Sessions through ReadJournalText Source.".into();
|
||||
let job = prepare_job_from_registry_with_grant(
|
||||
&journal_request,
|
||||
root.path(),
|
||||
@@ -249,7 +249,7 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s
|
||||
"ReadJournalText",
|
||||
"Source",
|
||||
"content_token",
|
||||
"Conflict",
|
||||
"input digest",
|
||||
] {
|
||||
assert!(
|
||||
prompt.contains(contract),
|
||||
|
||||
@@ -476,7 +476,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
(99, "Explicit hosting administrators and self-registration policy".into()),
|
||||
(100, "Provisional users and scoped passkey activation".into()),
|
||||
(101, "Durable explicit Subject body release".into()),
|
||||
(102, "Durable failed Subject creation recovery".into())
|
||||
(102, "Durable failed Subject creation recovery".into()),
|
||||
(103, "Job input retirement independent of immutable receipts".into())
|
||||
]
|
||||
);
|
||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||
|
||||
@@ -463,10 +463,52 @@ impl BackendJobRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Availability of the immutable input body, independent of execution outcome.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobInputDisposition {
|
||||
Available,
|
||||
Reclaimed,
|
||||
Erased,
|
||||
Invalidated,
|
||||
}
|
||||
|
||||
impl JobInputDisposition {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Available => "available",
|
||||
Self::Reclaimed => "reclaimed",
|
||||
Self::Erased => "erased",
|
||||
Self::Invalidated => "invalidated",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum JobInputRetirementReason {
|
||||
Completed,
|
||||
ExplicitErasure,
|
||||
LegacyInvalidation,
|
||||
}
|
||||
|
||||
impl JobInputRetirementReason {
|
||||
pub fn disposition(self) -> JobInputDisposition {
|
||||
match self {
|
||||
Self::Completed => JobInputDisposition::Reclaimed,
|
||||
Self::ExplicitErasure => JobInputDisposition::Erased,
|
||||
Self::LegacyInvalidation => JobInputDisposition::Invalidated,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct BackendJobRecord {
|
||||
pub workspace_id: String,
|
||||
/// Receipt metadata. After retirement `input` is null; use the verified loader to execute.
|
||||
pub request: BackendJobRequest,
|
||||
pub input_digest: String,
|
||||
pub input_disposition: JobInputDisposition,
|
||||
pub intent_fingerprint: String,
|
||||
pub state: BackendJobState,
|
||||
pub current_attempt: u8,
|
||||
@@ -479,6 +521,28 @@ pub struct BackendJobRecord {
|
||||
pub completed_at: Option<String>,
|
||||
}
|
||||
|
||||
impl BackendJobRecord {
|
||||
/// Verify the original immutable body and envelope before any execution or retry.
|
||||
/// No Session read or replay request may repair unavailable/corrupt input.
|
||||
pub fn load_executable_request(&self) -> Result<&BackendJobRequest> {
|
||||
if self.input_disposition != JobInputDisposition::Available {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job input is {}",
|
||||
self.input_disposition.as_str()
|
||||
)));
|
||||
}
|
||||
self.request.validate()?;
|
||||
if self.request.input_digest()? != self.input_digest
|
||||
|| self.request.fingerprint()? != self.intent_fingerprint
|
||||
{
|
||||
return Err(Error::Store(
|
||||
"Backend Job immutable input or intent fingerprint mismatch".into(),
|
||||
));
|
||||
}
|
||||
Ok(&self.request)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BackendJobWorkerCleanupState {
|
||||
@@ -665,6 +729,8 @@ mod tests {
|
||||
"workspace_id": "workspace-a",
|
||||
"request": request(),
|
||||
"intent_fingerprint": request().fingerprint().unwrap(),
|
||||
"input_digest": request().input_digest().unwrap(),
|
||||
"input_disposition": "available",
|
||||
"state": "pending",
|
||||
"current_attempt": 2,
|
||||
"created_at": "created",
|
||||
@@ -1124,3 +1190,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const INPUT_RETENTION_TRIGGERS: &str = r#"
|
||||
CREATE TRIGGER backend_job_identity_immutable BEFORE UPDATE OF
|
||||
workspace_id, job_id, purpose, input_digest, input_ref, resource_key, intent_fingerprint ON backend_jobs
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job identity'); END;
|
||||
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON backend_jobs
|
||||
WHEN (
|
||||
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
|
||||
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
|
||||
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
|
||||
AND json_type(NEW.request_json,'$.input') = 'null'
|
||||
) IS NOT TRUE
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job input retirement'); END;
|
||||
"#;
|
||||
|
||||
@@ -71,6 +71,11 @@ fn legacy_content_database(path: &Path) -> (Connection, String) {
|
||||
conn.execute("INSERT INTO workspace_config_tree_revisions VALUES('space',?1,?2,'toolchain','projection',?3,?4,'{}')",
|
||||
params![ordinal,snapshot.digest,manifest,created]).unwrap();
|
||||
}
|
||||
seed_legacy_content_job(&conn);
|
||||
(conn, snapshot.digest)
|
||||
}
|
||||
|
||||
fn seed_legacy_content_job(conn: &Connection) {
|
||||
let request = BackendJobRequest {
|
||||
job_id: "legacy-job".into(),
|
||||
purpose: "migration_test".into(),
|
||||
@@ -93,7 +98,19 @@ fn legacy_content_database(path: &Path) -> (Connection, String) {
|
||||
VALUES('space','legacy-job','legacy-job:attempt:1',1,'old-number','dispatched','runtime','worker','2026-08-14T00:00:00Z','2026-08-13T00:00:00Z','2026-08-13T00:00:00Z');
|
||||
INSERT INTO backend_job_deliveries(workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,created_at,updated_at)
|
||||
VALUES('space','delivery','legacy-job','legacy-job:attempt:1','runtime','target','pending','created','updated');") .unwrap();
|
||||
(conn, snapshot.digest)
|
||||
}
|
||||
|
||||
// Replay/resource checks use the current Store only after the full retained chain.
|
||||
// Keep the focused partial DDL fixture above for the cutover's schema assertions.
|
||||
fn legacy_content_job_database(path: &Path) -> Connection {
|
||||
super::tests::prepare_retained_schema(path, 85);
|
||||
let conn = Connection::open(path).unwrap();
|
||||
configure_sqlite(&conn).unwrap();
|
||||
conn.execute_batch("INSERT INTO accounts VALUES('owner','user','owner','Owner','created','updated');
|
||||
INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at)
|
||||
VALUES('space','owner','Space','active','created','updated');").unwrap();
|
||||
seed_legacy_content_job(&conn);
|
||||
conn
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -432,7 +449,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
|
||||
] {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("content.db");
|
||||
let (conn, _) = legacy_content_database(&path);
|
||||
let conn = legacy_content_job_database(&path);
|
||||
conn.execute(
|
||||
"UPDATE backend_job_attempts SET state=?1,
|
||||
runtime_id=CASE WHEN ?1='reserved' THEN NULL ELSE runtime_id END,
|
||||
@@ -441,11 +458,22 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
|
||||
)
|
||||
.unwrap();
|
||||
migrate_content_state_v85_to_v86(&conn).unwrap();
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 86);
|
||||
let cutover: (String, String, Option<String>) = conn.query_row(
|
||||
"SELECT j.state,a.state,a.worker_cleanup_state FROM backend_jobs j JOIN backend_job_attempts a USING(workspace_id,job_id)",
|
||||
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
).unwrap();
|
||||
assert_eq!(
|
||||
cutover,
|
||||
if state == "reserved" {
|
||||
("failed".into(), "failed".into(), Some("completed".into()))
|
||||
} else {
|
||||
("unknown".into(), "unknown".into(), Some("pending".into()))
|
||||
}
|
||||
);
|
||||
apply_migrations(&conn).unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
|
||||
};
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let job = store
|
||||
.get_backend_job("space", "legacy-job")
|
||||
.unwrap()
|
||||
@@ -478,7 +506,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_
|
||||
fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("content.db");
|
||||
let (conn, _) = legacy_content_database(&path);
|
||||
let conn = legacy_content_job_database(&path);
|
||||
let result = serde_json::json!({"finding":"preserved"});
|
||||
let (digest, encoded) = job::result_digest(&result, 4096).unwrap();
|
||||
conn.execute("UPDATE backend_jobs SET state='completed',result_json=?1,result_digest=?2,completed_at='finished'",params![encoded,digest]).unwrap();
|
||||
@@ -490,11 +518,23 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() {
|
||||
)
|
||||
.unwrap();
|
||||
migrate_content_state_v85_to_v86(&conn).unwrap();
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 86);
|
||||
let cutover: (String, String, String, String) = conn.query_row(
|
||||
"SELECT j.state,j.result_json,a.worker_cleanup_state,a.worker_cleanup_completed_at FROM backend_jobs j JOIN backend_job_attempts a USING(workspace_id,job_id)",
|
||||
[], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
).unwrap();
|
||||
assert_eq!(
|
||||
cutover,
|
||||
(
|
||||
"completed".into(),
|
||||
encoded,
|
||||
"completed".into(),
|
||||
"cleaned".into()
|
||||
)
|
||||
);
|
||||
apply_migrations(&conn).unwrap();
|
||||
drop(conn);
|
||||
let store = SqliteWorkspaceStore {
|
||||
db_executor: crate::db_executor::DbExecutor::shared(),
|
||||
conn: Arc::new(Mutex::new(Connection::open(path).unwrap())),
|
||||
};
|
||||
let store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let job = store
|
||||
.get_backend_job("space", "legacy-job")
|
||||
.unwrap()
|
||||
|
||||
@@ -1326,6 +1326,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
Err("Session source permanent erasure is unavailable on this Runtime".into())
|
||||
}
|
||||
|
||||
fn session_recording_capture(
|
||||
&self,
|
||||
_request: runtime_api::SessionRecordingCaptureRequest,
|
||||
) -> Result<runtime_api::SessionRecordingCaptureAvailability, String> {
|
||||
Err("committed recording capture is unavailable on this Runtime".into())
|
||||
}
|
||||
|
||||
fn session_material_catalog(
|
||||
&self,
|
||||
_request: runtime_api::SessionMaterialCatalogRequest,
|
||||
@@ -2336,6 +2343,21 @@ impl RuntimeRegistry {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn session_recording_capture(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
request: runtime_api::SessionRecordingCaptureRequest,
|
||||
) -> Result<runtime_api::SessionRecordingCaptureAvailability, RuntimeRegistryError> {
|
||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||
self.runtime(runtime_id)?
|
||||
.session_recording_capture(request)
|
||||
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: runtime_id.into(),
|
||||
code: "session_recording_capture_failed".into(),
|
||||
message,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn session_material_catalog(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -3441,6 +3463,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
fn session_recording_capture(
|
||||
&self,
|
||||
request: runtime_api::SessionRecordingCaptureRequest,
|
||||
) -> Result<runtime_api::SessionRecordingCaptureAvailability, String> {
|
||||
if request.workspace_id != self.workspace_id {
|
||||
return Err("recording capture Workspace mismatch".into());
|
||||
}
|
||||
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
|
||||
Ok(self
|
||||
.runtime
|
||||
.session_recording_capture_scoped(&scope, &request))
|
||||
}
|
||||
|
||||
fn session_material_catalog(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialCatalogRequest,
|
||||
@@ -5677,6 +5712,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
.map_err(|diagnostic| diagnostic.message)
|
||||
}
|
||||
|
||||
fn session_recording_capture(
|
||||
&self,
|
||||
request: runtime_api::SessionRecordingCaptureRequest,
|
||||
) -> Result<runtime_api::SessionRecordingCaptureAvailability, String> {
|
||||
self.run_runtime_api(
|
||||
self.request_timeout,
|
||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||
move |client| async move { client.session_recording_capture(request).await },
|
||||
)
|
||||
.map_err(|diagnostic| diagnostic.message)
|
||||
}
|
||||
|
||||
fn session_material_catalog(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialCatalogRequest,
|
||||
|
||||
@@ -381,6 +381,10 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
|
||||
DROP TABLE ticket_implementation_results;
|
||||
DROP TABLE ticket_implementation_delegations;
|
||||
DROP TABLE merge_request_source_observations;
|
||||
DROP TRIGGER backend_job_input_retirement;
|
||||
DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
||||
DROP TABLE subject_create_recoveries;
|
||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||
DROP TABLE subject_worker_releases;
|
||||
@@ -517,6 +521,10 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
|
||||
// Reconstruct the prior composition before its new business capture existed.
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TRIGGER backend_job_input_retirement;
|
||||
DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
||||
DROP TABLE subject_create_recoveries;
|
||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||
DROP TABLE subject_worker_releases;
|
||||
@@ -679,7 +687,11 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
|
||||
if historical {
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TABLE subject_create_recoveries;
|
||||
DROP TRIGGER backend_job_input_retirement;
|
||||
DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
||||
DROP TABLE subject_create_recoveries;
|
||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
//! lifecycle, are the durable source. Fixed Worker recipients never follow leases.
|
||||
use super::*;
|
||||
use crate::backend_job::{
|
||||
BackendJobAttemptState, BackendJobDeliveryRecord, BackendJobDeliveryState,
|
||||
BackendJobResultAcceptance, BackendJobState,
|
||||
BackendJobAttemptState, BackendJobDeliveryRecord, BackendJobDeliveryState, BackendJobRequest,
|
||||
BackendJobState,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -103,6 +103,32 @@ pub(in crate::store) fn install(conn: &Connection) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct NotificationResultSnapshot {
|
||||
request: BackendJobRequest,
|
||||
result: Option<serde_json::Value>,
|
||||
result_digest: Option<String>,
|
||||
}
|
||||
|
||||
// The schema-96 import runs before later Job lifecycle migrations. Read only
|
||||
// the retained result envelope, not the executable Job's current-schema columns.
|
||||
fn read_result_snapshot(
|
||||
conn: &Connection,
|
||||
workspace: &str,
|
||||
job: &str,
|
||||
) -> Result<NotificationResultSnapshot> {
|
||||
let (request, result, result_digest): (String, Option<String>, Option<String>) = conn
|
||||
.query_row(
|
||||
"SELECT request_json, result_json, result_digest FROM backend_jobs WHERE workspace_id=?1 AND job_id=?2",
|
||||
params![workspace, job],
|
||||
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
|
||||
)?;
|
||||
Ok(NotificationResultSnapshot {
|
||||
request: decode(&request)?,
|
||||
result: result.map(|value| decode(&value)).transpose()?,
|
||||
result_digest,
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::store) fn capture(
|
||||
conn: &Connection,
|
||||
workspace: &str,
|
||||
@@ -115,8 +141,7 @@ pub(in crate::store) fn capture(
|
||||
attempt_id: attempt.into(),
|
||||
delivery_id: id.clone(),
|
||||
};
|
||||
let job = read_backend_job(conn, workspace, job)?
|
||||
.ok_or_else(|| Error::Store("Job notification source missing".into()))?;
|
||||
let job = read_result_snapshot(conn, workspace, job)?;
|
||||
let target = &job.request.notification_target;
|
||||
let facts = serde_json::json!({"principal_kind":if target.is_some(){Some("worker")}else{None}, "principal_id":null,"runtime_id":target.as_ref().map(|w|&w.runtime_id),"worker_id":target.as_ref().map(|w|&w.worker_id),"status":null,"case_state":"open","released":false,"backend_job":source,"recipient_kind":"fixed_worker","job_purpose":job.request.purpose,"result_digest":job.result_digest});
|
||||
conn.execute("INSERT OR IGNORE INTO notification_sources(workspace_id,ticket_id,input_event_id,role,assignment_id,kind,facts_json) VALUES(?1,NULL,?2,'backend_job','','backend_job_result',?3)",params![workspace,id,json(&facts)?])?;
|
||||
@@ -290,22 +315,13 @@ fn import_legacy(conn: &Connection, workspace: &str, job_id: &str, attempt_id: &
|
||||
| "notification_too_large",
|
||||
) => ("defer", NotificationReason::SourceUnavailable),
|
||||
_ => {
|
||||
let job = read_backend_job(conn, workspace, job_id)?
|
||||
.ok_or_else(|| Error::Store("Legacy Job result missing".into()))?;
|
||||
let attempt = read_backend_job_attempt(conn, workspace, job_id, attempt_id)?
|
||||
.ok_or_else(|| Error::Store("Legacy Job attempt missing".into()))?;
|
||||
let acceptance = BackendJobResultAcceptance {
|
||||
job,
|
||||
attempt,
|
||||
replayed: true,
|
||||
};
|
||||
let facts = if acceptance.job.request.notification_target.as_ref() != Some(&old.target)
|
||||
{
|
||||
let snapshot = read_result_snapshot(conn, workspace, job_id)?;
|
||||
let facts = if snapshot.request.notification_target.as_ref() != Some(&old.target) {
|
||||
JobNotificationFacts::Failed(NotificationFailureCode::PreparationFailed)
|
||||
} else {
|
||||
legacy_result_facts(&acceptance)
|
||||
legacy_result_facts(&snapshot, job_id, attempt_id)
|
||||
};
|
||||
let decision = policy(acceptance.job.request.notification_target.is_some(), &facts);
|
||||
let decision = policy(snapshot.request.notification_target.is_some(), &facts);
|
||||
match facts {
|
||||
JobNotificationFacts::Result(body) => content = Some(body),
|
||||
JobNotificationFacts::Failed(code) => failure = Some(code),
|
||||
@@ -338,28 +354,26 @@ fn import_legacy(conn: &Connection, workspace: &str, job_id: &str, attempt_id: &
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn legacy_result_facts(acceptance: &BackendJobResultAcceptance) -> JobNotificationFacts {
|
||||
let content = if acceptance.job.request.purpose == crate::ticket_item_checker::PURPOSE {
|
||||
let Some(result) = &acceptance.job.result else {
|
||||
fn legacy_result_facts(
|
||||
snapshot: &NotificationResultSnapshot,
|
||||
job_id: &str,
|
||||
attempt_id: &str,
|
||||
) -> JobNotificationFacts {
|
||||
let content = if snapshot.request.purpose == crate::ticket_item_checker::PURPOSE {
|
||||
let Some(result) = &snapshot.result else {
|
||||
return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult);
|
||||
};
|
||||
match crate::ticket_item_checker::validate_result(&acceptance.job.request.input, result) {
|
||||
match crate::ticket_item_checker::validate_result(&snapshot.request.input, result) {
|
||||
Ok(result) if result.findings.is_empty() => return JobNotificationFacts::NoFindings,
|
||||
Ok(_) => {}
|
||||
Err(_) => return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult),
|
||||
}
|
||||
crate::ticket_item_checker::render_result(acceptance)
|
||||
crate::ticket_item_checker::render_result(&snapshot.request.input, result)
|
||||
} else {
|
||||
Ok(format!(
|
||||
"Backend Job `{}` completed attempt `{}` for `{}`. Durable result digest: {}.",
|
||||
acceptance.job.request.job_id,
|
||||
acceptance.attempt.attempt_id,
|
||||
acceptance.job.request.purpose,
|
||||
acceptance
|
||||
.job
|
||||
.result_digest
|
||||
.as_deref()
|
||||
.unwrap_or("unavailable")
|
||||
"Backend Job `{job_id}` completed attempt `{attempt_id}` for `{}`. Durable result digest: {}.",
|
||||
snapshot.request.purpose,
|
||||
snapshot.result_digest.as_deref().unwrap_or("unavailable")
|
||||
))
|
||||
};
|
||||
match content {
|
||||
|
||||
@@ -789,6 +789,7 @@ CREATE TABLE "worker_registry" (
|
||||
FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TABLE backend_jobs (
|
||||
input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated')),
|
||||
workspace_id TEXT NOT NULL,
|
||||
job_id TEXT NOT NULL,
|
||||
purpose TEXT NOT NULL,
|
||||
@@ -1642,6 +1643,19 @@ CREATE TABLE IF NOT EXISTS ticket_assignment_work_releases (
|
||||
FOREIGN KEY (workspace_id, assignment_id)
|
||||
REFERENCES ticket_worker_assignments(workspace_id, assignment_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TRIGGER backend_job_identity_immutable BEFORE UPDATE OF
|
||||
workspace_id, job_id, purpose, input_digest, input_ref, resource_key, intent_fingerprint ON backend_jobs
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job identity'); END;
|
||||
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON backend_jobs
|
||||
WHEN (
|
||||
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
|
||||
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
|
||||
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
|
||||
AND json_type(NEW.request_json,'$.input') = 'null'
|
||||
) IS NOT TRUE
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job input retirement'); END;
|
||||
|
||||
-- Ticket case assignment work constraints (schema 91).
|
||||
DROP VIEW IF EXISTS ticket_active_worker_assignments;
|
||||
CREATE VIEW ticket_active_worker_assignments AS
|
||||
|
||||
@@ -4635,11 +4635,79 @@ impl WorkspaceApi {
|
||||
&self,
|
||||
reservation: BackendJobReservation,
|
||||
) -> Result<BackendJobReservation> {
|
||||
// Reread even a terminal/stale reservation before deciding whether it can
|
||||
// dispatch or replay: an explicit erasure may have retired its saved input.
|
||||
let journal_gate = reservation
|
||||
.job
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
.map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id))
|
||||
.transpose()?;
|
||||
let _journal_guard = journal_gate
|
||||
.as_ref()
|
||||
.map(|gate| {
|
||||
gate.lock()
|
||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
let current_job = self
|
||||
.store
|
||||
.get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)?
|
||||
.ok_or_else(|| Error::Store("reserved Backend Job disappeared".into()))?;
|
||||
let current_attempt = self
|
||||
.store
|
||||
.get_backend_job_attempt(
|
||||
&self.config.workspace_id,
|
||||
&reservation.job.request.job_id,
|
||||
&reservation.attempt.attempt_id,
|
||||
)?
|
||||
.ok_or_else(|| Error::Store("reserved Backend attempt disappeared".into()))?;
|
||||
let reservation = BackendJobReservation {
|
||||
job: current_job,
|
||||
attempt: current_attempt,
|
||||
replayed: reservation.replayed,
|
||||
resource_reused: reservation.resource_reused,
|
||||
};
|
||||
if matches!(
|
||||
reservation.job.input_disposition,
|
||||
crate::backend_job::JobInputDisposition::Erased
|
||||
| crate::backend_job::JobInputDisposition::Invalidated
|
||||
) {
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"erased or invalidated Backend Job input cannot dispatch".into(),
|
||||
));
|
||||
}
|
||||
if reservation.job.state != BackendJobState::Pending
|
||||
|| reservation.attempt.state != BackendJobAttemptState::Reserved
|
||||
|| reservation.attempt.worker.is_some()
|
||||
{
|
||||
return Ok(reservation);
|
||||
return Ok(BackendJobReservation {
|
||||
replayed: true,
|
||||
..reservation
|
||||
});
|
||||
}
|
||||
reservation.job.load_executable_request()?;
|
||||
if subjektiv_journal::migrate_reserved_legacy(self, &reservation)? {
|
||||
let job = self
|
||||
.store
|
||||
.get_backend_job(self.workspace_id(), &reservation.job.request.job_id)?
|
||||
.ok_or_else(|| Error::Store("retired legacy Job missing".into()))?;
|
||||
let attempt = self
|
||||
.store
|
||||
.get_backend_job_attempt(
|
||||
self.workspace_id(),
|
||||
&reservation.job.request.job_id,
|
||||
&reservation.attempt.attempt_id,
|
||||
)?
|
||||
.ok_or_else(|| Error::Store("retired legacy attempt missing".into()))?;
|
||||
return Ok(BackendJobReservation {
|
||||
job,
|
||||
attempt,
|
||||
replayed: reservation.replayed,
|
||||
resource_reused: reservation.resource_reused,
|
||||
});
|
||||
}
|
||||
subjektiv_journal::prepare(self, &reservation)?;
|
||||
let (reservation, claimed) = self.store.claim_backend_job_attempt_dispatch(
|
||||
@@ -4844,6 +4912,7 @@ impl WorkspaceApi {
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
subjektiv_journal::attest_delivery(self, &reservation)?;
|
||||
let job = self
|
||||
.store
|
||||
.get_backend_job(&self.config.workspace_id, &request.job_id)?
|
||||
@@ -5220,6 +5289,16 @@ impl WorkspaceApi {
|
||||
subjektiv_journal::close(self, &job.request, ¤t)?;
|
||||
if current.state == BackendJobAttemptState::Completed {
|
||||
subjektiv_journal::drain_after_completion(self, &job.request);
|
||||
if job.input_disposition
|
||||
== crate::backend_job::JobInputDisposition::Available
|
||||
{
|
||||
self.store.reclaim_backend_job_input(
|
||||
self.workspace_id(),
|
||||
job_id,
|
||||
&job.input_digest,
|
||||
crate::backend_job::JobInputRetirementReason::Completed,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use super::*;
|
||||
use crate::backend_job::{
|
||||
BackendJobGrants, BackendJobRecord, SubjektivJournalGrant, journal_serialization_key,
|
||||
};
|
||||
use subjektiv::journal::recording::{JournalMaterial, select_materials_bounded};
|
||||
use subjektiv::journal::recording::JournalInput;
|
||||
use subjektiv::journal::transport::*;
|
||||
use subjektiv::journal::{self, *};
|
||||
|
||||
@@ -92,20 +92,82 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) -
|
||||
};
|
||||
let store = store(api)?;
|
||||
let host = host(api, &store, grant)?;
|
||||
store
|
||||
.prepare_journal_generation(
|
||||
let input = saved_input(&reservation.job.request, grant)?;
|
||||
let g = store
|
||||
.prepare_journal_input(
|
||||
&host,
|
||||
&grant.journal_id,
|
||||
&input,
|
||||
subjektiv::api::JobAttemptBinding {
|
||||
job_id: reservation.job.request.job_id.clone(),
|
||||
attempt_id: reservation.attempt.attempt_id.clone(),
|
||||
},
|
||||
grant.selected_ranges.clone(),
|
||||
)
|
||||
.map_err(domain)?;
|
||||
input.validate_generation(&g).map_err(domain)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn legacy_descriptor(request: &BackendJobRequest) -> bool {
|
||||
let Some(grant) = request.grants.subjektiv_journal.as_ref() else {
|
||||
return false;
|
||||
};
|
||||
let Some(object) = request.input.as_object() else {
|
||||
return false;
|
||||
};
|
||||
object.len() == 2
|
||||
&& object.get("journal_id").and_then(|v| v.as_str()) == Some(&grant.journal_id)
|
||||
&& object.get("selected_ranges")
|
||||
== serde_json::to_value(&grant.selected_ranges).ok().as_ref()
|
||||
}
|
||||
|
||||
/// Retire only the explicitly recognized descriptor wire. Malformed body snapshots are
|
||||
/// failures, not migration requests and never a reason to reselect from current Sessions.
|
||||
pub(super) fn migrate_reserved_legacy(
|
||||
api: &WorkspaceApi,
|
||||
reservation: &BackendJobReservation,
|
||||
) -> Result<bool> {
|
||||
if !legacy_descriptor(&reservation.job.request) {
|
||||
return Ok(false);
|
||||
}
|
||||
reservation.job.load_executable_request()?;
|
||||
close(api, &reservation.job.request, &reservation.attempt)?;
|
||||
api.store.finish_backend_job_attempt(
|
||||
api.workspace_id(),
|
||||
&reservation.job.request.job_id,
|
||||
&reservation.attempt.attempt_id,
|
||||
BackendJobAttemptState::Failed,
|
||||
"journal_descriptor_invalidated",
|
||||
"Descriptor-only Journal input retired; saved checkpoints retained",
|
||||
&Utc::now().to_rfc3339(),
|
||||
)?;
|
||||
api.store.reclaim_backend_job_input(
|
||||
api.workspace_id(),
|
||||
&reservation.job.request.job_id,
|
||||
&reservation.job.input_digest,
|
||||
crate::backend_job::JobInputRetirementReason::LegacyInvalidation,
|
||||
)?;
|
||||
let queued = api.clone();
|
||||
let request = reservation.job.request.clone();
|
||||
api.host_executors.host()?.dispatch(move || {
|
||||
drain_after_completion(&queued, &request);
|
||||
})?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn saved_input(request: &BackendJobRequest, grant: &SubjektivJournalGrant) -> Result<JournalInput> {
|
||||
let input = JournalInput::decode(&request.input).map_err(domain)?;
|
||||
if input.subject_id != grant.subject_id
|
||||
|| input.journal_id != grant.journal_id
|
||||
|| input.capture.session_id != grant.session_id
|
||||
|| input.selected_ranges != grant.selected_ranges
|
||||
{
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"Journal saved input/capability mismatch".into(),
|
||||
));
|
||||
}
|
||||
Ok(input)
|
||||
}
|
||||
|
||||
fn live(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
@@ -125,6 +187,7 @@ fn live(
|
||||
.store
|
||||
.get_backend_job(api.workspace_id(), &binding.job_id)?
|
||||
.ok_or_else(|| Error::WorkspacePermissionDenied("Journal Job missing".into()))?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = api
|
||||
.store
|
||||
.get_backend_job_attempt(api.workspace_id(), &binding.job_id, &binding.attempt_id)?
|
||||
@@ -180,15 +243,44 @@ pub(super) fn execute(
|
||||
worker: &RuntimeWorkerRef,
|
||||
request: JournalJobRequest,
|
||||
) -> Result<JournalJobResponse> {
|
||||
let (grant, _, _) = live(api, worker)?;
|
||||
let gate = lifecycle_lock(api, &grant.subject_id)?;
|
||||
let _guard = gate
|
||||
.lock()
|
||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
||||
// Dispatch holds this same gate until matching Runtime acceptance is attested.
|
||||
// Revalidate after waiting: erasure, cancellation or replacement may have won.
|
||||
let (grant, job, attempt) = live(api, worker)?;
|
||||
let store = store(api)?;
|
||||
let host = host(api, &store, &grant)?;
|
||||
let g = generation(&store, &host, &grant, &job, &attempt)?;
|
||||
let input = saved_input(&job.request, &grant)?;
|
||||
input.validate_generation(&g).map_err(domain)?;
|
||||
let ctx = JournalJobContext::validated(&store, &host, &g.id, g.job.clone()).map_err(domain)?;
|
||||
journal::service::execute_bounded(&store, &ctx, request, |source, start, limit| {
|
||||
read_source(api, &grant, source, start, limit)
|
||||
.map_err(|e| JournalError::Conflict(e.to_string()))
|
||||
})
|
||||
journal::service::execute_snapshot(&store, &ctx, &attempt.input_digest, request).map_err(domain)
|
||||
}
|
||||
|
||||
/// Called only after matching tracked Runtime acceptance of the exact rendered saved input.
|
||||
/// The durable request and Worker binding alone do not establish body delivery.
|
||||
pub(super) fn attest_delivery(
|
||||
api: &WorkspaceApi,
|
||||
reservation: &BackendJobReservation,
|
||||
) -> Result<()> {
|
||||
let Some(grant) = &reservation.job.request.grants.subjektiv_journal else {
|
||||
return Ok(());
|
||||
};
|
||||
let input = saved_input(&reservation.job.request, grant)?;
|
||||
let store = store(api)?;
|
||||
let host = host(api, &store, grant)?;
|
||||
let g = generation(&store, &host, grant, &reservation.job, &reservation.attempt)?;
|
||||
input.validate_generation(&g).map_err(domain)?;
|
||||
let ctx = JournalJobContext::validated(&store, &host, &g.id, g.job.clone()).map_err(domain)?;
|
||||
journal::service::attest_input_delivery(
|
||||
&store,
|
||||
&ctx,
|
||||
&reservation.attempt.input_digest,
|
||||
&input.selected_ranges,
|
||||
)
|
||||
.map_err(domain)
|
||||
}
|
||||
|
||||
@@ -196,21 +288,29 @@ pub(super) fn finish(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> Result<JournalJobCompletion> {
|
||||
let (grant, _, _) = live(api, worker)?;
|
||||
let gate = lifecycle_lock(api, &grant.subject_id)?;
|
||||
let _guard = gate
|
||||
.lock()
|
||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
||||
// Dispatch holds this same gate until matching Runtime acceptance is attested.
|
||||
// Revalidate after waiting: erasure, cancellation or replacement may have won.
|
||||
let (grant, job, attempt) = live(api, worker)?;
|
||||
let store = store(api)?;
|
||||
let host = host(api, &store, &grant)?;
|
||||
let input = saved_input(&job.request, &grant)?;
|
||||
// prepare is an exact binding lookup too, including the saved completion of a closed generation.
|
||||
let g = store
|
||||
.prepare_journal_generation(
|
||||
.prepare_journal_input(
|
||||
&host,
|
||||
&grant.journal_id,
|
||||
&input,
|
||||
subjektiv::api::JobAttemptBinding {
|
||||
job_id: job.request.job_id,
|
||||
attempt_id: attempt.attempt_id,
|
||||
},
|
||||
grant.selected_ranges.clone(),
|
||||
)
|
||||
.map_err(domain)?;
|
||||
input.validate_generation(&g).map_err(domain)?;
|
||||
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
|
||||
.map_err(domain)?;
|
||||
journal::service::finish(&store, &ctx).map_err(domain)
|
||||
@@ -277,21 +377,6 @@ pub(super) fn close(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_source(
|
||||
api: &WorkspaceApi,
|
||||
grant: &SubjektivJournalGrant,
|
||||
source: &JournalSource,
|
||||
start: u64,
|
||||
limit: usize,
|
||||
) -> Result<JournalMaterial> {
|
||||
let store = store(api)?;
|
||||
let attr = store
|
||||
.session_attribution(&grant.session_id)
|
||||
.map_err(|e| Error::Store(e.to_string()))?
|
||||
.ok_or_else(|| Error::WorkspacePermissionDenied("Session attribution absent".into()))?;
|
||||
read_canonical(&material_reader(api, &attr)?, source, start, limit)
|
||||
}
|
||||
|
||||
pub(super) fn bind_host_session(
|
||||
api: &WorkspaceApi,
|
||||
attr: crate::subjektiv::SubjectSessionAttribution,
|
||||
@@ -524,22 +609,63 @@ fn dispatch(
|
||||
.close_journal_generation(&host, &g.id, &g.job)
|
||||
.map_err(domain)?;
|
||||
}
|
||||
let reader = material_reader(api, &attribution)?;
|
||||
let materials = select_materials_bounded(
|
||||
&store,
|
||||
&host,
|
||||
journal_id,
|
||||
96 * 1024,
|
||||
|source, start, limit| {
|
||||
read_canonical(&reader, source, start, limit)
|
||||
.map_err(|e| JournalError::Conflict(e.to_string()))
|
||||
},
|
||||
)
|
||||
let mut after = None;
|
||||
loop {
|
||||
let jobs = api.store.list_backend_journal_jobs(
|
||||
api.workspace_id(),
|
||||
&attribution.session_id,
|
||||
after.as_deref(),
|
||||
100,
|
||||
)?;
|
||||
for job in &jobs {
|
||||
if job
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
.is_none_or(|g| g.journal_id != journal_id)
|
||||
|| job.input_disposition != crate::backend_job::JobInputDisposition::Available
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if job.state == BackendJobState::Pending || job.state == BackendJobState::Unknown {
|
||||
return Ok(());
|
||||
}
|
||||
if job.state == BackendJobState::Failed && !legacy_descriptor(&job.request) {
|
||||
job.load_executable_request()?;
|
||||
saved_input(
|
||||
&job.request,
|
||||
job.request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
.expect("checked Journal grant"),
|
||||
)?;
|
||||
let reservation = api.store.reserve_backend_job_retry(
|
||||
api.workspace_id(),
|
||||
&job.request.job_id,
|
||||
&Utc::now().to_rfc3339(),
|
||||
)?;
|
||||
return queue_reservation(api, reservation);
|
||||
}
|
||||
}
|
||||
if jobs.len() < 100 {
|
||||
break;
|
||||
}
|
||||
after = jobs.last().map(|j| j.request.job_id.clone());
|
||||
}
|
||||
let input = JournalInput::build(&store, &host, journal_id, |order, offset, budget| {
|
||||
capture_input(api, &attribution, order, offset, budget)
|
||||
.map_err(|e| JournalError::Conflict(e.to_string()))
|
||||
})
|
||||
.map_err(domain)?;
|
||||
if materials.is_empty() {
|
||||
if input.selected_ranges.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let request = recording_request(&host, attribution, journal_id, materials)?;
|
||||
store
|
||||
.accept_journal_sources(&host, "Session activity", &input.selected_ranges)
|
||||
.map_err(domain)?;
|
||||
let request = recording_request(&host, attribution, journal_id, input)?;
|
||||
// The intent is saved before generation binding or dispatch. If binding/dispatch fails,
|
||||
// Reserved recovery repeats this exact intent, never creates a replacement blindly.
|
||||
api.resolve_backend_job_profile(&request)?;
|
||||
@@ -548,6 +674,10 @@ fn dispatch(
|
||||
&request,
|
||||
&Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
)?;
|
||||
queue_reservation(api, reservation)
|
||||
}
|
||||
|
||||
fn queue_reservation(api: &WorkspaceApi, reservation: BackendJobReservation) -> Result<()> {
|
||||
let queued = api.clone();
|
||||
if let Err(error)=api.host_executors.host()?.dispatch(move || {
|
||||
if let Err(error)=queued.dispatch_backend_job_reservation(reservation) {
|
||||
@@ -578,33 +708,21 @@ fn admit_sources(
|
||||
));
|
||||
}
|
||||
let mut id = store.journal_for_session(&host).map_err(domain)?;
|
||||
let reader = material_reader(api, &attribution)?;
|
||||
if reader.not_materialized {
|
||||
if store.journal_retention_required(&host).map_err(domain)? {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Admitted Journal original Session is not materialized".into(),
|
||||
));
|
||||
}
|
||||
retain_source(
|
||||
api.runtime.as_ref(),
|
||||
api.workspace_id(),
|
||||
&attribution,
|
||||
material_source(api, &attribution)?,
|
||||
true,
|
||||
)?;
|
||||
return Ok(JournalRecordingResponse::NoNewSources);
|
||||
}
|
||||
let (order, offset) = if let Some(id) = id.as_deref() {
|
||||
let journal = store.read_journal(&host, id).map_err(domain)?;
|
||||
(journal.process_order, journal.process_byte)
|
||||
} else {
|
||||
(0, 0)
|
||||
};
|
||||
let capture = capture_input(api, &attribution, order, offset, 48 * 1024)?;
|
||||
retain_source(
|
||||
api.runtime.as_ref(),
|
||||
api.workspace_id(),
|
||||
&attribution,
|
||||
material_source(api, &attribution)?,
|
||||
false,
|
||||
capture.segment_id == "not-materialized",
|
||||
)?;
|
||||
let sources: Vec<_> = reader
|
||||
.descriptors()
|
||||
.map(|d| source_from_descriptor(&attribution.session_id, d))
|
||||
.collect();
|
||||
let sources = journal::recording::capture_ranges(&capture).map_err(domain)?;
|
||||
for batch in sources.chunks(MAX_JOURNAL_RANGES) {
|
||||
id = Some(
|
||||
store
|
||||
@@ -995,7 +1113,7 @@ pub(super) fn release_subject(
|
||||
{
|
||||
// Legacy saved-before-admission Sessions are not optional. Prove canonical
|
||||
// source identity first, then bind only this original Host-attributed Session.
|
||||
let _canonical = material_reader(api, &attr)?;
|
||||
let _canonical = capture_input(api, &attr, 0, 0, 48 * 1024)?;
|
||||
let host =
|
||||
JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
||||
if host
|
||||
@@ -1281,80 +1399,6 @@ fn material_source(
|
||||
},
|
||||
)
|
||||
}
|
||||
fn material_reader<'a>(
|
||||
api: &'a WorkspaceApi,
|
||||
attribution: &crate::subjektiv::SubjectSessionAttribution,
|
||||
) -> Result<MaterialReader<'a>> {
|
||||
MaterialReader::open(
|
||||
api.runtime.as_ref(),
|
||||
api.workspace_id(),
|
||||
&attribution.runtime_id,
|
||||
material_source(api, attribution)?,
|
||||
&attribution.session_id,
|
||||
)
|
||||
}
|
||||
fn source_from_descriptor(
|
||||
session: &str,
|
||||
d: &session_store::public_material::PublicMaterialDescriptor,
|
||||
) -> JournalSourceRange {
|
||||
JournalSourceRange {
|
||||
source: JournalSource {
|
||||
session_id: session.into(),
|
||||
pointer: JournalSourcePointer {
|
||||
segment_id: d.identity.segment_id,
|
||||
entry_ref: d.identity.entry_ref.clone(),
|
||||
},
|
||||
order: d.canonical_order,
|
||||
content_sha256: d.sha256.clone(),
|
||||
total_bytes: d.total_bytes,
|
||||
},
|
||||
bytes: JournalByteRange {
|
||||
start: 0,
|
||||
end: d.total_bytes,
|
||||
},
|
||||
}
|
||||
}
|
||||
fn read_canonical(
|
||||
reader: &MaterialReader<'_>,
|
||||
source: &JournalSource,
|
||||
start: u64,
|
||||
limit: usize,
|
||||
) -> Result<JournalMaterial> {
|
||||
if reader.session_id != source.session_id {
|
||||
return Err(Error::WorkspacePermissionDenied(
|
||||
"Journal source Session mismatch".into(),
|
||||
));
|
||||
}
|
||||
let descriptor = reader
|
||||
.descriptors()
|
||||
.find(|d| source_from_descriptor(&source.session_id, d).source == *source)
|
||||
.ok_or_else(|| {
|
||||
Error::Store("Journal original public source changed or unavailable".into())
|
||||
})?;
|
||||
// The reader streams the complete canonical digest while returning only this bounded window.
|
||||
let page = reader
|
||||
.read_content(descriptor, start, limit)
|
||||
.map_err(|e| Error::Store(e.to_string()))?;
|
||||
if page.sha256 != source.content_sha256
|
||||
|| page.total_bytes != source.total_bytes
|
||||
|| page.next_offset != start.saturating_add(page.content.len() as u64)
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Journal canonical source identity changed".into(),
|
||||
));
|
||||
}
|
||||
Ok(JournalMaterial {
|
||||
range: JournalSourceRange {
|
||||
source: source.clone(),
|
||||
bytes: JournalByteRange {
|
||||
start,
|
||||
end: page.next_offset,
|
||||
},
|
||||
},
|
||||
text: page.content,
|
||||
})
|
||||
}
|
||||
|
||||
/// Close the first-run save/notification gap: Runtime can commit an initial turn before
|
||||
/// the create handler has finalized the Backend registry required by recording callbacks.
|
||||
/// Reconcile only after creation succeeds, using the ordinary current-owner/Feature checks.
|
||||
@@ -1475,7 +1519,7 @@ pub(super) fn recording_request(
|
||||
host: &JournalHostContext,
|
||||
attribution: crate::subjektiv::SubjectSessionAttribution,
|
||||
journal_id: &str,
|
||||
materials: Vec<journal::recording::JournalMaterial>,
|
||||
input: JournalInput,
|
||||
) -> Result<BackendJobRequest> {
|
||||
let binding_id = host
|
||||
.writer_binding_id()
|
||||
@@ -1488,18 +1532,88 @@ pub(super) fn recording_request(
|
||||
subject_id: attribution.subject_id,
|
||||
session_id: attribution.session_id,
|
||||
journal_id: journal_id.into(),
|
||||
selected_ranges: materials.iter().map(|m| m.range.clone()).collect(),
|
||||
selected_ranges: input.selected_ranges.clone(),
|
||||
};
|
||||
Ok(BackendJobRequest {
|
||||
job_id:format!("journal-recording:{}",Uuid::new_v4()),purpose:"subjektiv_journal".into(),
|
||||
input_ref:format!("subjektiv://{}/journal/{journal_id}",grant.subject_id),
|
||||
input:serde_json::json!({"journal_id":journal_id,"selected_ranges":grant.selected_ranges}),
|
||||
instruction:"Record ONLY this frozen original public selected_ranges prefix. The input contains descriptors, NOT source bodies. First ReadJournalState and use bounded ReadJournalText Source until every byte of every selected range is returned; record operations are refused until then. ReadJournalText also exposes initial/current position/open text. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, update current position and CheckpointJournal using retained record receipts. Never infer event completion from turn/task/job/compaction boundaries. Preserve exact operation ID/full input after unknown outcomes. SubmitBackendJobResult only after the full selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(),
|
||||
profile:"builtin:subjektiv-journal-recording".into(),
|
||||
grants:BackendJobGrants {subjektiv_journal:Some(grant),..Default::default()},
|
||||
serialization_key:Some(journal_serialization_key(journal_id)),source_worker:None,notification_target:None,
|
||||
limits:crate::backend_job::BackendJobLimits {timeout_seconds:600,..Default::default()},
|
||||
})
|
||||
let request = BackendJobRequest {
|
||||
job_id: format!("journal-recording:{}",Uuid::new_v4()), purpose: "subjektiv_journal".into(),
|
||||
input_ref: format!("subjektiv://{}/journal/{journal_id}",grant.subject_id),
|
||||
input: serde_json::to_value(input).map_err(|e| Error::Store(e.to_string()))?,
|
||||
instruction: "Record ONLY the immutable body-bearing capture supplied in this input. These exact canonical source bytes, identities and ranges are the whole input for this Job; do not fetch original Sessions or use ReadJournalText Source. First ReadJournalState; initial/current Journal position/open text remain separately paged by ReadJournalText. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, current position and receipt-backed CheckpointJournal. Input delivery is not understanding or reflection. Never infer event completion from turn/task/job/compaction boundaries. Preserve exact operation ID/full input after Unknown. SubmitBackendJobResult only after the complete selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(),
|
||||
profile: "builtin:subjektiv-journal-recording".into(),
|
||||
grants: BackendJobGrants { subjektiv_journal: Some(grant), ..Default::default() },
|
||||
serialization_key: Some(journal_serialization_key(journal_id)), source_worker: None, notification_target: None,
|
||||
limits: crate::backend_job::BackendJobLimits { timeout_seconds: 600, ..Default::default() },
|
||||
};
|
||||
saved_input(
|
||||
&request,
|
||||
request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
.expect("constructed grant"),
|
||||
)?;
|
||||
request.validate()?;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
pub(super) fn capture_input(
|
||||
api: &WorkspaceApi,
|
||||
attr: &crate::subjektiv::SubjectSessionAttribution,
|
||||
order: u64,
|
||||
offset: u64,
|
||||
budget: usize,
|
||||
) -> Result<session_store::capture::CaptureDto> {
|
||||
match api
|
||||
.runtime
|
||||
.session_recording_capture(
|
||||
&attr.runtime_id,
|
||||
runtime_api::SessionRecordingCaptureRequest {
|
||||
workspace_id: api.workspace_id().into(),
|
||||
source: material_source(api, attr)?,
|
||||
expected_session_id: attr.session_id.clone(),
|
||||
start: session_store::capture::RecordingPosition {
|
||||
canonical_order: order,
|
||||
byte_offset: offset,
|
||||
},
|
||||
max_entries: MAX_JOURNAL_RANGES,
|
||||
max_serialized_bytes: budget,
|
||||
},
|
||||
)
|
||||
.map_err(RuntimeRegistryError::into_error)?
|
||||
{
|
||||
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => {
|
||||
if slice.capture.session_id != attr.session_id
|
||||
|| serde_json::to_vec(&slice)
|
||||
.map_err(|e| Error::Store(e.to_string()))?
|
||||
.len()
|
||||
> budget
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"recording capture identity/budget mismatch".into(),
|
||||
));
|
||||
}
|
||||
Ok(slice.capture)
|
||||
}
|
||||
runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id }
|
||||
if session_id == attr.session_id && order == 0 && offset == 0 =>
|
||||
{
|
||||
Ok(session_store::capture::CaptureDto {
|
||||
version: 1,
|
||||
session_id,
|
||||
segment_id: "not-materialized".into(),
|
||||
view_entry_count: 0,
|
||||
entries: vec![],
|
||||
})
|
||||
}
|
||||
runtime_api::SessionRecordingCaptureAvailability::Unavailable { reason, .. } => {
|
||||
Err(Error::Store(format!(
|
||||
"committed Journal capture unavailable ({reason:?}); no latest-source fallback"
|
||||
)))
|
||||
}
|
||||
_ => Err(Error::Store(
|
||||
"committed Journal capture identity mismatch; no latest-source fallback".into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn removal_attribution(
|
||||
@@ -1575,6 +1689,16 @@ fn finish_erased_release(
|
||||
"explicit erase tombstone missing".into(),
|
||||
));
|
||||
}
|
||||
for session in store
|
||||
.subject_session_attributions(&attr.subject_id)
|
||||
.map_err(|e| Error::Store(e.to_string()))?
|
||||
.into_iter()
|
||||
.filter(|session| {
|
||||
session.runtime_id == attr.runtime_id && session.worker_id == attr.worker_id
|
||||
})
|
||||
{
|
||||
require_erased_job_copies(api, &session.session_id)?;
|
||||
}
|
||||
// Pre-binding bodies have no write token. This stable receipt label is not a writer
|
||||
// grant: the domain erase operation records an inactive lifetime without binding Sessions.
|
||||
let erased_body_id = format!("erased-body:{}", attr.worker_id);
|
||||
@@ -1835,183 +1959,6 @@ pub(super) async fn finish_route(
|
||||
}
|
||||
|
||||
/// A bounded Runtime transport view, not a second history index or a local-path fallback.
|
||||
/// Reopening it observes a new catalog; each generation still compares its frozen source identity.
|
||||
struct MaterialReader<'a> {
|
||||
runtime: &'a RuntimeRegistry,
|
||||
workspace_id: &'a str,
|
||||
runtime_id: String,
|
||||
source: runtime_api::SessionPublicSource,
|
||||
session_id: String,
|
||||
not_materialized: bool,
|
||||
descriptors: Vec<session_store::public_material::PublicMaterialDescriptor>,
|
||||
}
|
||||
fn material_descriptor(
|
||||
d: runtime_api::SessionMaterialDescriptor,
|
||||
) -> Result<session_store::public_material::PublicMaterialDescriptor> {
|
||||
Ok(session_store::public_material::PublicMaterialDescriptor {
|
||||
identity: session_store::public_material::PublicMaterialIdentity {
|
||||
segment_id: Uuid::parse_str(&d.segment_id).map_err(|e| Error::Store(e.to_string()))?,
|
||||
entry_ref: d.entry_ref,
|
||||
},
|
||||
canonical_order: d.canonical_order,
|
||||
origin: serde_json::from_str(&d.origin_json).map_err(|e| Error::Store(e.to_string()))?,
|
||||
sha256: d.sha256,
|
||||
total_bytes: d.total_bytes,
|
||||
})
|
||||
}
|
||||
impl<'a> MaterialReader<'a> {
|
||||
fn open(
|
||||
runtime: &'a RuntimeRegistry,
|
||||
workspace_id: &'a str,
|
||||
runtime_id: &str,
|
||||
source: runtime_api::SessionPublicSource,
|
||||
session_id: &str,
|
||||
) -> Result<Self> {
|
||||
let mut descriptors = Vec::new();
|
||||
let mut offset = 0;
|
||||
let mut condition = None;
|
||||
loop {
|
||||
let page = match runtime
|
||||
.session_material_catalog(
|
||||
runtime_id,
|
||||
runtime_api::SessionMaterialCatalogRequest {
|
||||
workspace_id: workspace_id.into(),
|
||||
source: source.clone(),
|
||||
expected_session_id: session_id.into(),
|
||||
expected_catalog_sha256: condition.clone(),
|
||||
offset,
|
||||
limit: 100,
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.into_error())?
|
||||
{
|
||||
runtime_api::SessionMaterialCatalogAvailability::NotMaterialized {
|
||||
session_id: observed,
|
||||
} => {
|
||||
if observed != session_id || offset != 0 || condition.is_some() {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Journal Session materialization proof changed during catalog read"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
return Ok(Self {
|
||||
runtime,
|
||||
workspace_id,
|
||||
runtime_id: runtime_id.into(),
|
||||
source,
|
||||
session_id: session_id.into(),
|
||||
not_materialized: true,
|
||||
descriptors,
|
||||
});
|
||||
}
|
||||
runtime_api::SessionMaterialCatalogAvailability::Page { page } => page,
|
||||
runtime_api::SessionMaterialCatalogAvailability::Unavailable {
|
||||
reason,
|
||||
message,
|
||||
} => {
|
||||
return Err(Error::Store(format!(
|
||||
"Journal original material unavailable ({reason:?}): {message}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
if page.session_id != session_id
|
||||
|| condition
|
||||
.as_ref()
|
||||
.is_some_and(|c| c != &page.catalog_sha256)
|
||||
|| page.descriptors.len() > 100
|
||||
|| descriptors.len().saturating_add(page.descriptors.len()) > 1_000_000
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Journal canonical Session catalog changed or exceeded its bound".into(),
|
||||
));
|
||||
}
|
||||
condition = Some(page.catalog_sha256);
|
||||
for d in page.descriptors {
|
||||
let d = material_descriptor(d)?;
|
||||
if d.canonical_order != descriptors.len() as u64 {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Journal canonical catalog order is discontinuous".into(),
|
||||
));
|
||||
}
|
||||
descriptors.push(d);
|
||||
}
|
||||
match page.next_offset {
|
||||
Some(next) if next > offset && next == descriptors.len() as u64 => offset = next,
|
||||
None => break,
|
||||
_ => {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Journal catalog continuation did not advance".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
runtime,
|
||||
workspace_id,
|
||||
runtime_id: runtime_id.into(),
|
||||
source,
|
||||
session_id: session_id.into(),
|
||||
not_materialized: false,
|
||||
descriptors,
|
||||
})
|
||||
}
|
||||
fn descriptors(
|
||||
&self,
|
||||
) -> impl Iterator<Item = &session_store::public_material::PublicMaterialDescriptor> {
|
||||
self.descriptors.iter()
|
||||
}
|
||||
fn read_content(
|
||||
&self,
|
||||
descriptor: &session_store::public_material::PublicMaterialDescriptor,
|
||||
offset: u64,
|
||||
limit: usize,
|
||||
) -> Result<session_store::public_material::PublicMaterialContent> {
|
||||
let page = match self
|
||||
.runtime
|
||||
.session_material_read(
|
||||
&self.runtime_id,
|
||||
runtime_api::SessionMaterialReadRequest {
|
||||
workspace_id: self.workspace_id.into(),
|
||||
source: self.source.clone(),
|
||||
expected_session_id: self.session_id.clone(),
|
||||
expected_descriptor: runtime_api::SessionMaterialDescriptor {
|
||||
segment_id: descriptor.identity.segment_id.to_string(),
|
||||
entry_ref: descriptor.identity.entry_ref.clone(),
|
||||
canonical_order: descriptor.canonical_order,
|
||||
origin_json: serde_json::to_string(&descriptor.origin)
|
||||
.map_err(|e| Error::Store(e.to_string()))?,
|
||||
sha256: descriptor.sha256.clone(),
|
||||
total_bytes: descriptor.total_bytes,
|
||||
},
|
||||
byte_offset: offset,
|
||||
max_content_bytes: limit.clamp(4, 16 * 1024),
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.into_error())?
|
||||
{
|
||||
runtime_api::SessionMaterialReadAvailability::Page { page } => page,
|
||||
runtime_api::SessionMaterialReadAvailability::Unavailable { reason, message } => {
|
||||
return Err(Error::Store(format!(
|
||||
"Journal original material read unavailable ({reason:?}): {message}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
if material_descriptor(page.descriptor)? != *descriptor
|
||||
|| page.content.len() > limit.clamp(4, 16 * 1024)
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Journal canonical material identity changed".into(),
|
||||
));
|
||||
}
|
||||
Ok(session_store::public_material::PublicMaterialContent {
|
||||
content: page.content,
|
||||
next_offset: page.next_offset,
|
||||
total_bytes: descriptor.total_bytes,
|
||||
sha256: descriptor.sha256.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn erase_source(
|
||||
runtime: &RuntimeRegistry,
|
||||
workspace: &str,
|
||||
@@ -2145,17 +2092,29 @@ pub(super) fn reconcile_unknown(
|
||||
let host = host(api, &store, grant)?;
|
||||
let erased = store.journal_session_erased(&host).map_err(domain)?;
|
||||
if !erased {
|
||||
let g = store
|
||||
.prepare_journal_generation(
|
||||
&host,
|
||||
&grant.journal_id,
|
||||
subjektiv::api::JobAttemptBinding {
|
||||
job_id: job.request.job_id.clone(),
|
||||
attempt_id: attempt.attempt_id.clone(),
|
||||
},
|
||||
grant.selected_ranges.clone(),
|
||||
)
|
||||
.map_err(domain)?;
|
||||
job.load_executable_request()?;
|
||||
let binding = subjektiv::api::JobAttemptBinding {
|
||||
job_id: job.request.job_id.clone(),
|
||||
attempt_id: attempt.attempt_id.clone(),
|
||||
};
|
||||
let g = if legacy_descriptor(&job.request) {
|
||||
// Only reconcile already-saved legacy receipts; never redispatch descriptors.
|
||||
store
|
||||
.prepare_journal_generation(
|
||||
&host,
|
||||
&grant.journal_id,
|
||||
binding,
|
||||
grant.selected_ranges.clone(),
|
||||
)
|
||||
.map_err(domain)?
|
||||
} else {
|
||||
let input = saved_input(&job.request, grant)?;
|
||||
let generation = store
|
||||
.prepare_journal_input(&host, &input, binding)
|
||||
.map_err(domain)?;
|
||||
input.validate_generation(&generation).map_err(domain)?;
|
||||
generation
|
||||
};
|
||||
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
|
||||
.map_err(domain)?;
|
||||
match journal::service::finish(&store, &ctx) {
|
||||
@@ -2187,6 +2146,14 @@ pub(super) fn reconcile_unknown(
|
||||
);
|
||||
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) {
|
||||
drain_after_completion(api, &job.request);
|
||||
if job.input_disposition == crate::backend_job::JobInputDisposition::Available {
|
||||
api.store.reclaim_backend_job_input(
|
||||
api.workspace_id(),
|
||||
&job.request.job_id,
|
||||
&job.input_digest,
|
||||
crate::backend_job::JobInputRetirementReason::Completed,
|
||||
)?;
|
||||
}
|
||||
} else {
|
||||
api.schedule_backend_job_worker_cleanup(¤t);
|
||||
}
|
||||
@@ -2226,8 +2193,8 @@ pub(super) fn reconcile_unknown(
|
||||
&Utc::now().to_rfc3339(),
|
||||
)?;
|
||||
if !erased {
|
||||
// The old immutable input may include reflected bytes. Select the remaining
|
||||
// prefix afresh only AFTER definitive lifecycle/domain fences; never replace Unknown.
|
||||
// Resume the same saved body input only after canonical execution/domain fences.
|
||||
// dispatch selects its retained intent before it can capture any newer source.
|
||||
let attr = store
|
||||
.session_attribution(&grant.session_id)
|
||||
.map_err(|e| Error::Store(e.to_string()))?
|
||||
@@ -2256,6 +2223,30 @@ fn reconcile_session_jobs(
|
||||
.store
|
||||
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
|
||||
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
|
||||
if !erased
|
||||
&& legacy_descriptor(&job.request)
|
||||
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available
|
||||
{
|
||||
job.load_executable_request()?;
|
||||
close(api, &job.request, &attempt)?;
|
||||
if !attempt.state.terminal() {
|
||||
attempt = api.store.finish_backend_job_attempt(
|
||||
api.workspace_id(),
|
||||
&job.request.job_id,
|
||||
&id,
|
||||
BackendJobAttemptState::Failed,
|
||||
"journal_descriptor_invalidated",
|
||||
"Descriptor-only Journal input retired; saved checkpoints retained",
|
||||
&Utc::now().to_rfc3339(),
|
||||
)?;
|
||||
}
|
||||
api.store.reclaim_backend_job_input(
|
||||
api.workspace_id(),
|
||||
&job.request.job_id,
|
||||
&job.input_digest,
|
||||
crate::backend_job::JobInputRetirementReason::LegacyInvalidation,
|
||||
)?;
|
||||
}
|
||||
if erased && !attempt.state.terminal() {
|
||||
// The domain tombstone precedes this cancellation and also fences any
|
||||
// crash window before it. No erased Job can prepare or use tools again.
|
||||
@@ -2269,6 +2260,30 @@ fn reconcile_session_jobs(
|
||||
&Utc::now().to_rfc3339(),
|
||||
)?;
|
||||
}
|
||||
if erased {
|
||||
api.store.reclaim_backend_job_input(
|
||||
api.workspace_id(),
|
||||
&job.request.job_id,
|
||||
&job.input_digest,
|
||||
crate::backend_job::JobInputRetirementReason::ExplicitErasure,
|
||||
)?;
|
||||
// Every historical execution can own a delivered copy, not just the current
|
||||
// attempt or an unacknowledged result. Cleanup is canonical Purge and retryable.
|
||||
for number in 1..=job.current_attempt {
|
||||
let historical_id = crate::backend_job::attempt_id(&job.request.job_id, number);
|
||||
let historical = api
|
||||
.store
|
||||
.get_backend_job_attempt(
|
||||
api.workspace_id(),
|
||||
&job.request.job_id,
|
||||
&historical_id,
|
||||
)?
|
||||
.ok_or_else(|| Error::Store("historical Journal attempt missing".into()))?;
|
||||
if historical.state.terminal() {
|
||||
api.schedule_backend_job_worker_cleanup(&historical);
|
||||
}
|
||||
}
|
||||
}
|
||||
if attempt.state == BackendJobAttemptState::Unknown {
|
||||
reconcile_unknown(api, job, &attempt)?;
|
||||
}
|
||||
@@ -2284,6 +2299,41 @@ fn reconcile_session_jobs(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_erased_job_copies(api: &WorkspaceApi, session: &str) -> Result<()> {
|
||||
let mut after = None;
|
||||
loop {
|
||||
let jobs = api.store.list_backend_journal_jobs(
|
||||
api.workspace_id(),
|
||||
session,
|
||||
after.as_deref(),
|
||||
100,
|
||||
)?;
|
||||
for job in &jobs {
|
||||
if job.input_disposition != crate::backend_job::JobInputDisposition::Erased {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Journal input erasure remains incomplete".into(),
|
||||
));
|
||||
}
|
||||
for number in 1..=job.current_attempt {
|
||||
let id = crate::backend_job::attempt_id(&job.request.job_id, number);
|
||||
let attempt = api
|
||||
.store
|
||||
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
|
||||
.ok_or_else(|| Error::Store("erased Journal attempt missing".into()))?;
|
||||
if attempt.worker.is_some()
|
||||
&& attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed)
|
||||
{
|
||||
return Err(Error::RepositoryConflict("Journal delivered-copy Purge remains pending; retry the same erasure after cleanup".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
if jobs.len() < 100 {
|
||||
return Ok(());
|
||||
}
|
||||
after = jobs.last().map(|j| j.request.job_id.clone());
|
||||
}
|
||||
}
|
||||
|
||||
impl WorkspaceApi {
|
||||
/// Host-only erasure saves the domain fence, cancels Jobs, then erases originals.
|
||||
/// Runtime requires a stopped source Worker. Physical failure leaves the domain
|
||||
@@ -2301,6 +2351,10 @@ impl WorkspaceApi {
|
||||
.ok_or_else(|| {
|
||||
Error::WorkspacePermissionDenied("Session attribution missing".into())
|
||||
})?;
|
||||
let gate = lifecycle_lock(self, &attr.subject_id)?;
|
||||
let _guard = gate
|
||||
.lock()
|
||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
||||
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
||||
store
|
||||
.erase_journal_session(&host, request_id)
|
||||
@@ -2311,7 +2365,8 @@ impl WorkspaceApi {
|
||||
self.workspace_id(),
|
||||
&attr,
|
||||
material_source(self, &attr)?,
|
||||
)
|
||||
)?;
|
||||
require_erased_job_copies(self, &attr.session_id)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,10 +14,33 @@ use std::sync::{Arc, Mutex};
|
||||
#[derive(Clone, Default)]
|
||||
struct ConnectedJournalModel {
|
||||
dialogs: Arc<Mutex<Vec<Request>>>,
|
||||
journal_inputs: Arc<Mutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
impl ConnectedJournalModel {
|
||||
fn journal_response(request: &Request) -> Vec<Event> {
|
||||
// Consume the actual Controller-rendered input, not a fixture generation or
|
||||
// a Source tool response. Decode also checks complete bodies/ranges/digests.
|
||||
let envelope = request
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(Item::as_text)
|
||||
.find(|text| text.lines().any(|line| line.starts_with("input_json: ")))
|
||||
.expect("rendered immutable Backend Job envelope");
|
||||
let input_json = envelope
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("input_json: "))
|
||||
.unwrap();
|
||||
let input_value: Value = serde_json::from_str(input_json).unwrap();
|
||||
let frozen = subjektiv::journal::recording::JournalInput::decode(&input_value).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_string(&input_value).unwrap(),
|
||||
input_json,
|
||||
"render preserves the exact persisted JSON bytes"
|
||||
);
|
||||
assert!(frozen.capture.entries.iter().all(|entry| matches!(
|
||||
&entry.body, session_store::capture::CaptureBody::CanonicalPublic { text, .. } if !text.is_empty())));
|
||||
let ranges = serde_json::to_value(&frozen.selected_ranges).unwrap();
|
||||
// Runtime Controllers deliberately retain alias-routed core schemas for their
|
||||
// lifetime even with zero attachments. Verify invocation authority, not schema
|
||||
// absence (standalone Internal Jobs use a different composition).
|
||||
@@ -89,7 +112,7 @@ impl ConnectedJournalModel {
|
||||
"request_id":format!("checkpoint:{}",state["value"]["generation_id"].as_str().unwrap()),
|
||||
"expected_state":position["value"]["state_token"],
|
||||
"action":{"action":"checkpoint","record_operation_ids":[format!("record:{}",state["value"]["generation_id"].as_str().unwrap())]},
|
||||
"covered_ranges":state["value"]["selected_ranges"],
|
||||
"covered_ranges":ranges,
|
||||
}),
|
||||
)
|
||||
} else if let Some(record) = result("record") {
|
||||
@@ -106,41 +129,14 @@ impl ConnectedJournalModel {
|
||||
)
|
||||
} else if let Some(state) = result("state") {
|
||||
let state = &state["value"];
|
||||
// Deliver the actual frozen Runtime source through authenticated Job tools, rather
|
||||
// than injecting a fixture Journal or claiming unseen source was processed.
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("source-{index}-{offset}");
|
||||
if let Some(page) = result(&id) {
|
||||
let page = &page["value"];
|
||||
if let Some(next) = page["next_byte"].as_u64() {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["content_token"].clone();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
assert!(
|
||||
request
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| tool.name == "ReadJournalText")
|
||||
);
|
||||
return vec![Event::tool_use_start(0, &id, "ReadJournalText"), Event::tool_input_delta(0, json!({
|
||||
"target":{"target":"source","pointer":range["source"]["pointer"]},
|
||||
"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token,
|
||||
}).to_string()), Event::tool_use_stop(0)];
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
state["selected_ranges"], ranges,
|
||||
"mutation coverage must come from the rendered frozen input"
|
||||
);
|
||||
assert_eq!(
|
||||
state["initial_journal"]["state_token"],
|
||||
input_value["initial_state_token"]
|
||||
);
|
||||
let (tool, action) = if state["current_open"].is_null() {
|
||||
(
|
||||
"CreateJournalEntry",
|
||||
@@ -158,7 +154,7 @@ impl ConnectedJournalModel {
|
||||
json!({
|
||||
"request_id":format!("record:{}",state["generation_id"].as_str().unwrap()),
|
||||
"expected_state":state["current_journal"]["state_token"],
|
||||
"action":action,"covered_ranges":state["selected_ranges"],
|
||||
"action":action,"covered_ranges":ranges,
|
||||
}),
|
||||
)
|
||||
} else {
|
||||
@@ -209,6 +205,18 @@ impl LlmClient for ConnectedJournalModel {
|
||||
.iter()
|
||||
.any(|tool| tool.name == "ReadJournalState")
|
||||
{
|
||||
let envelope = request
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(Item::as_text)
|
||||
.find(|text| text.lines().any(|line| line.starts_with("input_json: ")))
|
||||
.expect("actual Journal input in model render")
|
||||
.to_owned();
|
||||
let mut inputs = self.journal_inputs.lock().unwrap();
|
||||
if !inputs.contains(&envelope) {
|
||||
inputs.push(envelope);
|
||||
}
|
||||
drop(inputs);
|
||||
Self::journal_response(&request)
|
||||
} else if request.tools.iter().any(|tool| {
|
||||
matches!(
|
||||
@@ -458,6 +466,29 @@ async fn backend_real_runtime_journal_release_and_next_body_persisted_startup()
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &old_attr.session_id, None, 100)
|
||||
.unwrap();
|
||||
// Compare actual render with the durable identity, even if completion already
|
||||
// reclaimed the input body. Reconstituting only for comparison is test-local.
|
||||
for saved in &recorded {
|
||||
let inputs = model.journal_inputs.lock().unwrap();
|
||||
let prefix = format!("job_id: {}", saved.request.job_id);
|
||||
let rendered = inputs
|
||||
.iter()
|
||||
.find(|text| text.lines().any(|line| line == prefix))
|
||||
.expect("every saved Journal Job reached the real Controller");
|
||||
let actual = rendered
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("input_json: "))
|
||||
.unwrap();
|
||||
let mut original = saved.request.clone();
|
||||
original.input = serde_json::from_str(actual).unwrap();
|
||||
assert_eq!(original.input_digest().unwrap(), saved.input_digest);
|
||||
let attempt = crate::backend_job::attempt_id(&saved.request.job_id, saved.current_attempt);
|
||||
assert_eq!(*rendered, original.worker_input(&attempt).unwrap());
|
||||
let input = subjektiv::journal::recording::JournalInput::decode(&original.input).unwrap();
|
||||
assert_eq!(input.capture.session_id, old_attr.session_id);
|
||||
assert!(actual.contains("User chose B rather than A"));
|
||||
assert!(actual.contains("Partial B implementation"));
|
||||
}
|
||||
let uncertain_job = &recorded[0].request.job_id;
|
||||
let conn = rusqlite::Connection::open(&api.config.database_path).unwrap();
|
||||
let saved_result: (String, String) = conn.query_row(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -119,7 +119,7 @@ pub use ticket::{
|
||||
};
|
||||
|
||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 102;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 103;
|
||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||
@@ -437,6 +437,11 @@ const MIGRATIONS: &[Migration] = &[
|
||||
name: "Durable failed Subject creation recovery",
|
||||
apply: migrate_subject_create_recovery_v101_to_v102,
|
||||
},
|
||||
Migration {
|
||||
version: 103,
|
||||
name: "Job input retirement independent of immutable receipts",
|
||||
apply: migrate_job_input_retention_v102_to_v103,
|
||||
},
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -1771,6 +1776,15 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
||||
request: &BackendJobRequest,
|
||||
now: &str,
|
||||
) -> Result<BackendJobReservation>;
|
||||
/// Host-only retirement; fence domain mutations and delivered snapshots first for erasure.
|
||||
/// Completed reclamation additionally requires canonical Worker cleanup for all attempts.
|
||||
fn reclaim_backend_job_input(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
job_id: &str,
|
||||
expected_input_digest: &str,
|
||||
reason: crate::backend_job::JobInputRetirementReason,
|
||||
) -> Result<BackendJobRecord>;
|
||||
fn reserve_backend_job_retry(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -7358,6 +7372,48 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
})
|
||||
}
|
||||
|
||||
fn reclaim_backend_job_input(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
job_id: &str,
|
||||
expected_input_digest: &str,
|
||||
reason: crate::backend_job::JobInputRetirementReason,
|
||||
) -> Result<BackendJobRecord> {
|
||||
use crate::backend_job::{JobInputDisposition, JobInputRetirementReason};
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
if job.input_digest != expected_input_digest {
|
||||
return Err(Error::InvalidInput("Backend Job input digest mismatch".into()));
|
||||
}
|
||||
let desired = reason.disposition();
|
||||
if job.input_disposition == desired {
|
||||
tx.commit()?;
|
||||
reclaim_backend_job_input_storage(conn)?;
|
||||
return Ok(job);
|
||||
}
|
||||
if reason == JobInputRetirementReason::Completed {
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))?
|
||||
.ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?;
|
||||
if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed
|
||||
|| job.result.is_none() || backend_job_has_unfinished_cleanup(&tx, workspace_id, job_id)?
|
||||
{ return Err(Error::InvalidInput("input reclaim requires known completed result and completed Worker cleanup".into())); }
|
||||
}
|
||||
if job.input_disposition != JobInputDisposition::Available
|
||||
&& reason != JobInputRetirementReason::ExplicitErasure
|
||||
{ return Err(Error::InvalidInput("retired Backend Job input cannot be restored or reclassified".into())); }
|
||||
let mut metadata = job.request.clone();
|
||||
metadata.input = serde_json::Value::Null;
|
||||
tx.execute("UPDATE backend_jobs SET request_json=?3, input_disposition=?4 WHERE workspace_id=?1 AND job_id=?2", params![workspace_id,job_id,serde_json::to_string(&metadata).map_err(|e| Error::Store(e.to_string()))?,desired.as_str()])?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::Store("retired Backend Job disappeared".into()))?;
|
||||
tx.commit()?;
|
||||
reclaim_backend_job_input_storage(conn)?;
|
||||
Ok(job)
|
||||
})
|
||||
}
|
||||
|
||||
fn reserve_backend_job_retry(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
@@ -7368,6 +7424,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let mut job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
job.load_executable_request()?;
|
||||
let previous_id = attempt_id(job_id, job.current_attempt);
|
||||
let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)?
|
||||
.ok_or_else(|| Error::Store("Backend Job current attempt is missing".to_string()))?;
|
||||
@@ -7407,7 +7464,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
}
|
||||
tx.execute(
|
||||
"INSERT INTO backend_job_attempts (workspace_id, job_id, attempt_id, attempt, input_digest, state, deadline_at, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, 'reserved', ?6, ?7, ?7)",
|
||||
params![workspace_id, job_id, next_id, next as i64, job.request.input_digest()?, deadline, now],
|
||||
params![workspace_id, job_id, next_id, next as i64, job.input_digest, deadline, now],
|
||||
)?;
|
||||
job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::Store("retried Backend Job is missing".to_string()))?;
|
||||
@@ -7446,7 +7503,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
"SELECT job.job_id FROM backend_jobs job JOIN backend_job_attempts attempt
|
||||
ON attempt.workspace_id = job.workspace_id AND attempt.job_id = job.job_id
|
||||
AND attempt.attempt = job.current_attempt
|
||||
WHERE job.workspace_id = ?1 AND job.state = 'pending'
|
||||
WHERE job.workspace_id = ?1 AND job.state = 'pending' AND job.input_disposition = 'available'
|
||||
AND attempt.state = 'dispatched' AND attempt.runtime_id = ?2 AND attempt.worker_id = ?3
|
||||
AND attempt.runtime_run_id IS NOT NULL",
|
||||
params![workspace_id, worker.runtime_id.as_str(), worker.worker_id.as_str()],
|
||||
@@ -7506,6 +7563,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
||||
if attempt.state != BackendJobAttemptState::Reserved {
|
||||
@@ -7543,7 +7601,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
) -> Result<Vec<BackendJobAttemptRecord>> {
|
||||
self.with_conn(|conn| {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state = 'reserved' ORDER BY created_at, attempt_id LIMIT ?2",
|
||||
"SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='reserved' AND j.input_disposition='available' ORDER BY a.created_at, a.attempt_id LIMIT ?2",
|
||||
)?;
|
||||
let identities = statement.query_map(params![workspace_id, limit as i64], |row| {
|
||||
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||
@@ -7568,9 +7626,10 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
||||
if attempt.attempt != job.current_attempt || attempt.input_digest != job.request.input_digest()? {
|
||||
if attempt.attempt != job.current_attempt || attempt.input_digest != job.input_digest {
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Job attempt is stale for the current input digest".to_string(),
|
||||
));
|
||||
@@ -7629,6 +7688,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
}
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||
.ok_or_else(|| {
|
||||
Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`"))
|
||||
@@ -7682,7 +7743,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?;
|
||||
let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
|
||||
if submission.input_digest != job.request.input_digest()?
|
||||
if submission.input_digest != job.input_digest
|
||||
|| attempt.input_digest != submission.input_digest
|
||||
|| attempt.attempt != job.current_attempt
|
||||
{
|
||||
@@ -7709,6 +7770,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
tx.commit()?;
|
||||
return Ok(BackendJobResultAcceptance { job, attempt, replayed: true });
|
||||
}
|
||||
job.load_executable_request()?;
|
||||
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
|
||||
return Err(Error::InvalidInput(
|
||||
"Backend Job attempt no longer accepts results".to_string(),
|
||||
@@ -7776,7 +7838,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
// Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence.
|
||||
if &attempt != expected || job.current_attempt != attempt.attempt
|
||||
|| job.request.grants.subjektiv_journal.is_none()
|
||||
|| attempt.input_digest != job.request.input_digest()?
|
||||
|| attempt.input_digest != job.input_digest
|
||||
|| !matches!(job.state, BackendJobState::Unknown | BackendJobState::Completed | BackendJobState::Failed) {
|
||||
return Err(Error::InvalidInput("Journal reconciliation binding changed".into()));
|
||||
}
|
||||
@@ -12109,6 +12171,20 @@ fn read_memory_staging_resolution_record(
|
||||
})
|
||||
}
|
||||
|
||||
// The receipt/disposition commit precedes physical cleanup. An error here is
|
||||
// safely retried with the same expected digest/reason, not a new execution.
|
||||
fn reclaim_backend_job_input_storage(conn: &Connection) -> Result<()> {
|
||||
conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?;
|
||||
let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?;
|
||||
if busy != 0 {
|
||||
return Err(Error::Store(
|
||||
"Backend Job input retired; SQLite body cleanup blocked by a reader, retry retirement"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn backend_job_deadline(now: &str, timeout_seconds: u32) -> Result<String> {
|
||||
let now = chrono::DateTime::parse_from_rfc3339(now)
|
||||
.map_err(|error| Error::InvalidInput(format!("invalid Backend Job timestamp: {error}")))?;
|
||||
@@ -12172,7 +12248,7 @@ pub(crate) fn terminal_journal_job_worker(
|
||||
Ok(job.request.grants.subjektiv_journal.is_some()
|
||||
&& attempt.state.terminal()
|
||||
&& attempt.worker.as_ref() == Some(worker)
|
||||
&& attempt.input_digest == job.request.input_digest()?)
|
||||
&& attempt.input_digest == job.input_digest)
|
||||
}
|
||||
|
||||
fn read_backend_job(
|
||||
@@ -12181,13 +12257,13 @@ fn read_backend_job(
|
||||
job_id: &str,
|
||||
) -> Result<Option<BackendJobRecord>> {
|
||||
let row = conn.query_row(
|
||||
"SELECT request_json, intent_fingerprint, state, current_attempt, result_json, result_digest, failure_category, failure_detail, created_at, updated_at, completed_at FROM backend_jobs WHERE workspace_id = ?1 AND job_id = ?2",
|
||||
"SELECT request_json, intent_fingerprint, state, current_attempt, result_json, result_digest, failure_category, failure_detail, created_at, updated_at, completed_at, input_digest, input_disposition FROM backend_jobs WHERE workspace_id = ?1 AND job_id = ?2",
|
||||
params![workspace_id, job_id],
|
||||
|row| Ok((
|
||||
row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?,
|
||||
row.get::<_, i64>(3)?, row.get::<_, Option<String>>(4)?, row.get::<_, Option<String>>(5)?,
|
||||
row.get::<_, Option<String>>(6)?, row.get::<_, Option<String>>(7)?, row.get::<_, String>(8)?,
|
||||
row.get::<_, String>(9)?, row.get::<_, Option<String>>(10)?,
|
||||
row.get::<_, String>(9)?, row.get::<_, Option<String>>(10)?, row.get::<_, String>(11)?, row.get::<_, String>(12)?,
|
||||
)),
|
||||
).optional()?;
|
||||
let Some((
|
||||
@@ -12202,11 +12278,20 @@ fn read_backend_job(
|
||||
created_at,
|
||||
updated_at,
|
||||
completed_at,
|
||||
input_digest,
|
||||
input_disposition,
|
||||
)) = row
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let request: BackendJobRequest = serde_json::from_str(&request_json)
|
||||
let request_value: serde_json::Value = serde_json::from_str(&request_json)
|
||||
.map_err(|error| Error::Store(format!("invalid stored Backend Job request: {error}")))?;
|
||||
if request_value.get("input").is_none() {
|
||||
return Err(Error::Store(
|
||||
"stored Backend Job input field is missing".into(),
|
||||
));
|
||||
}
|
||||
let request: BackendJobRequest = serde_json::from_value(request_value)
|
||||
.map_err(|error| Error::Store(format!("invalid stored Backend Job request: {error}")))?;
|
||||
let result = result_json
|
||||
.map(|value| serde_json::from_str(&value))
|
||||
@@ -12218,6 +12303,9 @@ fn read_backend_job(
|
||||
workspace_id: workspace_id.to_string(),
|
||||
request,
|
||||
intent_fingerprint,
|
||||
input_digest,
|
||||
input_disposition: serde_json::from_value(serde_json::Value::String(input_disposition))
|
||||
.map_err(|e| Error::Store(format!("invalid Job input disposition: {e}")))?,
|
||||
state: BackendJobState::parse(&state)?,
|
||||
current_attempt,
|
||||
result,
|
||||
@@ -15130,6 +15218,24 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? != 102 {
|
||||
return Err(Error::Store(
|
||||
"Job input retention migration requires schema 102".into(),
|
||||
));
|
||||
}
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
tx.execute_batch("ALTER TABLE backend_jobs ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));")?;
|
||||
tx.execute_batch(crate::backend_job::INPUT_RETENTION_TRIGGERS)?;
|
||||
tx.execute(
|
||||
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(103,?1)",
|
||||
["Job input retirement independent of immutable receipts"],
|
||||
)?;
|
||||
verify_workspace_resource_constraints(&tx)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? != 101 {
|
||||
return Err(Error::Store(
|
||||
@@ -18534,7 +18640,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
assert!(conn.execute("INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) VALUES('invalid','missing','Invalid','active','1','1')", []).is_err());
|
||||
}
|
||||
|
||||
fn prepare_retained_schema(path: &Path, version: i64) {
|
||||
pub(super) fn prepare_retained_schema(path: &Path, version: i64) {
|
||||
// Cache only empty preconditions, built from frozen DDL and real retained
|
||||
// migrations. The test's seeded rows and migration under test are never shared.
|
||||
static TEMPLATES: std::sync::OnceLock<Mutex<std::collections::BTreeMap<i64, Connection>>> =
|
||||
@@ -18914,6 +19020,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
|
||||
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
|
||||
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
|
||||
WorkspaceSchemaMigrationStep {version:103, name:"Job input retirement independent of immutable receipts".into()},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -19026,6 +19133,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(100, "Provisional users and scoped passkey activation".to_string()),
|
||||
(101, "Durable explicit Subject body release".to_string()),
|
||||
(102, "Durable failed Subject creation recovery".to_string()),
|
||||
(103, "Job input retirement independent of immutable receipts".to_string()),
|
||||
]
|
||||
);
|
||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||
@@ -21293,7 +21401,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
drop(conn);
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
|
||||
reopened
|
||||
.with_conn(|c| verify_schema_history(c, 102))
|
||||
.with_conn(|c| verify_schema_history(c, LATEST_SCHEMA_VERSION))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
@@ -22232,7 +22340,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(99, "Explicit hosting administrators and self-registration policy".into()),
|
||||
(100, "Provisional users and scoped passkey activation".into()),
|
||||
(101, "Durable explicit Subject body release".into()),
|
||||
(102, "Durable failed Subject creation recovery".into())]);
|
||||
(102, "Durable failed Subject creation recovery".into()),
|
||||
(103, "Job input retirement independent of immutable receipts".into())]);
|
||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||
}
|
||||
@@ -26150,6 +26259,321 @@ INSERT INTO worker_registry (
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Storage component tests: seed already-saved result/cleanup receipts, without
|
||||
// constructing an execution Worker or claiming a model/dispatch integration.
|
||||
fn seed_completed_job_receipts(store: &SqliteWorkspaceStore, id: &str, cleanup: &str) {
|
||||
let result = serde_json::json!({"stored":true});
|
||||
let (digest, json) = result_digest(&result, job::ABSOLUTE_MAX_RESULT_BYTES).unwrap();
|
||||
store.with_conn(|conn| {
|
||||
conn.execute("UPDATE backend_job_attempts SET state='completed',result_json=?2,result_digest=?3,worker_cleanup_state=?4,worker_cleanup_completed_at=?5 WHERE job_id=?1", params![id,json,digest,cleanup,RESOURCE_JOB_NOW])?;
|
||||
conn.execute("UPDATE backend_jobs SET state='completed',result_json=?2,result_digest=?3,completed_at=?4 WHERE job_id=?1", params![id,json,digest,RESOURCE_JOB_NOW])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_job_input_reclaim_preserves_receipts_reopen_and_exact_replay() {
|
||||
use crate::backend_job::{JobInputDisposition, JobInputRetirementReason};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("input.sqlite");
|
||||
let store = SqliteWorkspaceStore::open_api_fixture(&path).unwrap();
|
||||
seed_resource_job_workspace(&store).await;
|
||||
let original = resource_job_request("reclaim", "subject-a", "candidate");
|
||||
let reservation = store
|
||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.reclaim_backend_job_input(
|
||||
"resource-workspace",
|
||||
"reclaim",
|
||||
&reservation.job.input_digest,
|
||||
JobInputRetirementReason::Completed
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
seed_completed_job_receipts(&store, "reclaim", "pending");
|
||||
assert!(
|
||||
store
|
||||
.reclaim_backend_job_input(
|
||||
"resource-workspace",
|
||||
"reclaim",
|
||||
&reservation.job.input_digest,
|
||||
JobInputRetirementReason::Completed
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
store.with_conn(|conn| { conn.execute("UPDATE backend_job_attempts SET worker_cleanup_state='completed' WHERE job_id='reclaim'",[])?; Ok(()) }).unwrap();
|
||||
let before = store
|
||||
.get_backend_job("resource-workspace", "reclaim")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let attempt = store
|
||||
.get_backend_job_attempt(
|
||||
"resource-workspace",
|
||||
"reclaim",
|
||||
&reservation.attempt.attempt_id,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.reclaim_backend_job_input(
|
||||
"resource-workspace",
|
||||
"reclaim",
|
||||
"wrong",
|
||||
JobInputRetirementReason::Completed
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let retired = store
|
||||
.reclaim_backend_job_input(
|
||||
"resource-workspace",
|
||||
"reclaim",
|
||||
&before.input_digest,
|
||||
JobInputRetirementReason::Completed,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed);
|
||||
assert!(retired.request.input.is_null());
|
||||
assert!(retired.load_executable_request().is_err());
|
||||
assert_eq!(retired.input_digest, original.input_digest().unwrap());
|
||||
assert_eq!(retired.intent_fingerprint, before.intent_fingerprint);
|
||||
assert_eq!(retired.request.grants, original.grants);
|
||||
assert_eq!(retired.result, before.result);
|
||||
assert_eq!(
|
||||
store
|
||||
.get_backend_job_attempt("resource-workspace", "reclaim", &attempt.attempt_id)
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
attempt
|
||||
);
|
||||
drop(store);
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let replay = reopened
|
||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
assert!(replay.replayed);
|
||||
assert_eq!(replay.job, retired);
|
||||
assert_eq!(
|
||||
replay.outcome().input_digest,
|
||||
original.input_digest().unwrap()
|
||||
);
|
||||
let mut changed = original.clone();
|
||||
changed.input = serde_json::json!({"changed":true});
|
||||
assert!(
|
||||
reopened
|
||||
.reserve_backend_job("resource-workspace", &changed, RESOURCE_JOB_NOW)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.reserve_backend_job_retry("resource-workspace", "reclaim", RESOURCE_JOB_NOW)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.claim_backend_job_attempt_dispatch(
|
||||
"resource-workspace",
|
||||
"reclaim",
|
||||
&attempt.attempt_id,
|
||||
RESOURCE_JOB_NOW
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(reopened.with_conn(|conn| { conn.execute("UPDATE backend_jobs SET input_disposition='available',request_json=?1 WHERE job_id='reclaim'", [serde_json::to_string(&original).unwrap()])?; Ok(()) }).is_err());
|
||||
let erased = reopened
|
||||
.reclaim_backend_job_input(
|
||||
"resource-workspace",
|
||||
"reclaim",
|
||||
&retired.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
|
||||
assert_eq!(erased.state, BackendJobState::Completed);
|
||||
assert_eq!(erased.result, before.result);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_job_input_unknown_retention_and_legacy_invalidation_do_not_reinterpret_outcome()
|
||||
{
|
||||
use crate::backend_job::{JobInputDisposition, JobInputRetirementReason};
|
||||
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
|
||||
seed_resource_job_workspace(&store).await;
|
||||
for (id, state) in [
|
||||
("unfinished", None),
|
||||
("unknown-input", Some(BackendJobAttemptState::Unknown)),
|
||||
("failed-input", Some(BackendJobAttemptState::Failed)),
|
||||
] {
|
||||
let original = resource_job_request(id, id, "candidate");
|
||||
let reservation = store
|
||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
if let Some(state) = state {
|
||||
store
|
||||
.finish_backend_job_attempt(
|
||||
"resource-workspace",
|
||||
id,
|
||||
&reservation.attempt.attempt_id,
|
||||
state,
|
||||
"fixture",
|
||||
"known stored outcome",
|
||||
RESOURCE_JOB_NOW,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let before = store
|
||||
.get_backend_job("resource-workspace", id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.reclaim_backend_job_input(
|
||||
"resource-workspace",
|
||||
id,
|
||||
&before.input_digest,
|
||||
JobInputRetirementReason::Completed
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(before.load_executable_request().unwrap(), &original);
|
||||
assert_eq!(
|
||||
store
|
||||
.get_backend_job("resource-workspace", id)
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
before
|
||||
);
|
||||
let retired = store
|
||||
.reclaim_backend_job_input(
|
||||
"resource-workspace",
|
||||
id,
|
||||
&before.input_digest,
|
||||
JobInputRetirementReason::LegacyInvalidation,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
|
||||
assert_eq!(retired.state, before.state);
|
||||
assert_eq!(retired.failure_category, before.failure_category);
|
||||
assert_eq!(retired.completed_at, before.completed_at);
|
||||
assert!(retired.load_executable_request().is_err());
|
||||
assert!(
|
||||
store
|
||||
.reserve_backend_job_retry("resource-workspace", id, RESOURCE_JOB_NOW)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.claim_backend_job_attempt_dispatch(
|
||||
"resource-workspace",
|
||||
id,
|
||||
&reservation.attempt.attempt_id,
|
||||
RESOURCE_JOB_NOW
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||
.unwrap()
|
||||
.job,
|
||||
retired
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_job_input_available_loader_and_dispatch_refuse_missing_tampered_or_changed_intent()
|
||||
{
|
||||
for mutation in [
|
||||
"json_set(request_json,'$.input',json('{\"tampered\":true}'))",
|
||||
"json_remove(request_json,'$.input')",
|
||||
"json_set(request_json,'$.instruction','changed')",
|
||||
] {
|
||||
let store = SqliteWorkspaceStore::in_memory_fixture().unwrap();
|
||||
seed_resource_job_workspace(&store).await;
|
||||
let original = resource_job_request("tamper", "subject", "candidate");
|
||||
let reservation = store
|
||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement")?;
|
||||
conn.execute_batch(&format!(
|
||||
"UPDATE backend_jobs SET request_json={mutation}"
|
||||
))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
if let Ok(Some(job)) = store.get_backend_job("resource-workspace", "tamper") {
|
||||
assert!(job.load_executable_request().is_err());
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.claim_backend_job_attempt_dispatch(
|
||||
"resource-workspace",
|
||||
"tamper",
|
||||
&reservation.attempt.attempt_id,
|
||||
RESOURCE_JOB_NOW
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_job_input_migration_preserves_original_identity_and_rolls_back_failure() {
|
||||
use crate::backend_job::{JobInputDisposition, JobInputRetirementReason};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("migration.sqlite");
|
||||
let store = SqliteWorkspaceStore::open_api_fixture(&path).unwrap();
|
||||
seed_resource_job_workspace(&store).await;
|
||||
let original = resource_job_request("upgrade", "subject", "candidate");
|
||||
let before = store
|
||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
store.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
||||
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
|
||||
}).unwrap();
|
||||
drop(store);
|
||||
assert!(SqliteWorkspaceStore::open_for_test(&path).is_err());
|
||||
let conn = Connection::open(&path).unwrap();
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 102);
|
||||
assert!(!column_exists(&conn, "backend_jobs", "input_disposition").unwrap());
|
||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement")
|
||||
.unwrap();
|
||||
drop(conn);
|
||||
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
.get_backend_job("resource-workspace", "upgrade")
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
before.job
|
||||
);
|
||||
let erased = reopened
|
||||
.reclaim_backend_job_input(
|
||||
"resource-workspace",
|
||||
"upgrade",
|
||||
&before.job.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
|
||||
assert_eq!(erased.state, BackendJobState::Pending);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||
.unwrap()
|
||||
.job,
|
||||
erased
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_job_resource_reserve_converges_across_connections_without_changing_input() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -231,12 +231,10 @@ pub fn result_facts(
|
||||
current_ticket: &ticket::Ticket,
|
||||
) -> Result<TicketItemCheckFacts> {
|
||||
let input = parse_input(&acceptance.job.request.input)?;
|
||||
let result = validate_result(
|
||||
&acceptance.job.request.input,
|
||||
acceptance.job.result.as_ref().ok_or_else(|| {
|
||||
Error::InvalidInput("completed Ticket item checker Job has no result".to_string())
|
||||
})?,
|
||||
)?;
|
||||
let result_value = acceptance.job.result.as_ref().ok_or_else(|| {
|
||||
Error::InvalidInput("completed Ticket item checker Job has no result".to_string())
|
||||
})?;
|
||||
let result = validate_result(&acceptance.job.request.input, result_value)?;
|
||||
if input.content_digest != content_digest(current_ticket)
|
||||
|| input.ticket_id != current_ticket.meta.id
|
||||
{
|
||||
@@ -246,21 +244,17 @@ pub fn result_facts(
|
||||
return Ok(TicketItemCheckFacts::NoFindings);
|
||||
}
|
||||
|
||||
Ok(TicketItemCheckFacts::Findings(render_result(acceptance)?))
|
||||
Ok(TicketItemCheckFacts::Findings(render_result(
|
||||
&acceptance.job.request.input,
|
||||
result_value,
|
||||
)?))
|
||||
}
|
||||
|
||||
/// Stable rendering from the original checked snapshot. Used to pin pre-upgrade
|
||||
/// unacknowledged requests without reinterpreting them against newer Ticket text.
|
||||
pub(crate) fn render_result(acceptance: &BackendJobResultAcceptance) -> Result<String> {
|
||||
let input = parse_input(&acceptance.job.request.input)?;
|
||||
let result = validate_result(
|
||||
&acceptance.job.request.input,
|
||||
acceptance
|
||||
.job
|
||||
.result
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::InvalidInput("completed checker has no result".into()))?,
|
||||
)?;
|
||||
pub(crate) fn render_result(input: &Value, result: &Value) -> Result<String> {
|
||||
let checked = parse_input(input)?;
|
||||
let result = validate_result(input, result)?;
|
||||
if result.findings.is_empty() {
|
||||
return Err(Error::InvalidInput(
|
||||
"clean checker result has no advisory body".into(),
|
||||
@@ -268,7 +262,7 @@ pub(crate) fn render_result(acceptance: &BackendJobResultAcceptance) -> Result<S
|
||||
}
|
||||
let mut content = format!(
|
||||
"[Ticket item checker advisory]\nTicket: {}\nContent digest: {}\n\nThis is a post-save advisory based only on the inspected Ticket text. It is not a new user request, approval gate, workflow blocker, or implementation instruction.\n",
|
||||
input.ticket_resource_key, input.content_digest
|
||||
checked.ticket_resource_key, checked.content_digest
|
||||
);
|
||||
for (index, finding) in result.findings.iter().enumerate() {
|
||||
let quoted = serde_json::to_string(&finding.quote).map_err(|error| {
|
||||
@@ -467,6 +461,8 @@ mod tests {
|
||||
job: BackendJobRecord {
|
||||
workspace_id: "checker-unit".into(),
|
||||
intent_fingerprint: request.fingerprint().unwrap(),
|
||||
input_digest: request.input_digest().unwrap(),
|
||||
input_disposition: crate::backend_job::JobInputDisposition::Available,
|
||||
request,
|
||||
state: BackendJobState::Completed,
|
||||
current_attempt: 1,
|
||||
|
||||
@@ -12,15 +12,15 @@
|
||||
|
||||
## 有界な読取り
|
||||
|
||||
Job inputには今回のimmutable selected_rangesがあり、原資料のraw本文はありません。selected_rangesは凍結された処理対象の指定であって、本文を既に受け取った・読んだ・反映した証拠ではありません。Hostが示すoriginsや、Source本文のoriginを由来として扱い、モデル出力や派生要約をユーザー発言に読み替えません。
|
||||
Job inputには今回のimmutable selected_rangesと、その範囲のraw本文・順序・由来が保存されています。この不変入力だけを原資料として扱います。Hostは保存済みinput digestと実際の配送計画を検証して入力供給をattestしますが、入力供給は理解・記録・反映完了の証拠ではありません。Hostが示すoriginsや本文のoriginを由来として扱い、モデル出力や派生要約をユーザー発言に読み替えません。
|
||||
|
||||
Sourceの`run_boundary`はHostが保存した正常終了・pause/yield/resume・取消・エラー・放棄などの実行境界です。発言やtool結果を偽装するものではありません。`completed`もそのoutcomeとinterruptedを確認し、仕事の達成や出来事のfixed化とは区別します。本文の差分なしで後から届く中断も新しい確定事実として扱いますが、既出の本文を二重に記録しません。
|
||||
|
||||
1. 最初にReadJournalStateで開始時と現在のmetadata、openの有無、前のfixed終端、今回のimmutable selected_rangesを確認します。ReadState、receipt照会、ReadJournalOperationCoverageは本文取得前でも使えますが、本文取得の代わりにはなりません。
|
||||
2. 全selected_rangesについて、古い順にReadJournalTextのSource targetを使い、Hostが指定したpointerと各rangeのbytes.startから原資料を有界に実取得します。next_byteとcontent_tokenで続け、各rangeのbytes.endまで漏れなく読みます。entry全体ではなく今回選択された各rangeの全bytesが対象です。現在地とopen本文/ongoingはinitial/current targetで別に必要なchunkを読みます。
|
||||
3. 全selected_rangesのSource本文を取得し終えるまで、Create/Rewrite/Fix、UpdateJournalPosition、ConfirmJournalUnchanged、CheckpointJournalを呼びません。Hostは実供給したbytesをdurable delivery ledgerで検査し、未取得があれば記録操作をConflict拒否します。source fetch失敗やpreview、一部chunk、descriptorだけを実読取りに数えず、未読rangeを反映済みにしません。取得不能は不明・未反映として残し、本文を推測で補いません。
|
||||
1. 最初にReadJournalStateで開始時と現在のmetadata、openの有無、前のfixed終端、今回のimmutable selected_rangesを確認します。receipt照会、ReadJournalOperationCoverageも保存状態の確認に使いますが、入力本文を読む代わりにはなりません。
|
||||
2. 全selected_rangesについて、古い順にJob input内の対応するraw本文を読みます。entry全体ではなく今回選択された各rangeの全bytesが対象です。ReadJournalTextのSource targetで元Sessionを取り直しません。このtargetは拒否され、ライブSessionへのfallbackもありません。現在地とopen本文/ongoingはReadJournalTextのinitial/current targetで別に必要なchunkを読み、next_byteとcontent_tokenで継続します。
|
||||
3. 入力本文を読んでからCreate/Rewrite/Fix、UpdateJournalPosition、ConfirmJournalUnchanged、CheckpointJournalを逐次保存します。descriptor、preview、派生要約だけを本文の代わりにしません。本文の欠損や不一致があれば明示的な失敗として残し、元Sessionの再取得や推測で補いません。配送attestationを自称したり、入力供給だけでcheckpointを進めたりしません。
|
||||
|
||||
全Source取得後もcovered_rangesへは実際に記録へ反映した範囲だけを渡します。生成中に増えた会話は後続Jobの対象であり、同じgenerationのselected_rangesや入力へ混ぜません。別Sessionを取り込まず、compaction summaryや注入文を未記録の原履歴の代用品・独立した新しい証拠にしません。
|
||||
入力本文を読んだ後もcovered_rangesへは実際に記録へ反映した範囲だけを渡します。生成中に増えた会話は後続Jobの対象であり、同じgenerationのselected_rangesや入力へ混ぜません。別Sessionを取り込まず、compaction summaryや注入文を未記録の原履歴の代用品・独立した新しい証拠にしません。
|
||||
|
||||
## 逐次保存
|
||||
|
||||
|
||||
Reference in New Issue
Block a user