Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat(journal): execute frozen body inputs with durable delivery and erasure fences

Browse Source
This commit is contained in:
Hare
2026-10-27 08:03:43 +09:00
parent 6776973e4c
commit 897c025b85
46 changed files with 10368 additions and 2180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/standalone/src/job_store.rs
+529 -20
View File
@@ -25,7 +25,46 @@ use thiserror::Error;
#[path = "job_store/legacy_migrations.rs"]
mod legacy_migrations;
const SCHEMA_VERSION: i64 = 4;
const SCHEMA_VERSION: i64 = 5;
/// Availability of the immutable input body, independent of execution outcome.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum JobInputDisposition {
Available,
Reclaimed,
Erased,
Invalidated,
}
impl JobInputDisposition {
pub fn as_str(self) -> &'static str {
match self {
Self::Available => "available",
Self::Reclaimed => "reclaimed",
Self::Erased => "erased",
Self::Invalidated => "invalidated",
}
}
}
/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum JobInputRetirementReason {
Completed,
ExplicitErasure,
LegacyInvalidation,
}
impl JobInputRetirementReason {
pub fn disposition(self) -> JobInputDisposition {
match self {
Self::Completed => JobInputDisposition::Reclaimed,
Self::ExplicitErasure => JobInputDisposition::Erased,
Self::LegacyInvalidation => JobInputDisposition::Invalidated,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -67,7 +106,11 @@ impl JobAttemptState {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct JobSnapshot {
/// Receipt metadata. After retirement `input` is null; use the verified loader to execute.
pub request: JobRequest,
pub input_digest: String,
pub intent_fingerprint: String,
pub input_disposition: JobInputDisposition,
/// Durable Host-issued domain delegation, not model input or Profile identity.
pub domain_grant: Option<Value>,
pub state: JobState,
@@ -77,14 +120,27 @@ pub struct JobSnapshot {
}
impl JobSnapshot {
/// Load only the original available body, checking both content and immutable intent.
pub fn load_executable_request(&self) -> Result<&JobRequest, JobStoreError> {
if self.input_disposition != JobInputDisposition::Available {
return Err(JobStoreError::InvalidTransition("Job input is retired"));
}
self.request.validate()?;
if self.request.input_digest()? != self.input_digest
|| self.request.fingerprint()? != self.intent_fingerprint
{
return Err(JobStoreError::Corrupt(
"immutable input or intent fingerprint mismatch".into(),
));
}
Ok(&self.request)
}
/// Shared Feature-facing read contract; no Workspace or execution identity leaks.
pub fn outcome(&self) -> job::JobOutcome {
job::JobOutcome {
job_id: self.request.job_id.clone(),
input_digest: self
.request
.input_digest()
.expect("validated immutable Job input"),
input_digest: self.input_digest.clone(),
attempt_id: self.attempt.attempt_id.clone(),
attempt: self.attempt.number,
state: match self.state {
@@ -204,9 +260,23 @@ impl JobStore {
tx.execute_batch(RECONCILIATION_SCHEMA)?;
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
SCHEMA_VERSION => {}
4 | SCHEMA_VERSION => {}
other => return Err(JobStoreError::UnsupportedSchema(other)),
}
if version < 5 {
tx.execute_batch(INPUT_RETENTION_SCHEMA)?;
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
if has_legacy_archive {
tx.execute_batch("DROP TRIGGER legacy_job_archive_immutable;
CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END;
CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive
WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available')
OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;")?;
}
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
// An accepted result is the only evidence of success. Reserved attempts
// were never started and remain pending; cancelled attempts never resume.
tx.execute_batch(
@@ -249,10 +319,7 @@ impl JobStore {
[&request.job_id],
|row| row.get(0),
)?;
if stored != fingerprint
|| snapshot.request != request
|| snapshot.domain_grant != grant
{
if stored != fingerprint || snapshot.domain_grant != grant {
return Err(JobStoreError::IntentConflict(request.job_id));
}
tx.commit()?;
@@ -334,6 +401,7 @@ impl JobStore {
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let previous = require_snapshot(&tx, job_id)?;
previous.load_executable_request()?;
if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed {
return Err(JobStoreError::InvalidTransition(
"retry requires known failure",
@@ -354,6 +422,81 @@ impl JobStore {
Ok(snapshot)
}
/// Historical snapshots for one domain Session, including terminal and retired Jobs.
pub(crate) fn list_session_snapshots(
&self,
session_id: &str,
after: Option<&str>,
limit: usize,
) -> Result<Vec<JobSnapshot>, JobStoreError> {
let mut stmt = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_domain_grants g ON g.job_id=i.job_id WHERE json_extract(g.grant_json,'$.session_id')=?1 AND (?2 IS NULL OR i.job_id>?2) ORDER BY i.job_id LIMIT ?3")?;
let ids = stmt
.query_map(
params![session_id, after, limit.clamp(1, 100) as i64],
|r| r.get::<_, String>(0),
)?
.collect::<rusqlite::Result<Vec<_>>>()?;
ids.into_iter()
.map(|id| require_snapshot(&self.conn, &id))
.collect()
}
/// The Host must fence domain mutation and delivery before erasure/invalidation.
/// This atomically prevents dispatch/retry/result acceptance, without rewriting outcome.
pub(crate) fn reclaim_input(
&mut self,
job_id: &str,
expected_input_digest: &str,
reason: JobInputRetirementReason,
) -> Result<JobSnapshot, JobStoreError> {
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let snapshot = require_snapshot(&tx, job_id)?;
if snapshot.input_digest != expected_input_digest {
return Err(JobStoreError::DigestMismatch);
}
let desired = reason.disposition();
if snapshot.input_disposition == desired {
tx.commit()?;
reclaim_retired_input_storage(&self.conn)?;
return Ok(snapshot);
}
if reason == JobInputRetirementReason::Completed
&& (snapshot.state != JobState::Completed
|| snapshot.attempt.state != JobAttemptState::Completed
|| snapshot.result.is_none())
{
return Err(JobStoreError::InvalidTransition(
"reclaim requires known completed result",
));
}
if snapshot.input_disposition != JobInputDisposition::Available
&& reason != JobInputRetirementReason::ExplicitErasure
{
return Err(JobStoreError::InvalidTransition(
"retired input cannot be restored or reclassified",
));
}
if reason == JobInputRetirementReason::Completed {
snapshot.load_executable_request()?;
}
let mut metadata = snapshot.request.clone();
metadata.input = Value::Null;
tx.execute(
"UPDATE job_intents SET request_json=?2, input_disposition=?3 WHERE job_id=?1",
params![job_id, serde_json::to_string(&metadata)?, desired.as_str()],
)?;
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
if has_legacy_archive {
tx.execute("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input',json('null')) WHERE job_id=?1", [job_id])?;
}
let snapshot = require_snapshot(&tx, job_id)?;
tx.commit()?;
reclaim_retired_input_storage(&self.conn)?;
Ok(snapshot)
}
pub(crate) fn get(&self, job_id: &str) -> Result<JobSnapshot, JobStoreError> {
require_snapshot(&self.conn, job_id)
}
@@ -383,7 +526,7 @@ impl JobStore {
let mut statement = self.conn.prepare(
"SELECT i.job_id FROM job_intents i JOIN job_attempts a
ON a.job_id = i.job_id AND a.number = i.current_attempt
WHERE i.state = 'pending' AND a.state = 'reserved' ORDER BY i.rowid",
WHERE i.state = 'pending' AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid",
)?;
let ids = statement
.query_map([], |row| row.get::<_, String>(0))?
@@ -406,6 +549,7 @@ impl JobStore {
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let snapshot = require_snapshot(&tx, job_id)?;
check_attempt(&snapshot, attempt_id)?;
snapshot.load_executable_request()?;
if snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Reserved
{
@@ -447,7 +591,7 @@ impl JobStore {
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let snapshot = require_snapshot(&tx, &bound.job_id)?;
check_attempt(&snapshot, &bound.attempt_id)?;
if snapshot.request.input_digest()? != bound.input_digest {
if snapshot.input_digest != bound.input_digest {
return Err(JobStoreError::DigestMismatch);
}
let (digest, encoded) =
@@ -466,6 +610,7 @@ impl JobStore {
tx.commit()?;
return Ok(snapshot);
}
snapshot.load_executable_request()?;
if snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Dispatched
{
@@ -649,7 +794,7 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
let row = conn
.query_row(
"SELECT i.request_json, i.state, i.acknowledged, a.attempt_id, a.number,
a.state, a.failure, a.result_json FROM job_intents i JOIN job_attempts a
a.state, a.failure, a.result_json, a.input_digest, i.fingerprint, i.input_disposition FROM job_intents i JOIN job_attempts a
ON a.job_id = i.job_id AND a.number = i.current_attempt WHERE i.job_id = ?1",
[job_id],
|row| {
@@ -662,12 +807,27 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
row.get::<_, String>(5)?,
row.get::<_, Option<String>>(6)?,
row.get::<_, Option<String>>(7)?,
row.get::<_, String>(8)?,
row.get::<_, String>(9)?,
row.get::<_, String>(10)?,
))
},
)
.optional()?;
row.map(
|(request, state, acknowledged, attempt_id, number, attempt_state, failure, result)| {
|(
request,
state,
acknowledged,
attempt_id,
number,
attempt_state,
failure,
result,
input_digest,
intent_fingerprint,
disposition,
)| {
let state = match state.as_str() {
"pending" => JobState::Pending,
"completed" => JobState::Completed,
@@ -693,8 +853,17 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
)));
}
};
let request_value: Value = serde_json::from_str(&request)?;
if request_value.get("input").is_none() {
return Err(JobStoreError::Corrupt(
"stored Job input field is missing".into(),
));
}
Ok(JobSnapshot {
request: serde_json::from_str(&request)?,
request: serde_json::from_value(request_value)?,
input_digest,
intent_fingerprint,
input_disposition: serde_json::from_value(Value::String(disposition))?,
domain_grant: conn
.query_row(
"SELECT grant_json FROM job_domain_grants WHERE job_id = ?1",
@@ -719,6 +888,37 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
.transpose()
}
// Retirement is already durable before this maintenance step. Failure must be
// retried with the same Job/digest/reason; it must never restore input. VACUUM
// removes historical free-page copies, including stores written before secure_delete.
fn reclaim_retired_input_storage(conn: &Connection) -> Result<(), JobStoreError> {
conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?;
let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?;
if busy != 0 {
return Err(JobStoreError::InvalidTransition(
"input retired; SQLite body cleanup blocked by a reader, retry retirement",
));
}
Ok(())
}
const INPUT_RETENTION_SCHEMA: &str = "
ALTER TABLE job_intents ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available'
CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));
DROP TRIGGER job_intent_immutable;
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF
job_id, fingerprint, serialization_key, max_concurrent ON job_intents
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
CREATE TRIGGER job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON job_intents
WHEN (
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
AND json_type(NEW.request_json,'$.input') = 'null'
) IS NOT TRUE
BEGIN SELECT RAISE(ABORT, 'immutable Job input retirement'); END;
";
const RECONCILIATION_SCHEMA: &str = "
DROP INDEX job_serialization_singleflight;
CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key)
@@ -885,7 +1085,11 @@ mod tests {
Some("unconfirmed"),
)
.unwrap();
store.conn.execute_batch("DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
ALTER TABLE job_intents DROP COLUMN input_disposition;
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
assert_eq!(reopened.get("first").unwrap(), unknown);
@@ -911,10 +1115,7 @@ mod tests {
JobResultSubmission {
job_id: snapshot.request.job_id.clone(),
attempt_id: snapshot.attempt.attempt_id.clone(),
input_digest: snapshot
.request
.input_digest()
.expect("validated immutable Job input"),
input_digest: snapshot.input_digest.clone(),
result,
}
}
@@ -926,6 +1127,314 @@ mod tests {
.unwrap()
}
#[test]
fn input_reclaim_reopens_as_receipt_and_exact_replay_never_rehydrates() {
let (dir, mut store) = fixture();
let original = request("retained");
let grant = Some(json!({"session_id":"session-a", "consumer":"fixture"}));
let reserved = store
.reserve_granted(original.clone(), grant.clone())
.unwrap();
let running = store
.dispatch("retained", &reserved.attempt.attempt_id, 8)
.unwrap();
let result = submission(&running, json!({"saved":true}));
let completed = store.accept(&result).unwrap();
store.ack("retained").unwrap();
let retired = store
.reclaim_input(
"retained",
&completed.input_digest,
JobInputRetirementReason::Completed,
)
.unwrap();
assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed);
assert!(retired.request.input.is_null());
assert!(retired.load_executable_request().is_err());
assert_eq!(
retired.outcome().input_digest,
original.input_digest().unwrap()
);
assert_eq!(retired.result, completed.result);
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
assert_eq!(
reopened
.reserve_granted(original.clone(), grant.clone())
.unwrap(),
retired
);
assert_eq!(reopened.accept(&result).unwrap(), retired);
assert_eq!(
reopened
.list_session_snapshots("session-a", None, 1)
.unwrap(),
vec![retired.clone()]
);
assert!(
reopened
.list_session_snapshots("session-b", None, 100)
.unwrap()
.is_empty()
);
assert!(
reopened
.list_session_snapshots("session-a", Some("retained"), 100)
.unwrap()
.is_empty()
);
let mut changed = original.clone();
changed.input = json!({"changed":true});
assert!(matches!(
reopened.reserve_granted(changed, grant),
Err(JobStoreError::IntentConflict(_))
));
assert!(reopened.retry("retained").is_err());
assert!(
reopened
.dispatch("retained", &retired.attempt.attempt_id, 8)
.is_err()
);
assert!(reopened.conn.execute("UPDATE job_intents SET request_json=?1,input_disposition='available' WHERE job_id='retained'", [serde_json::to_string(&original).unwrap()]).is_err());
let erased = reopened
.reclaim_input(
"retained",
&retired.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
assert_eq!(erased.state, JobState::Completed);
assert!(erased.acknowledged);
assert_eq!(
reopened
.reserve_granted(original, erased.domain_grant.clone())
.unwrap(),
erased
);
}
#[test]
fn unfinished_unknown_and_failed_inputs_are_kept_until_explicit_invalidation() {
let (_dir, mut store) = fixture();
for (id, state) in [
("pending", None),
("unknown", Some(JobAttemptState::Unknown)),
("failed", Some(JobAttemptState::Failed)),
] {
let original = request(id);
let snapshot = store.reserve(original.clone()).unwrap();
if let Some(state) = state {
store.dispatch(id, &snapshot.attempt.attempt_id, 8).unwrap();
store
.finish(
id,
&snapshot.attempt.attempt_id,
state,
Some("not complete"),
)
.unwrap();
}
let before = store.get(id).unwrap();
assert!(
store
.reclaim_input(
id,
&before.input_digest,
JobInputRetirementReason::Completed
)
.is_err()
);
assert_eq!(store.get(id).unwrap(), before);
assert_eq!(before.load_executable_request().unwrap(), &original);
assert!(
store
.reclaim_input(id, "wrong", JobInputRetirementReason::ExplicitErasure)
.is_err()
);
let retired = store
.reclaim_input(
id,
&before.input_digest,
JobInputRetirementReason::LegacyInvalidation,
)
.unwrap();
assert_eq!(retired.state, before.state);
assert_eq!(retired.attempt, before.attempt);
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
assert!(store.retry(id).is_err());
assert!(store.dispatch(id, &before.attempt.attempt_id, 8).is_err());
assert_eq!(store.reserve(original).unwrap(), retired);
}
}
#[test]
fn erased_dispatched_input_refuses_delayed_result_without_rewriting_unknown() {
let (dir, mut store) = fixture();
let running = dispatched(&mut store, request("erase-running"));
let result = submission(&running, json!({"late":true}));
let erased = store
.reclaim_input(
"erase-running",
&running.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert!(store.accept(&result).is_err());
assert_eq!(erased.state, JobState::Pending);
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
let recovered = reopened.get("erase-running").unwrap();
assert_eq!(recovered.state, JobState::Unknown);
assert_eq!(recovered.input_disposition, JobInputDisposition::Erased);
assert_eq!(recovered.input_digest, running.input_digest);
assert!(reopened.accept(&result).is_err());
assert!(reopened.retry("erase-running").is_err());
}
#[test]
fn available_input_loader_refuses_tamper_missing_body_and_envelope_changes() {
for mutation in [
"json_set(request_json,'$.input',json('{\"tampered\":true}'))",
"json_remove(request_json,'$.input')",
"json_set(request_json,'$.instruction','changed')",
] {
let (_dir, mut store) = fixture();
store.reserve(request("tamper")).unwrap();
// Simulate external corruption bypassing SQLite's normal write guard.
store
.conn
.execute_batch("DROP TRIGGER job_input_retirement")
.unwrap();
store
.conn
.execute_batch(&format!("UPDATE job_intents SET request_json={mutation}"))
.unwrap();
match store.get("tamper") {
Ok(snapshot) => {
assert!(snapshot.load_executable_request().is_err());
assert!(
store
.dispatch("tamper", &snapshot.attempt.attempt_id, 8)
.is_err()
);
}
Err(JobStoreError::Json(_) | JobStoreError::Corrupt(_)) => {}
other => panic!("unexpected corrupt input read: {other:?}"),
}
}
}
#[test]
fn input_retirement_cleanup_failure_is_durable_and_exact_retry_finishes_without_rehydration() {
let (dir, mut store) = fixture();
let mut original = request("physical-erasure");
let private_body = "private-input-retirement-original-bytes".repeat(1000);
original.input = json!({"body":private_body});
let reserved = store.reserve(original.clone()).unwrap();
// A reader retaining the old SQLite snapshot prevents truncation of the old WAL.
let reader = Connection::open(dir.path().join("jobs.sqlite3")).unwrap();
reader
.execute_batch("BEGIN; SELECT request_json FROM job_intents;")
.unwrap();
store.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
assert!(
store
.reclaim_input(
"physical-erasure",
&reserved.input_digest,
JobInputRetirementReason::ExplicitErasure
)
.is_err()
);
let retired = store.get("physical-erasure").unwrap();
assert_eq!(retired.input_disposition, JobInputDisposition::Erased);
assert!(retired.load_executable_request().is_err());
reader.execute_batch("ROLLBACK").unwrap();
drop(reader);
let replay = store
.reclaim_input(
"physical-erasure",
&reserved.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert_eq!(replay, retired);
assert_eq!(store.reserve(original).unwrap(), retired);
let needle = b"private-input-retirement-original-bytes";
for entry in std::fs::read_dir(dir.path()).unwrap() {
let path = entry.unwrap().path();
if path.is_file() {
let bytes = std::fs::read(&path).unwrap();
assert!(
!bytes.windows(needle.len()).any(|bytes| bytes == needle),
"retired body remained in {}",
path.display()
);
}
}
}
#[test]
fn missing_original_null_input_is_not_treated_as_available_null() {
let (_dir, mut store) = fixture();
let mut original = request("null-original");
original.input = Value::Null;
let reserved = store.reserve(original).unwrap();
assert!(reserved.load_executable_request().is_ok());
store.conn.execute_batch("DROP TRIGGER job_input_retirement; UPDATE job_intents SET request_json=json_remove(request_json,'$.input')").unwrap();
assert!(store.get("null-original").is_err());
}
#[test]
fn input_retirement_migration_preserves_receipts_and_rolls_back_failed_ddl() {
let (dir, mut store) = fixture();
let original = store.reserve(request("upgrade")).unwrap();
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
ALTER TABLE job_intents DROP COLUMN input_disposition;
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
PRAGMA user_version=4;
CREATE TRIGGER job_input_retirement BEFORE UPDATE ON job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;").unwrap();
drop(store);
let path = dir.path().join("jobs.sqlite3");
assert!(JobStore::open(&path).is_err());
let conn = Connection::open(&path).unwrap();
let version: i64 = conn
.pragma_query_value(None, "user_version", |r| r.get(0))
.unwrap();
assert_eq!(version, 4);
let present: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info('job_intents') WHERE name='input_disposition')",[],|r|r.get(0)).unwrap();
assert!(!present);
conn.execute_batch("DROP TRIGGER job_input_retirement")
.unwrap();
drop(conn);
let mut reopened = JobStore::open(&path).unwrap();
assert_eq!(reopened.get("upgrade").unwrap(), original);
assert_eq!(
reopened
.get("upgrade")
.unwrap()
.load_executable_request()
.unwrap(),
&request("upgrade")
);
let retired = reopened
.reclaim_input(
"upgrade",
&original.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert_eq!(retired.state, JobState::Pending);
assert!(
reopened
.conn
.execute_batch("UPDATE job_intents SET input_disposition='available'")
.is_err()
);
}
#[test]
fn database_configuration_schema_and_identity_constraints() {
let (_dir, mut store) = fixture();
crates/standalone/src/job_store/legacy_migrations.rs
+36
View File
@@ -143,6 +143,42 @@ mod tests {
(original, receipt, grant)
}
#[test]
fn legacy_archive_input_erasure_preserves_fingerprints_and_attempt_receipts() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("jobs.sqlite3");
seed(&path, 2);
let mut store = JobStore::open(&path).unwrap();
let before: (String, String) = store
.conn
.query_row(
"SELECT fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
let snapshot = store.get("done").unwrap();
store
.reclaim_input(
"done",
&snapshot.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
let (body, fingerprint, attempts): (String,String,String) = store.conn.query_row("SELECT request_json,fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
assert!(serde_json::from_str::<Value>(&body).unwrap()["input"].is_null());
assert_eq!((fingerprint, attempts), before);
assert!(store.conn.execute_batch("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input','restored') WHERE job_id='done'").is_err());
drop(store);
let mut reopened = JobStore::open(&path).unwrap();
let receipt = reopened
.reserve_granted(request("done"), snapshot.domain_grant)
.unwrap();
assert_eq!(receipt.state, JobState::Completed);
assert_eq!(receipt.input_disposition, JobInputDisposition::Erased);
assert!(receipt.load_executable_request().is_err());
}
#[test]
fn legacy_versions_preserve_intent_receipts_attempts_grants_and_replay() {
for version in [1, 2] {
crates/standalone/src/jobs.rs
+113 -48
View File
@@ -26,7 +26,10 @@ use job::{JobRequest, JobResultSubmission};
use tokio::sync::watch;
use crate::job_store::JobStore;
pub use crate::job_store::{JobAttempt, JobAttemptState, JobSnapshot, JobState, JobStoreError};
pub use crate::job_store::{
JobAttempt, JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot,
JobState, JobStoreError,
};
#[derive(Debug, thiserror::Error)]
pub enum JobServiceError {
@@ -40,6 +43,8 @@ pub enum JobServiceError {
Capacity,
#[error("standalone Job cleanup failed: {0}")]
Cleanup(String),
#[error("standalone Job shutdown is incomplete; execution resources are still owned")]
ShutdownIncomplete,
}
pub(crate) trait JobAttemptFence: Send + Sync {
@@ -173,6 +178,36 @@ impl StandaloneJobs {
Ok(self.with_store(|store| store.reserve(request))?)
}
/// Enumerate historical domain snapshots for a Session, including retired input receipts.
pub fn list_session_snapshots(
&self,
session_id: &str,
after: Option<&str>,
limit: usize,
) -> Result<Vec<JobSnapshot>, JobServiceError> {
self.with_store(|store| store.list_session_snapshots(session_id, after, limit))
}
/// Retire body only. The caller fences domain effects/delivered snapshots before erasure.
/// Normal reclamation also requires that execution cleanup has finished.
pub fn reclaim_input(
&self,
job_id: &str,
expected_input_digest: &str,
reason: JobInputRetirementReason,
) -> Result<JobSnapshot, JobServiceError> {
let executions = self.executions.lock().expect("Job executions poisoned");
if reason == JobInputRetirementReason::Completed
&& executions
.jobs
.get(job_id)
.is_some_and(|execution| !execution.task.is_finished())
{
return Err(JobServiceError::Capacity);
}
self.with_store(|store| store.reclaim_input(job_id, expected_input_digest, reason))
}
pub fn get(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
Ok(self.with_store(|store| store.get(job_id))?)
}
@@ -274,6 +309,12 @@ impl StandaloneJobs {
job_id: &str,
client: Option<Box<dyn LlmClient>>,
) -> Result<(), JobServiceError> {
// Preparation owns copies of the immutable input and domain adapter too. Exclude
// owner shutdown before creating them, not only before publishing the task.
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
}
let client = client.or_else(|| {
self.model_client
.lock()
@@ -282,16 +323,14 @@ impl StandaloneJobs {
.map(|c| c.clone_boxed())
});
let snapshot = self.get(job_id)?;
snapshot.load_executable_request()?;
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
let deadline = tokio::time::Instant::now() + timeout;
let fence: Arc<dyn JobAttemptFence> = Arc::new(BoundAttemptFence {
store: self.store.clone(),
job_id: job_id.into(),
attempt_id: snapshot.attempt.attempt_id.clone(),
input_digest: snapshot
.request
.input_digest()
.map_err(JobStoreError::from)?,
input_digest: snapshot.input_digest.clone(),
deadline,
});
let grant = match (
@@ -311,10 +350,6 @@ impl StandaloneJobs {
let prepared =
worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant)
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
}
executions.jobs.retain(|_, run| {
!(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(()))))
});
@@ -367,10 +402,7 @@ impl StandaloneJobs {
store: store.clone(),
job_id: id.clone(),
attempt_id: attempt_id.clone(),
input_digest: snapshot
.request
.input_digest()
.map_err(JobStoreError::from)?,
input_digest: snapshot.input_digest.clone(),
deadline,
});
let (cancel, mut cancellation) = watch::channel(false);
@@ -446,23 +478,25 @@ impl StandaloneJobs {
.clone()
.expect("completion");
result.map_err(JobServiceError::Cleanup)?;
let task = {
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions
.jobs
.get(job_id)
.is_some_and(|run| run.attempt_id == attempt)
{
executions.jobs.remove(job_id)
} else {
None
}
};
if let Some(execution) = task {
execution
.task
.await
.map_err(|error| JobServiceError::Cleanup(error.to_string()))??;
// Completion is published immediately before the task returns. Do not remove
// its ownership record until the runtime confirms that return (and future drop).
while self
.executions
.lock()
.expect("Job executions poisoned")
.jobs
.get(job_id)
.is_some_and(|run| run.attempt_id == attempt && !run.task.is_finished())
{
tokio::task::yield_now().await;
}
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions
.jobs
.get(job_id)
.is_some_and(|run| run.attempt_id == attempt)
{
executions.jobs.remove(job_id);
}
}
self.get(job_id)
@@ -476,33 +510,63 @@ impl StandaloneJobs {
}
}
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
/// Initiate owner shutdown without blocking a synchronous caller. Success confirms all
/// local executions have returned, including request/history and Feature cleanup. An
/// incomplete/error result retains the service resources and must retain the Host lease.
pub(crate) fn try_shutdown(&self) -> Result<(), JobServiceError> {
self.close();
let tasks = {
let mut executions = self.executions.lock().expect("Job executions poisoned");
std::mem::take(&mut executions.jobs)
};
let mut error = None;
for execution in tasks.into_values() {
match execution.task.await {
Ok(Ok(())) => {}
Ok(Err(failure)) => {
error = Some(failure);
}
Err(failure) => {
error = Some(JobServiceError::Cleanup(failure.to_string()));
let mut executions = self.executions.lock().expect("Job executions poisoned");
for execution in executions.jobs.values() {
if !execution.task.is_finished() {
return Err(JobServiceError::ShutdownIncomplete);
}
match execution.complete.borrow().as_ref() {
Some(Ok(())) => {}
Some(Err(error)) => return Err(JobServiceError::Cleanup(error.clone())),
None => {
return Err(JobServiceError::Cleanup(
"Internal Worker ended without cleanup confirmation".into(),
));
}
}
}
// Close SQLite before the Host releases its lease. Cloned handles may
// outlive the Host but cannot mutate or retain the live connection.
executions.jobs.clear();
// Retained external handles share these slots; they cannot retain a live store,
// provider or model client after successful owner shutdown.
self.store.lock().expect("Job store poisoned").take();
self.domain.lock().expect("Job domain poisoned").take();
self.model_client
.lock()
.expect("Job model transport poisoned")
.take();
error.map_or(Ok(()), Err)
Ok(())
}
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
self.close();
let completions: Vec<_> = self
.executions
.lock()
.expect("Job executions poisoned")
.jobs
.values()
.map(|execution| execution.complete.clone())
.collect();
// Never take execution ownership out of the shared service across an await: a
// cancelled shutdown future or a concurrent close must still see the barrier.
for mut completion in completions {
completion
.wait_for(Option::is_some)
.await
.map_err(|error| JobServiceError::Cleanup(error.to_string()))?;
}
loop {
match self.try_shutdown() {
// Only the task's final return follows its completion notification.
Err(JobServiceError::ShutdownIncomplete) => tokio::task::yield_now().await,
result => return result,
}
}
}
}
@@ -574,7 +638,8 @@ impl JobAttemptFence for BoundAttemptFence {
if snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Dispatched
|| snapshot.attempt.attempt_id != self.attempt_id
|| snapshot.request.input_digest().map_err(|e| e.to_string())? != self.input_digest
|| snapshot.input_digest != self.input_digest
|| snapshot.input_disposition != JobInputDisposition::Available
{
return Err("Job domain grant is not bound to the current live attempt".into());
}
crates/standalone/src/jobs/tests.rs
+101
View File
@@ -468,3 +468,104 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han
assert!(matches!(operation, Err(JobServiceError::Closed)));
}
}
#[tokio::test]
async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() {
// Actual generic runner + SQLite; only the provider stream is scripted/blocked.
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("jobs.sqlite3");
let jobs = StandaloneJobs::open(&path, temp.path().into()).unwrap();
let reserved = jobs.request(request("cancel-shutdown-wait")).unwrap();
let model = DomainTestClient {
stall: true,
..Default::default()
};
jobs.start_with_model_client(&reserved.request.job_id, model.clone())
.await
.unwrap();
await_model(&model).await;
{
let shutdown = jobs.shutdown();
tokio::pin!(shutdown);
assert!(futures::poll!(shutdown.as_mut()).is_pending());
// Drop the caller's waiter while cancellation/cleanup is still pending.
}
assert!(matches!(
jobs.try_shutdown(),
Err(JobServiceError::ShutdownIncomplete)
));
assert!(jobs.store.lock().unwrap().is_some());
assert!(matches!(
jobs.request(request("after-shutdown")),
Err(JobServiceError::Closed)
));
tokio::time::timeout(Duration::from_secs(5), jobs.shutdown())
.await
.unwrap()
.unwrap();
assert!(jobs.store.lock().unwrap().is_none());
let reopened = StandaloneJobs::open(&path, temp.path().into()).unwrap();
let unknown = reopened.get(&reserved.request.job_id).unwrap();
assert_eq!(unknown.attempt.state, JobAttemptState::Unknown);
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
assert_eq!(unknown.request, reserved.request);
assert_eq!(unknown.input_digest, reserved.input_digest);
assert!(reopened.retry(&unknown.request.job_id).is_err());
reopened.shutdown().await.unwrap();
}
#[tokio::test]
async fn input_retirement_service_fences_live_attempt_and_preserves_session_receipts() {
// Production service + SQLite + attempt/result fences, without a model or Worker fixture.
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("jobs.sqlite3");
let jobs = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap();
let original = request("input-retirement");
let grant = json!({"session_id":"session-a"});
let reserved = jobs
.request_granted(original.clone(), grant.clone())
.unwrap();
jobs.with_store(|store| store.dispatch("input-retirement", &reserved.attempt.attempt_id, 8))
.unwrap();
let fence = BoundAttemptFence {
store: jobs.store.clone(),
job_id: "input-retirement".into(),
attempt_id: reserved.attempt.attempt_id.clone(),
input_digest: reserved.input_digest.clone(),
deadline: tokio::time::Instant::now() + Duration::from_secs(10),
};
assert!(fence.ensure_live().is_ok());
assert!(
jobs.reclaim_input(
"input-retirement",
&reserved.input_digest,
JobInputRetirementReason::Completed
)
.is_err()
);
let erased = jobs
.reclaim_input(
"input-retirement",
&reserved.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert!(fence.ensure_live().is_err());
assert!(
sink(&jobs, "input-retirement", &reserved.attempt.attempt_id)
.submit(result("input-retirement", &reserved.attempt.attempt_id))
.is_err()
);
assert!(jobs.start("input-retirement").await.is_err());
assert_eq!(
jobs.list_session_snapshots("session-a", None, 100).unwrap(),
vec![erased.clone()]
);
assert!(
jobs.list_session_snapshots("session-b", None, 100)
.unwrap()
.is_empty()
);
assert_eq!(jobs.request_granted(original, grant).unwrap(), erased);
jobs.shutdown().await.unwrap();
}
crates/standalone/src/subjektiv.rs
+11 -1
View File
@@ -54,6 +54,8 @@ pub enum SubjectError {
Journal(#[from] subjektiv::journal::JournalError),
#[error(transparent)]
Storage(#[from] feature_storage::FeatureStorageError),
#[error("local Subject Job shutdown: {0}")]
JobShutdown(#[from] crate::jobs::JobServiceError),
#[error("local Subject storage metadata: {0}")]
Json(#[from] serde_json::Error),
#[error("local Subject storage permission or identity is invalid: {0}")]
@@ -338,8 +340,16 @@ impl SubjectConnection {
}
}
pub fn close(&mut self) -> Result<(), SubjectError> {
self.host.jobs.lock().expect("Subject Jobs poisoned").take();
self.host.live.store(false, Ordering::Release);
{
let mut jobs = self.host.jobs.lock().expect("Subject Jobs poisoned");
if let Some(service) = jobs.as_ref() {
// Signal cancellation, but retain the handle and exclusive lease until
// the generic owner confirms local request/history/resource teardown.
service.try_shutdown()?;
}
jobs.take();
}
self.host.catalog.manager.shutdown()?;
if let Some(file) = &self.lease {
file.unlock()?;
crates/standalone/src/subjektiv/journal.rs
+498 -231
View File
@@ -5,9 +5,12 @@
//! recovery authority; shutdown does not delete either original Sessions or pending requests.
use super::*;
use crate::jobs::{JobAttemptState, JobSnapshot, JobState};
use crate::jobs::{
JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, JobState,
};
use session_store::capture::{CaptureDto, CaptureLimits, RecordingPosition};
use subjektiv::api::JobAttemptBinding;
use subjektiv::journal::recording::JournalMaterial;
use subjektiv::journal::recording::JournalInput;
use subjektiv::journal::transport::*;
use subjektiv::journal::*;
use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature};
@@ -16,7 +19,6 @@ use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeat
mod tests;
pub(super) const PURPOSE: &str = "subjektiv_journal";
const INPUT_BUDGET: usize = 48 * 1024;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
@@ -27,6 +29,7 @@ struct LocalJournalGrant {
session_id: String,
journal_id: String,
ranges: Vec<JournalSourceRange>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
origins: Vec<SourceOrigin>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -35,11 +38,6 @@ struct SourceOrigin {
pointer: JournalSourcePointer,
origin: session_store::public_material::PublicMaterialOrigin,
}
impl LocalJournalGrant {
fn input(&self) -> serde_json::Value {
serde_json::json!({"selected_ranges": self.ranges, "origins": self.origins})
}
}
impl StandaloneSubjects {
/// Explicit operator erasure, separate from ordinary Worker removal. Requires the
@@ -84,32 +82,29 @@ impl StandaloneSubjects {
self.state_root.clone(),
)
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
for snapshot in jobs
.pending()
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
.into_iter()
.chain(
jobs.unknown_granted()
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?,
)
{
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
}) {
let mut after = None;
loop {
let page = jobs
.list_session_snapshots(session_id, after.as_deref(), 100)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
if page.is_empty() {
break;
}
for snapshot in &page {
jobs.cancel(&snapshot.request.job_id)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
jobs.reclaim_input(
&snapshot.request.job_id,
&snapshot.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
if snapshot.state == JobState::Completed && !snapshot.acknowledged {
jobs.acknowledge(&snapshot.request.job_id)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
}
}
}
for snapshot in jobs
.unacknowledged_results()
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
{
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
}) {
jobs.acknowledge(&snapshot.request.job_id)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
}
after = page.last().map(|job| job.request.job_id.clone());
}
session_store::FsStore::new_with_retention(
path,
@@ -590,17 +585,28 @@ impl LocalSubjectHost {
self.ensure_live()?;
let jobs = self.jobs()?;
for saved in jobs.unacknowledged_results().map_err(service_error)? {
if self.journal_grant(&saved).is_ok() {
if self.journal_grant_scope(&saved).is_ok() {
self.complete_journal_job(&jobs, &saved)?;
}
}
for pending in jobs.pending().map_err(service_error)? {
if self.journal_grant(&pending).is_ok() {
if pending.request.purpose != PURPOSE {
continue;
}
self.journal_grant_scope(&pending)?;
if pending.request.input.get("capture").is_some() {
self.journal_grant(&pending)?;
return Ok(Some((jobs, pending.request.job_id)));
}
// Old descriptor jobs never acquire new execution authority. Their immutable
// identity/input is retired, not upgraded in place; saved checkpoint wins.
let replacement = self.replace_legacy_journal_job(&jobs, &pending)?;
if let Some(id) = replacement {
return Ok(Some((jobs, id)));
}
}
for unknown in jobs.unknown_granted().map_err(service_error)? {
let Ok(grant) = self.journal_grant(&unknown) else {
let Ok(grant) = self.journal_grant_scope(&unknown) else {
continue;
};
let context = authority(self, &grant.session_id)?;
@@ -685,32 +691,96 @@ impl LocalSubjectHost {
}
}
}
let materials = self.select_journal_materials(&context, &journal.id)?;
if materials.is_empty() {
let mut retained = None;
// Saved body snapshots always win over descriptor migration. In particular,
// a failed continuation must not recapture its original text even if an
// invalidated legacy grant still bounds a larger unprocessed suffix.
for legacy_pass in [false, true] {
let mut after_job = None;
loop {
let page = jobs
.list_session_snapshots(
&journal.source_session_id,
after_job.as_deref(),
100,
)
.map_err(service_error)?;
if page.is_empty() {
break;
}
for saved in &page {
if saved.request.purpose != PURPOSE {
continue;
}
let legacy = saved.input_disposition
== JobInputDisposition::Invalidated
|| (saved.input_disposition == JobInputDisposition::Available
&& saved.request.input.get("capture").is_none());
if legacy != legacy_pass
|| (!legacy
&& saved.input_disposition != JobInputDisposition::Available)
{
continue;
}
let saved_grant = self.journal_grant_scope(saved)?;
if saved_grant.journal_id != journal.id {
continue;
}
if legacy {
if let Some(id) = self.replace_legacy_journal_job(&jobs, saved)? {
return Ok(Some((jobs, id)));
}
continue;
}
self.journal_grant(saved)?;
let original = JournalInput::decode(&saved.request.input)
.map_err(journal_error)?;
let continuation = self.build_journal_input(
&context,
&journal.id,
Some(&original.capture),
)?;
if !continuation.selected_ranges.is_empty() {
retained = Some(continuation);
break;
}
if saved.state == JobState::Completed && saved.acknowledged {
// Recover the acknowledgement-before-reclamation gap without
// turning a retained receipt into an executable empty input.
jobs.reclaim_input(
&saved.request.job_id,
&saved.input_digest,
JobInputRetirementReason::Completed,
)
.map_err(service_error)?;
}
}
if retained.is_some() {
break;
}
after_job = page.last().map(|job| job.request.job_id.clone());
}
if retained.is_some() {
break;
}
}
let input = match retained {
Some(input) => input,
None => self.build_journal_input(&context, &journal.id, None)?,
};
if input.selected_ranges.is_empty() {
continue;
}
let ranges = materials
.into_iter()
.map(|material| material.range)
.collect::<Vec<_>>();
let origins = self.journal_origins(&ranges)?;
let grant = LocalJournalGrant {
writer_binding_id: Some(self.writer.binding_id.clone()),
binding: self.binding.clone(),
session_id: journal.source_session_id,
journal_id: journal.id.clone(),
ranges,
origins,
};
let request = job::JobRequest {
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()),
purpose: PURPOSE.into(), input_ref: format!("subjektiv://{}/journal/{}", self.binding.subject_id, journal.id),
input: grant.input(),
instruction: "ReadJournalState first. Input descriptors are the frozen canonical public ranges authorized for this Job, not already read material. Read every selected source byte with bounded ReadJournalText before recording; then read initial/current position and open text in bounded pages. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
profile: "builtin:subjektiv-journal-recording".into(),
serialization_key: Some(format!("subjektiv-journal:{}", journal.id)),
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
ranges: input.selected_ranges.clone(),
origins: vec![],
};
let request = self.journal_request(&journal.id, &input)?;
let saved = jobs
.request_granted(
request,
@@ -727,6 +797,138 @@ impl LocalSubjectHost {
}
}
fn replace_legacy_journal_job(
&self,
jobs: &crate::jobs::StandaloneJobs,
old: &JobSnapshot,
) -> Result<Option<String>, SubjektivHostError> {
let grant = self.journal_grant_scope(old)?;
if old.input_disposition == JobInputDisposition::Available {
old.load_executable_request()
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
if old.request.input
!= serde_json::json!({"selected_ranges":grant.ranges,"origins":grant.origins})
{
return Err(SubjektivHostError::InvalidArgument(
"invalid legacy Journal descriptor input".into(),
));
}
} else if old.input_disposition != JobInputDisposition::Invalidated {
return Err(SubjektivHostError::InvalidArgument(
"legacy Journal input is unavailable".into(),
));
}
let context = authority(self, &grant.session_id)?;
if let Some(active) = self
.catalog
.store
.active_journal_generation(&context, &grant.journal_id)
.map_err(journal_error)?
{
if active.job.job_id != old.request.job_id {
return Err(SubjektivHostError::InvalidArgument(
"another generation owns legacy migration".into(),
));
}
self.catalog
.store
.close_journal_generation(&context, &active.id, &active.job)
.map_err(journal_error)?;
}
jobs.cancel(&old.request.job_id).map_err(service_error)?;
let current = self
.catalog
.store
.read_journal(&context, &grant.journal_id)
.map_err(journal_error)?;
let end = grant
.ranges
.last()
.ok_or_else(|| SubjektivHostError::InvalidArgument("empty legacy selection".into()))?;
let end_position = if end.bytes.end == end.source.total_bytes {
(end.source.order + 1, 0)
} else {
(end.source.order, end.bytes.end)
};
let replacement = if (current.process_order, current.process_byte) >= end_position {
None
} else {
let mut capture = self.recording_capture(&grant.session_id)?;
capture
.entries
.retain(|entry| grant.ranges.iter().any(|r| r.source.order == entry.order));
for entry in &mut capture.entries {
let range = grant
.ranges
.iter()
.find(|r| r.source.order == entry.order)
.unwrap();
let session_store::capture::CaptureBody::CanonicalPublic {
descriptor,
byte_offset,
text,
..
} = &mut entry.body
else {
unreachable!()
};
if source_descriptor(&grant.session_id, descriptor) != range.source
|| !grant.origins.iter().any(|origin| {
origin.pointer == range.source.pointer && origin.origin == descriptor.origin
})
|| !text.is_char_boundary(range.bytes.start as usize)
|| !text.is_char_boundary(range.bytes.end as usize)
{
return Err(SubjektivHostError::InvalidArgument(
"legacy original source identity/range changed".into(),
));
}
*text = text[range.bytes.start as usize..range.bytes.end as usize].to_owned();
*byte_offset = range.bytes.start;
}
let input = self.build_journal_input(&context, &grant.journal_id, Some(&capture))?;
let request = self.journal_request(&grant.journal_id, &input)?;
let next_grant = LocalJournalGrant {
ranges: input.selected_ranges.clone(),
origins: vec![],
..grant
};
Some(
jobs.request_granted(
request,
serde_json::to_value(next_grant)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
)
.map_err(service_error)?
.request
.job_id,
)
};
jobs.reclaim_input(
&old.request.job_id,
&old.input_digest,
JobInputRetirementReason::LegacyInvalidation,
)
.map_err(service_error)?;
Ok(replacement)
}
fn journal_request(
&self,
journal_id: &str,
input: &JournalInput,
) -> Result<job::JobRequest, SubjektivHostError> {
Ok(job::JobRequest {
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()), purpose: PURPOSE.into(),
input_ref: format!("subjektiv://{}/journal/{journal_id}", self.binding.subject_id),
input: serde_json::to_value(input).map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
instruction: "Input contains the frozen canonical public source bodies, provenance and byte ranges. Use only this saved input, never re-fetch Source. ReadJournalState first and read initial/current Journal position/open text in bounded pages if needed. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
profile: "builtin:subjektiv-journal-recording".into(),
serialization_key: Some(format!("subjektiv-journal:{journal_id}")),
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
})
}
fn pin_journal_session(&self, session: &str) -> Result<(), SubjektivHostError> {
let path = crate::subjektiv_sessions::authorized_sessions_path(
&self.catalog.state_root,
@@ -869,11 +1071,16 @@ impl LocalSubjectHost {
Ok(())
}
fn material_reader(
fn with_recording_boundary<T>(
&self,
session: &str,
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
{
read: impl FnOnce(
&Path,
session_store::SessionId,
session_store::SegmentId,
&session_store::retention::SessionRetentionGuard,
) -> Result<T, SubjektivHostError>,
) -> Result<T, SubjektivHostError> {
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
&self.catalog.state_root,
&self.binding.scope_id,
@@ -906,6 +1113,11 @@ impl LocalSubjectHost {
.ok_or_else(|| SubjektivHostError::InvalidArgument("invalid Session storage".into()))?
.join("worker")
.join(&record.storage_key);
let session_id = session
.parse()
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?;
let retention = crate::store::session_retention(&self.catalog.state_root);
let guard = retention.lock_live(session_id).map_err(source_error)?;
// This is the persisted active selector, not the newest filename/mtime or a model claim.
let metadata =
session_store::WorkerAggregateStore::open_read_only(metadata_root, &record.storage_key)
@@ -933,154 +1145,203 @@ impl LocalSubjectHost {
"no attested active lineage for the retained Session".into(),
)
})?;
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
&sessions,
session
.parse()
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
active,
session_store::public_material::PublicMaterialLimits {
max_segments: 256,
max_entries: 100_000,
max_record_bytes: 64 * 1024 * 1024,
},
crate::store::session_retention(&self.catalog.state_root),
)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
read(&sessions, session_id, active, &guard)
}
fn recording_capture(&self, session: &str) -> Result<CaptureDto, SubjektivHostError> {
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
session_store::capture::capture_recording_fs_guarded(
sessions,
session_id,
active,
session_store::public_material::PublicMaterialLimits::default(),
CaptureLimits {
max_bytes: 128 * 1024 * 1024,
max_entries: 100_000,
max_text_bytes: 64 * 1024 * 1024,
..Default::default()
},
guard,
)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
})
}
fn journal_sources(
&self,
session: &str,
) -> Result<Vec<JournalSourceRange>, SubjektivHostError> {
let reader = self.material_reader(session)?;
Ok(reader
.descriptors()
.map(|descriptor| JournalSourceRange {
source: source_descriptor(session, descriptor),
bytes: JournalByteRange {
start: 0,
end: descriptor.total_bytes,
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
session_store::capture::recording_catalog_fs_guarded(
sessions,
session_id,
active,
session_store::public_material::PublicMaterialLimits::default(),
guard,
|descriptors| {
descriptors
.iter()
.map(|descriptor| JournalSourceRange {
source: source_descriptor(session, descriptor),
bytes: JournalByteRange {
start: 0,
end: descriptor.total_bytes,
},
})
.collect()
},
})
.collect())
}
fn source_reader(
&self,
source: &JournalSource,
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
{
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
&self.catalog.state_root,
&self.binding.scope_id,
&self.binding.subject_id,
&self.catalog.store,
&source.session_id,
)?;
// Frozen, already admitted original pointers do not follow the Worker's later active
// Session/Segment. This reads that original lineage and still checks descriptor identity,
// canonical order and digest below; the shared service enforces the frozen byte range.
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
&sessions,
source
.session_id
.parse()
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
source.pointer.segment_id,
session_store::public_material::PublicMaterialLimits {
max_segments: 256,
max_entries: 100_000,
max_record_bytes: 64 * 1024 * 1024,
},
crate::store::session_retention(&self.catalog.state_root),
)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
}
fn journal_origins(
&self,
ranges: &[JournalSourceRange],
) -> Result<Vec<SourceOrigin>, SubjektivHostError> {
if ranges.is_empty() {
return Err(SubjektivHostError::InvalidArgument(
"empty material selection".into(),
));
}
ranges
.iter()
.map(|range| {
let reader = self.source_reader(&range.source)?;
reader
.descriptors()
.find(|descriptor| {
source_descriptor(&range.source.session_id, descriptor) == range.source
})
.map(|descriptor| SourceOrigin {
pointer: range.source.pointer.clone(),
origin: descriptor.origin.clone(),
})
.ok_or_else(|| {
SubjektivHostError::InvalidArgument(
"source provenance/identity changed".into(),
)
})
})
.collect()
}
fn journal_source_window(
&self,
source: &JournalSource,
offset: u64,
max_bytes: usize,
) -> Result<String, SubjektivHostError> {
let reader = self.source_reader(source)?;
let descriptor = reader.descriptors().find(|descriptor| source_descriptor(&source.session_id, descriptor) == *source)
.ok_or_else(|| SubjektivHostError::InvalidArgument("canonical source identity/order/digest changed or is outside the authorized Session".into()))?;
reader
.read_content(descriptor, offset, max_bytes)
.map(|window| window.content)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
}
fn journal_source_material(
&self,
source: &JournalSource,
start: u64,
limit: usize,
) -> JournalResult<JournalMaterial> {
let text = self
.journal_source_window(source, start, limit.max(4))
.map_err(|error| JournalError::Conflict(error.to_string()))?;
Ok(JournalMaterial {
range: JournalSourceRange {
source: source.clone(),
bytes: JournalByteRange {
start,
end: start + text.len() as u64,
},
},
text,
)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
})
}
fn select_journal_materials(
fn build_journal_input(
&self,
context: &JournalHostContext,
id: &str,
) -> Result<Vec<JournalMaterial>, SubjektivHostError> {
subjektiv::journal::recording::select_materials_bounded(
&self.catalog.store,
context,
id,
INPUT_BUDGET,
|source, start, limit| self.journal_source_material(source, start, limit),
)
saved: Option<&CaptureDto>,
) -> Result<JournalInput, SubjektivHostError> {
let session = self
.catalog
.store
.read_journal(context, id)
.map_err(journal_error)?
.source_session_id;
JournalInput::build(&self.catalog.store, context, id, |order, offset, budget| {
let capture = match saved {
Some(capture) => capture.clone(),
None => {
let capture = self
.with_recording_boundary(&session, |sessions, session_id, active, guard| {
session_store::capture::capture_recording_slice_fs_guarded(
sessions,
session_id,
active,
session_store::public_material::PublicMaterialLimits::default(),
RecordingPosition {
canonical_order: order,
byte_offset: offset,
},
MAX_JOURNAL_RANGES,
budget,
guard,
)
.map(|slice| slice.capture)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
})
.map_err(|e| JournalError::Conflict(e.to_string()))?;
// Admission and capture are separate saves. An append between them
// may be in replay but has not yet been accepted for this Journal.
return self.accepted_capture_prefix(context, id, capture);
}
};
let end = capture
.entries
.last()
.map(|entry| {
capture
.recording_text(entry.order)
.map(|(descriptor, start, text)| {
if start + text.len() as u64 == descriptor.total_bytes {
RecordingPosition {
canonical_order: entry.order + 1,
byte_offset: 0,
}
} else {
RecordingPosition {
canonical_order: entry.order,
byte_offset: start + text.len() as u64,
}
}
})
})
.transpose()
.map_err(|e| JournalError::Conflict(e.to_string()))?;
if end.is_none_or(|end| (order, offset) >= (end.canonical_order, end.byte_offset)) {
return Ok(CaptureDto {
entries: vec![],
..capture
});
}
capture
.recording_slice(
RecordingPosition {
canonical_order: order,
byte_offset: offset,
},
MAX_JOURNAL_RANGES,
budget,
)
.map(|slice| slice.capture)
.map_err(|e| JournalError::Conflict(e.to_string()))
})
.map_err(journal_error)
}
fn journal_grant(
fn accepted_capture_prefix(
&self,
context: &JournalHostContext,
id: &str,
mut capture: CaptureDto,
) -> JournalResult<CaptureDto> {
let mut selected = Vec::new();
for mut entry in capture.entries {
let status = match self
.catalog
.store
.read_journal_source(context, id, entry.order)
{
Ok(status) => status,
Err(JournalError::NotFound(_)) => break,
Err(error) => return Err(error),
};
let session_store::capture::CaptureBody::CanonicalPublic {
descriptor,
byte_offset,
text,
..
} = &mut entry.body
else {
return Err(JournalError::InvalidInput("non-recording capture".into()));
};
if source_descriptor(&capture.session_id, descriptor) != status.source {
return Err(JournalError::Conflict(
"captured source differs from admitted identity".into(),
));
}
let Some(accepted) = status
.accepted
.iter()
.find(|range| range.start <= *byte_offset && *byte_offset < range.end)
else {
break;
};
let end = status
.unavailable
.iter()
.filter(|range| range.end > *byte_offset)
.fold(accepted.end, |end, range| {
end.min(range.start.max(*byte_offset))
});
let mut length = text.len().min(end.saturating_sub(*byte_offset) as usize);
while !text.is_char_boundary(length) {
length -= 1;
}
if length == 0 {
break;
}
text.truncate(length);
let partial = *byte_offset + (text.len() as u64) < descriptor.total_bytes;
selected.push(entry);
if partial {
break;
}
}
capture.entries = selected;
Ok(capture)
}
fn journal_grant_scope(
&self,
snapshot: &JobSnapshot,
) -> Result<LocalJournalGrant, SubjektivHostError> {
@@ -1097,7 +1358,6 @@ impl LocalSubjectHost {
|| snapshot.request.purpose != PURPOSE
|| snapshot.request.serialization_key.as_deref()
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
|| snapshot.request.input != grant.input()
{
return Err(SubjektivHostError::InvalidArgument(
"Job input/grant does not match this Journal capability".into(),
@@ -1135,12 +1395,33 @@ impl LocalSubjectHost {
Ok(grant)
}
fn journal_grant(
&self,
snapshot: &JobSnapshot,
) -> Result<LocalJournalGrant, SubjektivHostError> {
snapshot
.load_executable_request()
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
let grant = self.journal_grant_scope(snapshot)?;
let input = JournalInput::decode(&snapshot.request.input).map_err(journal_error)?;
if input.subject_id != self.binding.subject_id
|| input.journal_id != grant.journal_id
|| input.capture.session_id != grant.session_id
|| input.selected_ranges != grant.ranges
{
return Err(SubjektivHostError::InvalidArgument(
"saved Journal body identity/ranges differ from grant".into(),
));
}
Ok(grant)
}
fn complete_journal_job(
&self,
jobs: &crate::jobs::StandaloneJobs,
snapshot: &JobSnapshot,
) -> Result<(), SubjektivHostError> {
let grant = self.journal_grant(snapshot)?;
let grant = self.journal_grant_scope(snapshot)?;
let context = authority(self, &grant.session_id)?;
let completion: JournalJobCompletion =
serde_json::from_value(snapshot.result.clone().ok_or_else(|| {
@@ -1174,6 +1455,12 @@ impl LocalSubjectHost {
}
jobs.acknowledge(&snapshot.request.job_id)
.map_err(service_error)?;
jobs.reclaim_input(
&snapshot.request.job_id,
&snapshot.input_digest,
JobInputRetirementReason::Completed,
)
.map_err(service_error)?;
if let Err(error) = self.prune_retained_workers() {
// Pruning is independent of a successful checkpoint/result. Keep the originals
// and continue draining other saved prefixes; a retention error is not model failure.
@@ -1192,47 +1479,18 @@ pub(super) fn grant(
.journal_grant(snapshot)
.map_err(|error| error.to_string())?;
let context = authority(&host, &grant.session_id).map_err(|error| error.to_string())?;
// Re-fetch selected identities before opening mutation authority. Only descriptors are
// retained in Job storage; the model must fetch actual source bytes with bounded tools.
for range in &grant.ranges {
let length = usize::try_from(
range
.bytes
.end
.checked_sub(range.bytes.start)
.ok_or("invalid source range")?,
)
.map_err(|_| "source range overflow")?;
if length > INPUT_BUDGET {
return Err("source range exceeds frozen input budget".into());
}
let text = host
.journal_source_window(&range.source, range.bytes.start, length.max(4))
.map_err(|error| error.to_string())?;
if text.len() != length {
return Err("frozen Journal source byte range changed".into());
}
}
if host
.journal_origins(&grant.ranges)
.map_err(|error| error.to_string())?
!= grant.origins
{
return Err(
"frozen Journal provenance does not match the authorized original Session".into(),
);
}
let input = JournalInput::decode(&snapshot.request.input).map_err(|e| e.to_string())?;
let input_digest = snapshot.input_digest.clone();
let generation = host
.catalog
.store
.prepare_journal_generation(
.prepare_journal_input(
&context,
&grant.journal_id,
&input,
JobAttemptBinding {
job_id: snapshot.request.job_id.clone(),
attempt_id: snapshot.attempt.attempt_id.clone(),
},
grant.ranges.clone(),
)
.map_err(|error| error.to_string())?;
let job_context = JournalJobContext::validated(
@@ -1242,12 +1500,17 @@ pub(super) fn grant(
generation.job.clone(),
)
.map_err(|error| error.to_string())?;
input
.validate_generation(&generation)
.map_err(|e| e.to_string())?;
let adapter = Arc::new(LocalJournalHost {
host,
grant,
context,
job_context,
#[cfg(test)]
generation,
input_digest,
fence,
});
Ok(worker::job::JobFeatureGrant {
@@ -1263,7 +1526,9 @@ struct LocalJournalHost {
grant: LocalJournalGrant,
context: JournalHostContext,
job_context: JournalJobContext,
#[cfg(test)]
generation: JournalGeneration,
input_digest: String,
fence: Arc<dyn crate::jobs::JobAttemptFence>,
}
impl LocalJournalHost {
@@ -1296,18 +1561,20 @@ impl JournalJobHost for LocalJournalHost {
request: JournalJobRequest,
) -> Result<JournalJobResponse, SubjektivHostError> {
let context = self.live()?;
subjektiv::journal::service::execute_bounded(
// The generic runner has supplied this exact immutable input before it can
// invoke this bound tool. Preparation alone is not evidence of delivery.
subjektiv::journal::service::attest_input_delivery(
&self.host.catalog.store,
&context,
&self.input_digest,
&self.grant.ranges,
)
.map_err(journal_error)?;
subjektiv::journal::service::execute_snapshot(
&self.host.catalog.store,
&context,
&self.input_digest,
request,
|source, start, limit| {
if source.session_id != self.generation.initial_journal.source_session_id {
return Err(JournalError::PermissionDenied(
"foreign original Session".into(),
));
}
self.host.journal_source_material(source, start, limit)
},
)
.map_err(journal_error)
}
crates/standalone/src/subjektiv/journal/tests.rs
+747 -112
View File
@@ -1,6 +1,6 @@
//! Integration of standalone attribution, original FsStore material, shared Journal storage
//! and the local Job adapter. Only the outer model/execution lease is replaced in these cases.
//! The generic Job runner is exercised separately by the standalone Jobs integration suite.
//! The scripted model also exercises saved direct input through the actual generic Job runner.
use super::*;
use crate::store::{StaleLeasePolicy, StandaloneWorkerAllocation, StandaloneWorkerStore};
@@ -144,8 +144,8 @@ impl Fixture {
.unwrap()
.contains(sentinel)
);
// Include SQLite/WAL/journal files: immutable request/grant storage must never hold
// original text, even in freed pages left behind by explicit source erasure.
// Include SQLite/WAL/journal files: erasure must scrub retired original bodies,
// including any freed pages or journal left by the immutable request store.
for entry in fs::read_dir(self.connection.jobs_path.parent().unwrap()).unwrap() {
let entry = entry.unwrap();
if entry.file_type().unwrap().is_file() {
@@ -194,9 +194,9 @@ impl Fixture {
context,
job_context,
generation,
input_digest: snapshot.input_digest.clone(),
fence: fence.clone(),
};
adapter.read_all_sources();
(adapter, fence, snapshot)
}
}
@@ -220,37 +220,6 @@ fn message(id: &str, text: &str) -> session_store::LoggedHistoryEntry {
}
}
impl LocalJournalHost {
fn read_all_sources(&self) {
for range in &self.generation.selected_ranges {
let mut offset = range.bytes.start as usize;
let mut content_token = None;
loop {
let response = self
.execute(JournalJobRequest::ReadText {
read: JournalTextRead {
target: JournalTextTarget::Source {
pointer: range.source.pointer.clone(),
},
offset,
limit: MAX_JOURNAL_READ_BYTES,
content_token: content_token.clone(),
},
})
.unwrap();
let JournalJobResponse::Text(text) = response else {
panic!()
};
match text.next_byte {
Some(next) => {
assert!(next > offset);
offset = next;
content_token = Some(text.content_token);
}
None => break,
}
}
}
}
fn state(&self) -> Result<JournalJobState, SubjektivHostError> {
match self.execute(JournalJobRequest::ReadState)? {
JournalJobResponse::State(state) => Ok(state),
@@ -362,8 +331,8 @@ fn duplicate_admission_reserves_one_job_and_excludes_private_prompt() {
assert_eq!(jobs.pending().unwrap().len(), 1);
let input = jobs.get(&id).unwrap().request.input.to_string();
assert!(
!input.contains("short request"),
"Job storage retains descriptors, never a second copy of original text"
input.contains("short request"),
"Job storage supplies the canonical body directly"
);
assert!(!input.contains("PRIVATE PROMPT"));
let foreign = SubjektivHostContext {
@@ -498,14 +467,16 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() {
&adapter.generation.job,
)
.unwrap();
let materials = fixture
let next = fixture
.host()
.select_journal_materials(&adapter.context, &adapter.grant.journal_id)
.build_journal_input(&adapter.context, &adapter.grant.journal_id, None)
.unwrap();
assert_eq!(materials[0].range.source.pointer.entry_ref, "Elast");
assert_eq!(next.selected_ranges[0].source.pointer.entry_ref, "Elast");
assert!(
materials[0]
.text
next.capture
.recording_text(next.selected_ranges[0].source.order)
.unwrap()
.2
.contains("last input while generation runs")
);
}
@@ -547,7 +518,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
let (adapter, _, _) = fixture.adapter();
let range = &adapter.grant.ranges[0];
assert!(range.bytes.end < range.source.total_bytes);
assert!(range.bytes.end - range.bytes.start < INPUT_BUDGET as u64);
assert!(range.bytes.end - range.bytes.start < job::MAX_JOB_INPUT_BYTES as u64);
let snapshot = fixture
.host()
.jobs()
@@ -557,7 +528,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
.pop()
.unwrap();
assert!(serde_json::to_vec(&snapshot.request.input).unwrap().len() < 128 * 1024);
assert!(!snapshot.request.input.to_string().contains("漢字"));
assert!(snapshot.request.input.to_string().contains("漢字"));
assert!(adapter.finish().is_err());
}
@@ -628,43 +599,35 @@ impl JournalScript {
)
} else if let Some(state) = result("state") {
let state = &state["value"];
for (index, range) in state["selected_ranges"]
.as_array()
.unwrap()
let input: serde_json::Value = request
.items
.iter()
.enumerate()
{
let mut offset = range["bytes"]["start"].as_u64().unwrap();
let mut token = Value::Null;
loop {
let id = format!("source-{index}-{offset}");
if let Some(page) = result(&id) {
let page = &page["value"];
let key = format!("{}:{id}", state["generation_id"].as_str().unwrap());
let text = page["text"].as_str().unwrap().to_owned();
if let Some(previous) =
self.source_reads.lock().unwrap().insert(key, text.clone())
{
assert_eq!(
previous, text,
"frozen source read changed within a generation"
);
}
if let Some(next) = page["next_byte"].as_u64() {
assert!(next > offset);
offset = next;
token = page["content_token"].clone();
} else {
break;
}
} else {
let input = json!({"target":{"target":"source","pointer":range["source"]["pointer"]},"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token});
return vec![
Event::tool_use_start(0, &id, "ReadJournalText"),
Event::tool_input_delta(0, input.to_string()),
Event::tool_use_stop(0),
];
}
.filter_map(Item::as_text)
.find_map(|text| {
text.lines()
.find_map(|line| line.strip_prefix("input_json: "))
})
.map(|json| serde_json::from_str(json).unwrap())
.expect("direct saved Journal input");
let input = JournalInput::decode(&input).unwrap();
assert_eq!(
serde_json::to_value(&input.selected_ranges).unwrap(),
state["selected_ranges"]
);
for entry in &input.capture.entries {
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
let key = format!(
"{}:{}:{offset}",
state["generation_id"].as_str().unwrap(),
entry.order
);
if let Some(previous) = self
.source_reads
.lock()
.unwrap()
.insert(key, text.to_owned())
{
assert_eq!(previous, text, "saved source changed within generation");
}
}
let (name, action) = if state["current_open"].is_null() {
@@ -1166,6 +1129,11 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
.items;
assert_eq!(entries.len(), 1);
let saved_entry = entries[0].id.clone();
let original_input = jobs.get(&id).unwrap().request.input;
fixture.append(
"newer-than-snapshot",
"NEW INPUT MUST NOT MIX INTO RESUMED SNAPSHOT",
);
jobs.shutdown().await.unwrap();
fixture.connection.close().unwrap();
let mut reopened = StandaloneSubjects::open_existing(&state)
@@ -1176,8 +1144,29 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
.unwrap();
reopened.host.bind_jobs(jobs.clone()).unwrap();
assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown);
// Domain reconciliation inspects the now-fenced prior attempt and preserves its receipts.
// Deny every original-source read while preparing and running the replacement.
let source_path = fixture
.sessions
.root_dir()
.join(&fixture.context.session_id);
let denied = fixture
._temp
.path()
.join("temporarily-unavailable-original");
fs::rename(&source_path, &denied).unwrap();
let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap();
let replacement_input = jobs.get(&replacement).unwrap().request.input;
assert_eq!(replacement_input["capture"], original_input["capture"]);
assert!(
!replacement_input
.to_string()
.contains("NEW INPUT MUST NOT MIX")
);
assert_eq!(jobs.get(&id).unwrap().request.input, original_input);
assert_eq!(
jobs.get(&id).unwrap().input_disposition,
JobInputDisposition::Available
);
assert_ne!(
replacement, id,
"unknown external execution is not blindly replayed"
@@ -1205,8 +1194,26 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
.items;
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].id, saved_entry);
fs::rename(&denied, &source_path).unwrap();
let journal = reopened
.host
.catalog
.store
.read_journal(&context, &journal_id)
.unwrap();
let original = JournalInput::decode(&original_input).unwrap();
let end = original.selected_ranges.last().unwrap();
assert_eq!(
(journal.process_order, journal.process_byte),
if end.bytes.end == end.source.total_bytes {
(end.source.order + 1, 0)
} else {
(end.source.order, end.bytes.end)
}
);
assert!(reopened.host.admit_journal(&fixture.context).unwrap());
assert!(
reopened
!reopened
.host
.session_fully_reflected(&fixture.context.session_id)
.unwrap()
@@ -1302,9 +1309,9 @@ fn missing_original_material_blocks_worker_removal_without_erasing_state() {
}
#[test]
fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
fn frozen_input_survives_active_session_change_and_source_tool_remains_refused() {
let fixture = Fixture::new("old Session original");
let (adapter, _, _) = fixture.adapter();
let (adapter, _, snapshot) = fixture.adapter();
let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap();
let worker = fixture._allocation.as_ref().unwrap().worker_id();
let record = workers.load(worker).unwrap();
@@ -1318,22 +1325,30 @@ fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
)),
))
.unwrap();
let response = adapter
.execute(JournalJobRequest::ReadText {
read: JournalTextRead {
target: JournalTextTarget::Source {
pointer: adapter.generation.selected_ranges[0].source.pointer.clone(),
},
offset: 0,
limit: MAX_JOURNAL_READ_BYTES,
content_token: None,
},
})
.unwrap();
let JournalJobResponse::Text(page) = response else {
panic!()
};
assert!(page.text.contains("old Session original"));
assert!(
adapter
.execute(JournalJobRequest::ReadText {
read: JournalTextRead {
target: JournalTextTarget::Source {
pointer: adapter.generation.selected_ranges[0].source.pointer.clone()
},
offset: 0,
limit: MAX_JOURNAL_READ_BYTES,
content_token: None
}
})
.is_err(),
"original Source lookup is explicitly unavailable"
);
let input = JournalInput::decode(&snapshot.request.input).unwrap();
assert!(
input
.capture
.recording_text(0)
.unwrap()
.2
.contains("old Session original")
);
assert!(
fixture
.host()
@@ -1512,14 +1527,18 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins()
const SENTINEL: &str = "original-plaintext-sentinel-must-never-enter-immutable-job-storage";
let mut fixture = Fixture::new(SENTINEL);
let (_, _, snapshot) = fixture.adapter();
fixture.assert_job_storage_excludes(&snapshot, SENTINEL);
assert!(snapshot.request.input.to_string().contains(SENTINEL));
let state = fixture.host().catalog.state_root.clone();
let binding = fixture.connection.binding.clone();
let session: session_store::SessionId = fixture.context.session_id.parse().unwrap();
let old_reader = fixture
.host()
.material_reader(&fixture.context.session_id)
.unwrap();
let old_reader = SessionPublicMaterialReader::open_fs_with_retention(
fixture.sessions.root_dir(),
session,
fixture.segment,
PublicMaterialLimits::default(),
crate::store::session_retention(&state),
)
.unwrap();
let descriptor = old_reader.descriptors().next().unwrap().clone();
let (worker, workers) = fixture.stop_worker();
assert!(workers.delete(worker).is_err());
@@ -1759,14 +1778,12 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
let snapshot = jobs.get(&next).unwrap();
let grant = fixture.host().journal_grant(&snapshot).unwrap();
assert_eq!(grant.ranges.len(), 1);
assert_eq!(
grant.origins[0].origin,
PublicMaterialOrigin::HostRunBoundary
);
let content = fixture
.host()
.journal_source_window(&grant.ranges[0].source, 0, 4096)
let input = JournalInput::decode(&snapshot.request.input).unwrap();
let (descriptor, _, content) = input
.capture
.recording_text(grant.ranges[0].source.order)
.unwrap();
assert_eq!(descriptor.origin, PublicMaterialOrigin::HostRunBoundary);
let value: serde_json::Value = serde_json::from_str(&content).unwrap();
assert_eq!(value["kind"], "run_boundary");
assert_eq!(value["boundary"]["kind"], "errored");
@@ -1791,3 +1808,621 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
assert!(fixture.host().next_journal_job().unwrap().is_none());
jobs.shutdown().await.unwrap();
}
/// Original FsStore -> bounded common capture -> persisted Job -> actual generic runner ->
/// sequential Journal tools. Only the LLM transport is scripted. Source files are deliberately
/// unavailable after persistence, and input bodies are inspected in the provider request.
#[tokio::test]
async fn saved_body_jobs_process_over_a_hundred_entries_and_utf8_without_live_source_reads() {
let fixture = Fixture::new("oldest original");
for index in 0..115 {
fixture.append(
&format!("entry-{index}"),
&format!("public original {index}"),
);
}
fixture.append("long-utf8", &"漢字🙂\n\"".repeat(24_000));
fixture.host().admit_journal(&fixture.context).unwrap();
let expected = fixture
.host()
.recording_capture(&fixture.context.session_id)
.unwrap();
let source_path = fixture
.sessions
.root_dir()
.join(&fixture.context.session_id);
let denied = fixture._temp.path().join("denied-original-source");
let mut seen: std::collections::BTreeMap<u64, String> = Default::default();
let mut jobs_processed = 0;
loop {
let Some((jobs, id)) = fixture.host().next_journal_job().unwrap() else {
break;
};
let saved = jobs.get(&id).unwrap();
let input = JournalInput::decode(&saved.request.input).unwrap();
assert!(
serde_json::to_vec(&saved.request.input).unwrap().len() <= job::MAX_JOB_INPUT_BYTES
);
assert!(input.capture.entries.len() <= MAX_JOURNAL_RANGES);
for entry in &input.capture.entries {
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
let previous = seen.entry(entry.order).or_default();
assert_eq!(
previous.len() as u64,
offset,
"no gap, preview or duplicate prefix"
);
previous.push_str(text);
}
// Deny all catalogs/reads, including grant preparation. The model still receives
// exactly the persisted body, and the Host must not follow the active pointer.
fs::rename(&source_path, &denied).unwrap();
let model = JournalScript::default();
jobs.start_with_model_client(&id, model.clone())
.await
.unwrap();
let result = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
.await
.unwrap()
.unwrap();
assert_eq!(result.state, JobState::Completed);
assert!(
!model.source_reads.lock().unwrap().is_empty(),
"provider saw saved source bodies"
);
fixture.host().complete_journal_job(&jobs, &result).unwrap();
assert_eq!(
jobs.get(&id).unwrap().input_disposition,
JobInputDisposition::Reclaimed
);
fs::rename(&denied, &source_path).unwrap();
jobs_processed += 1;
assert!(jobs_processed < 20, "bounded prefix drain must progress");
}
assert!(
jobs_processed > 2,
"entry count and long text both require continuation"
);
assert_eq!(seen.len(), expected.entries.len());
for entry in &expected.entries {
assert_eq!(
seen[&entry.order],
expected.recording_text(entry.order).unwrap().2
);
}
fixture.host().jobs().unwrap().shutdown().await.unwrap();
}
#[tokio::test]
async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful_records() {
let fixture = Fixture::new("original descriptor-only input");
let (adapter, _, modern) = fixture.adapter();
let jobs = fixture.host().jobs().unwrap();
fixture
.host()
.catalog
.store
.close_journal_generation(
&adapter.context,
&adapter.generation.id,
&adapter.generation.job,
)
.unwrap();
jobs.cancel(&modern.request.job_id).unwrap();
let input = JournalInput::decode(&modern.request.input).unwrap();
let origins: Vec<_> = input
.capture
.entries
.iter()
.map(|entry| {
let (descriptor, _, _) = input.capture.recording_text(entry.order).unwrap();
SourceOrigin {
pointer: source_descriptor(&fixture.context.session_id, descriptor).pointer,
origin: descriptor.origin.clone(),
}
})
.collect();
let mut old_grant = adapter.grant.clone();
old_grant.origins = origins;
old_grant.writer_binding_id = None;
let mut request = modern.request.clone();
request.job_id = format!("legacy-descriptor:{}", uuid::Uuid::now_v7());
request.input =
serde_json::json!({"selected_ranges":old_grant.ranges,"origins":old_grant.origins});
let old = jobs
.request_granted(request, serde_json::to_value(&old_grant).unwrap())
.unwrap();
assert!(
fixture.host().journal_grant(&old).is_err(),
"descriptor jobs must not execute as snapshots"
);
let generation = fixture
.host()
.catalog
.store
.prepare_journal_generation(
&adapter.context,
&old_grant.journal_id,
JobAttemptBinding {
job_id: old.request.job_id.clone(),
attempt_id: old.attempt.attempt_id.clone(),
},
old_grant.ranges.clone(),
)
.unwrap();
let context = JournalJobContext::validated(
&fixture.host().catalog.store,
&adapter.context,
&generation.id,
generation.job.clone(),
)
.unwrap();
let saved = fixture
.host()
.catalog
.store
.execute_journal(
&context,
JournalOperation {
request_id: "legacy-record-success-response-lost".into(),
expected_state: generation.initial_journal.state_token.clone(),
action: JournalAction::Create {
title: "already saved".into(),
body: "legacy successful operation".into(),
ongoing: "unfinished".into(),
state: JournalEntryState::Open,
source_end: None,
correction_refs: vec![],
},
covered_ranges: generation.selected_ranges.clone(),
},
)
.unwrap();
fixture.append(
"new-not-selected",
"LATEST SOURCE MUST WAIT UNTIL OLD INPUT FINISHES",
);
fixture.host().admit_journal(&fixture.context).unwrap();
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
assert_ne!(next, old.request.job_id);
let replacement = jobs.get(&next).unwrap();
let replacement_input = JournalInput::decode(&replacement.request.input).unwrap();
assert!(
!replacement
.request
.input
.to_string()
.contains("LATEST SOURCE MUST WAIT")
);
assert_eq!(replacement_input.selected_ranges, old_grant.ranges);
let retired = jobs.get(&old.request.job_id).unwrap();
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
assert_eq!(retired.input_digest, old.input_digest);
assert_eq!(retired.intent_fingerprint, old.intent_fingerprint);
assert!(retired.request.input.is_null());
assert!(
fixture
.host()
.catalog
.store
.execute_journal(
&context,
JournalOperation {
request_id: "legacy-delayed".into(),
expected_state: saved.state_token,
action: JournalAction::ConfirmUnchanged,
covered_ranges: old_grant.ranges,
}
)
.is_err()
);
jobs.start_with_model_client(&next, JournalScript::default())
.await
.unwrap();
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&next))
.await
.unwrap()
.unwrap();
assert_eq!(completed.state, JobState::Completed);
fixture
.host()
.complete_journal_job(&jobs, &completed)
.unwrap();
let entries = fixture
.host()
.catalog
.store
.list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32)
.unwrap();
assert_eq!(entries.items.len(), 1);
assert_eq!(entries.items[0].id, saved.entry_id.unwrap());
jobs.shutdown().await.unwrap();
}
#[test]
fn explicit_erase_retires_every_historical_body_across_job_history_pages() {
const SENTINEL: &str = "historical-input-erasure-sentinel-781";
let mut fixture = Fixture::new("ordinary original");
let (adapter, _, first) = fixture.adapter();
let jobs = fixture.host().jobs().unwrap();
for index in 0..105 {
let mut request = first.request.clone();
request.job_id = format!("historical-{index:03}");
request.purpose = "historical-other-domain-job".into();
request.serialization_key = None;
request.input = serde_json::json!({"body":SENTINEL, "index":index});
let saved = jobs
.request_granted(
request,
serde_json::json!({"session_id":fixture.context.session_id}),
)
.unwrap();
jobs.cancel(&saved.request.job_id).unwrap();
}
let state = fixture.host().catalog.state_root.clone();
let subject = fixture.connection.binding.subject_id.clone();
fixture.connection.close().unwrap();
StandaloneSubjects::open_existing(&state)
.unwrap()
.erase_session_completely(&subject, &fixture.context.session_id, "erase-all-history")
.unwrap();
let jobs = crate::jobs::StandaloneJobs::open(
&fixture.connection.jobs_path,
fixture._temp.path().into(),
)
.unwrap();
let mut after = None;
let mut count = 0;
loop {
let page = jobs
.list_session_snapshots(&fixture.context.session_id, after.as_deref(), 100)
.unwrap();
if page.is_empty() {
break;
}
for saved in &page {
assert_eq!(saved.input_disposition, JobInputDisposition::Erased);
assert!(saved.request.input.is_null());
assert!(saved.load_executable_request().is_err());
}
count += page.len();
after = page.last().map(|job| job.request.job_id.clone());
}
assert_eq!(count, 106);
assert!(
adapter.execute(JournalJobRequest::ReadState).is_err(),
"prepared local adapter is invalidated"
);
fixture.assert_job_storage_excludes(&jobs.get(&first.request.job_id).unwrap(), SENTINEL);
}
#[test]
fn saved_snapshot_missing_tampered_foreign_or_changed_initial_state_is_not_recaptured() {
let fixture = Fixture::new("saved immutable original");
fixture.host().admit_journal(&fixture.context).unwrap();
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
let saved = jobs.get(&id).unwrap();
for changed in [serde_json::Value::Null, serde_json::json!({"capture":{}})] {
let mut malformed = saved.clone();
malformed.request.input = changed;
assert!(fixture.host().journal_grant(&malformed).is_err());
}
let original = JournalInput::decode(&saved.request.input).unwrap();
for field in ["subject_id", "journal_id", "initial_state_token"] {
let mut changed = saved.request.clone();
changed.job_id = format!("changed-{field}");
changed.serialization_key = None;
changed.input[field] = serde_json::json!("foreign-or-altered");
// Even a separately reserved immutable Job may not adopt the original capability.
let changed = jobs
.request_granted(changed, saved.domain_grant.clone().unwrap())
.unwrap();
assert!(
grant(
fixture.host().clone(),
&changed,
Arc::new(TestFence(AtomicBool::new(true)))
)
.is_err()
);
}
let mut generation = fixture
.host()
.catalog
.store
.prepare_journal_generation(
&authority(fixture.host(), &fixture.context.session_id).unwrap(),
&original.journal_id,
JobAttemptBinding {
job_id: id,
attempt_id: saved.attempt.attempt_id.clone(),
},
original.selected_ranges.clone(),
)
.unwrap();
generation.initial_journal.state_token = "changed-after-snapshot".into();
assert!(original.validate_generation(&generation).is_err());
assert_eq!(
saved.request.input,
jobs.get(&saved.request.job_id).unwrap().request.input
);
}
#[tokio::test]
async fn append_between_admission_and_body_capture_does_not_publish_unaccepted_input() {
let fixture = Fixture::new("accepted oldest prefix");
fixture.host().admit_journal(&fixture.context).unwrap();
fixture.append("not-yet-admitted", "newly saved but not accepted");
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
let input = jobs.get(&id).unwrap().request.input;
assert!(input.to_string().contains("accepted oldest prefix"));
assert!(!input.to_string().contains("newly saved but not accepted"));
jobs.start_with_model_client(&id, JournalScript::default())
.await
.unwrap();
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
.await
.unwrap()
.unwrap();
assert_eq!(completed.state, JobState::Completed);
fixture
.host()
.complete_journal_job(&jobs, &completed)
.unwrap();
fixture.host().admit_journal(&fixture.context).unwrap();
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
let remaining = jobs.get(&next).unwrap().request.input;
assert!(
remaining
.to_string()
.contains("newly saved but not accepted")
);
assert!(!remaining.to_string().contains("accepted oldest prefix"));
jobs.shutdown().await.unwrap();
}
/// Generic Job preparation must not attest delivery. The saved body is supplied by the
/// runner before a bound Journal tool may execute; no model-provided digest is trusted.
#[tokio::test]
async fn actual_bound_tool_attests_delivery_but_prepared_blocked_job_does_not() {
let fixture = Fixture::new("body delivered through generic runner");
fixture.host().admit_journal(&fixture.context).unwrap();
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
let saved = jobs.get(&id).unwrap();
let model = JournalScript {
gate_first: true,
pause_after_record: true,
..Default::default()
};
jobs.start_with_model_client(&id, model.clone())
.await
.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
.await
.unwrap();
let host_context = authority(fixture.host(), &fixture.context.session_id).unwrap();
let journal_id = fixture
.host()
.catalog
.store
.journal_for_session(&host_context)
.unwrap()
.unwrap();
let generation = fixture
.host()
.catalog
.store
.active_journal_generation(&host_context, &journal_id)
.unwrap()
.unwrap();
let context = JournalJobContext::validated(
&fixture.host().catalog.store,
&host_context,
&generation.id,
generation.job.clone(),
)
.unwrap();
let undelivered = subjektiv::journal::service::execute_snapshot(
&fixture.host().catalog.store,
&context,
&saved.input_digest,
JournalJobRequest::ReadState,
)
.unwrap_err();
assert!(
undelivered.to_string().contains("delivery"),
"prepared grant must not satisfy delivery gate: {undelivered}"
);
let before = fixture
.host()
.catalog
.store
.read_journal(&host_context, &journal_id)
.unwrap();
assert_eq!((before.process_order, before.process_byte), (0, 0));
model.release.notify_one();
tokio::time::timeout(
std::time::Duration::from_secs(20),
model.record_saved.notified(),
)
.await
.unwrap();
assert!(
subjektiv::journal::service::execute_snapshot(
&fixture.host().catalog.store,
&context,
&saved.input_digest,
JournalJobRequest::ReadState
)
.is_ok(),
"actual bound tool execution attests exact saved input"
);
model.release.notify_one();
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
.await
.unwrap()
.unwrap();
assert_eq!(completed.state, JobState::Completed);
fixture
.host()
.complete_journal_job(&jobs, &completed)
.unwrap();
jobs.shutdown().await.unwrap();
}
#[derive(Clone, Default)]
struct BlockedJournalRequest {
entered: Arc<tokio::sync::Notify>,
dropped: Arc<tokio::sync::Notify>,
request_dropped: Arc<AtomicBool>,
}
struct OwnedBlockedJournalRequest {
// A local request copy, including the saved input, must be dropped before erasure
// can acquire the lease. This does not claim remote/provider bytes are retractable.
_request: agen::llm_client::Request,
model: BlockedJournalRequest,
}
impl Drop for OwnedBlockedJournalRequest {
fn drop(&mut self) {
self.model.request_dropped.store(true, Ordering::Release);
self.model.dropped.notify_one();
}
}
#[async_trait::async_trait]
impl agen::llm_client::LlmClient for BlockedJournalRequest {
async fn stream(
&self,
request: agen::llm_client::Request,
) -> Result<
std::pin::Pin<
Box<
dyn futures::Stream<
Item = Result<
agen::llm_client::event::Event,
agen::llm_client::error::ClientError,
>,
> + Send,
>,
>,
agen::llm_client::error::ClientError,
> {
let owned = OwnedBlockedJournalRequest {
_request: request,
model: self.clone(),
};
assert!(
owned
._request
.items
.iter()
.filter_map(agen::llm_client::types::Item::as_text)
.any(|text| text.contains("local blocked source body"))
);
self.entered.notify_one();
std::future::pending::<()>().await;
drop(owned);
unreachable!()
}
fn clone_boxed(&self) -> Box<dyn agen::llm_client::LlmClient> {
Box::new(self.clone())
}
}
/// Real generic Jobs and a retained public handle, with only the provider transport blocked.
/// Connection close must initiate cancellation and preserve the exclusive lease until local
/// request/history ownership is gone (or return an explicit incomplete barrier).
#[tokio::test]
async fn close_cancels_retained_local_job_before_erase_can_acquire_exclusive_lease() {
let mut fixture = Fixture::new("local blocked source body");
fixture.host().admit_journal(&fixture.context).unwrap();
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
let retained = jobs.clone();
let model = BlockedJournalRequest::default();
retained
.start_with_model_client(&id, model.clone())
.await
.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
.await
.unwrap();
let state = fixture.host().catalog.state_root.clone();
let subject = fixture.connection.binding.subject_id.clone();
let close_error = fixture.connection.close().unwrap_err();
assert!(matches!(
close_error,
SubjectError::JobShutdown(crate::jobs::JobServiceError::ShutdownIncomplete)
));
assert!(fixture.connection.lease.is_some());
assert!(fixture.host().jobs.lock().unwrap().is_some());
assert!(matches!(
retained.start(&id).await,
Err(crate::jobs::JobServiceError::Closed)
));
let locally_dropped_before_erase = model.request_dropped.load(Ordering::Acquire);
let early_erasure = StandaloneSubjects::open_existing(&state)
.unwrap()
.erase_session_completely(
&subject,
&fixture.context.session_id,
"erase-blocked-local-run",
);
let early_lease_was_retained = matches!(early_erasure, Err(SubjectError::Active(_)));
let teardown_from_close = model.request_dropped.load(Ordering::Acquire)
|| tokio::time::timeout(std::time::Duration::from_secs(5), model.dropped.notified())
.await
.is_ok();
// Always tear down before asserting the observed violation, so a failed regression
// cannot leak the blocked model future into another test.
let _ = retained.shutdown().await;
assert!(model.request_dropped.load(Ordering::Acquire));
fixture.connection.close().unwrap();
assert!(fixture.connection.lease.is_none());
assert!(fixture.host().jobs.lock().unwrap().is_none());
// Cancellation is not completion/erasure. Reopening retains the same unknown
// attempt's immutable body until the explicit erasure transaction retires it.
let reopened = crate::jobs::StandaloneJobs::open(
&fixture.connection.jobs_path,
fixture._temp.path().into(),
)
.unwrap();
let unknown = reopened.get(&id).unwrap();
assert_eq!(unknown.attempt.state, crate::jobs::JobAttemptState::Unknown);
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
assert!(
unknown
.request
.input
.to_string()
.contains("local blocked source body")
);
reopened.shutdown().await.unwrap();
StandaloneSubjects::open_existing(&state)
.unwrap()
.erase_session_completely(
&subject,
&fixture.context.session_id,
"erase-blocked-local-run",
)
.unwrap();
let erased_store = crate::jobs::StandaloneJobs::open(
&fixture.connection.jobs_path,
fixture._temp.path().into(),
)
.unwrap();
let erased = erased_store.get(&id).unwrap();
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
fixture.assert_job_storage_excludes(&erased, "local blocked source body");
erased_store.shutdown().await.unwrap();
assert!(
locally_dropped_before_erase || early_lease_was_retained,
"close released exclusive lease while a retained generic Job still owned local input/history"
);
assert!(
teardown_from_close,
"connection close did not initiate cancellation of the externally retained local Job"
);
assert!(
retained.get(&id).is_err(),
"retained handles are closed after teardown"
);
}
crates/standalone/tests/host.rs
+8 -4
View File
@@ -615,10 +615,14 @@ async fn headless_unix_client_uses_production_history_lease_and_local_authority(
use tokio::io::AsyncWriteExt;
let temp = tempfile::tempdir()?;
// Only the transport endpoint needs a short private pathname. Persistence remains the
// same production store exercised by the in-process Host tests above.
std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o700))?;
let endpoint_path = temp.path().join("operator.sock");
// Shorten only the owned endpoint's components, preserving the harness TMPDIR
// even when it is checkout-local. No process-global cwd or external /tmp fallback.
let socket_dir = tempfile::Builder::new()
.prefix("")
.rand_bytes(4)
.tempdir()?;
std::fs::set_permissions(socket_dir.path(), std::fs::Permissions::from_mode(0o700))?;
let endpoint_path = socket_dir.path().join("s");
let endpoint = standalone::UnixEndpoint::bind(&endpoint_path)?;
let state_dir = temp.path().join("state");
let launch = StandaloneLaunchConfig::new(
crates/standalone/tests/subjektiv.rs
+21 -31
View File
@@ -479,38 +479,28 @@ fn journal_response(request: &Request) -> Vec<LlmEvent> {
}),
);
}
for (index, range) in state["selected_ranges"]
.as_array()
.unwrap()
let input: Value = request
.items
.iter()
.enumerate()
{
let mut offset = range["bytes"]["start"].as_u64().unwrap();
let mut token = Value::Null;
loop {
let id = format!("journal-source-{index}-{offset}");
match result(&id) {
Some(page) => match page["value"]["next_byte"].as_u64() {
Some(next) => {
assert!(next > offset);
offset = next;
token = page["value"]["content_token"].clone();
}
None => break,
},
None => {
return tool(
&id,
"ReadJournalText",
json!({
"target":{"target":"source","pointer":range["source"]["pointer"]},
"offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token
}),
);
}
}
}
}
.filter_map(Item::as_text)
.find_map(|text| {
text.lines()
.find_map(|line| line.strip_prefix("input_json: "))
})
.map(|json| serde_json::from_str(json).unwrap())
.expect("saved direct Journal input");
let input = subjektiv::journal::recording::JournalInput::decode(&input).unwrap();
assert_eq!(
serde_json::to_value(&input.selected_ranges).unwrap(),
state["selected_ranges"]
);
assert!(
input
.capture
.entries
.iter()
.all(|entry| input.capture.recording_text(entry.order).is_ok())
);
let (name, action) = if state["current_open"].is_null() {
(
"CreateJournalEntry",