fix: release removed Subject bodies while preserving journal history

This commit is contained in:
2026-10-28 12:15:35 +09:00
parent 618880d38d
commit b0733e53b2
7 changed files with 1370 additions and 173 deletions
+23
View File
@@ -194,6 +194,29 @@ impl SubjektivStore {
}) })
} }
/// End a body after the Host has confirmed its Runtime removal. This is not a
/// completed Journal handoff: source coverage, positions and saved receipts stay
/// unchanged. Revocation and closing generations are atomic, so no retained Job
/// can write after deletion even while Backend Job cleanup is being retried.
pub fn release_removed_journal_body(&self, host: &JournalHostContext) -> JournalResult<()> {
self.database.try_transaction(|tx| {
super::store::check_session_host(self, tx, host)?;
let attr = host.attribution.as_ref().expect("checked Session authority");
let binding: Option<String> = tx.query_row(
"SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 ORDER BY rowid DESC LIMIT 1",
params![host.subject_id, attr.runtime_id, attr.worker_id], |r| r.get(0),
).optional()?;
let binding = binding.unwrap_or_else(|| crate::issued_id("journal-removed-body"));
tx.execute("INSERT OR IGNORE INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,0)",params![binding,host.subject_id,attr.runtime_id,attr.worker_id])?;
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
// Bind legacy attributed Sessions to an inactive lifetime, never to the
// successor. Retained unfinished history is not an active writer.
tx.execute("INSERT OR IGNORE INTO journal_session_bindings(session_id,binding_id) SELECT session_id,?4 FROM subject_session_attributions WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id,binding])?;
tx.execute("UPDATE journal_generations SET active=0 WHERE journal_id IN (SELECT j.id FROM journals j JOIN subject_session_attributions a ON a.session_id=j.source_session_id WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3)",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
Ok(())
})
}
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original /// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs. /// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
/// This transaction rejects retained writers, pending source bytes and active generations. /// This transaction rejects retained writers, pending source bytes and active generations.
+26
View File
@@ -651,6 +651,32 @@ impl SubjektivStore {
Ok(()) Ok(())
}) })
} }
/// Read the exact retained Job/attempt snapshot without granting writer
/// authority or creating a generation. Used for outcome recovery after removal.
pub fn retained_journal_generation(
&self,
host: &JournalHostContext,
id: &str,
binding: &JobAttemptBinding,
) -> JournalResult<Option<JournalGeneration>> {
self.database.try_with_connection(|tx| {
check_host(self, tx, host, false)?;
check_session_host(self, tx, host)?;
journal(tx, host, id, true)?;
let raw: Option<String> = tx.query_row(
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
params![id, binding.job_id, binding.attempt_id], |r| r.get(0),
).optional()?;
raw.map(|raw| {
let generation: JournalGeneration = serde_json::from_str(&raw)?;
if generation.journal_id != id || &generation.job != binding {
return Err(conflict("retained generation identity mismatch"));
}
Ok(generation)
}).transpose()
})
}
pub fn read_journal_generation( pub fn read_journal_generation(
&self, &self,
host: &JournalHostContext, host: &JournalHostContext,
+22 -2
View File
@@ -325,7 +325,7 @@ impl SqliteWorkspaceStore {
let now = Utc::now().to_rfc3339(); let now = Utc::now().to_rfc3339();
self.with_conn_mut(|conn| { self.with_conn_mut(|conn| {
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?; let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal,true)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? { let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
Some(v)=>v, Some(v)=>v,
None=>{ None=>{
@@ -371,7 +371,7 @@ impl SqliteWorkspaceStore {
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state)))); return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
} }
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));} if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?; let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID,plan.policy_id=="backend-subject-removal-archive-v1")?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{ if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?; mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
return Err(stale_error(&plan,"policy content changed")); return Err(stale_error(&plan,"policy content changed"));
@@ -1020,6 +1020,7 @@ fn load_removal_policy(
w: &str, w: &str,
worker: &RuntimeWorkerRef, worker: &RuntimeWorkerRef,
erased_journal: bool, erased_journal: bool,
preserve_subject_history: bool,
) -> crate::Result<Option<WorkerRetentionPolicy>> { ) -> crate::Result<Option<WorkerRetentionPolicy>> {
let Some(mut policy) = load_policy(c, w)? else { let Some(mut policy) = load_policy(c, w)? else {
return Ok(None); return Ok(None);
@@ -1044,6 +1045,25 @@ fn load_removal_policy(
policy.archive_retention = update.archive_retention; policy.archive_retention = update.archive_retention;
policy.diagnostics_disposition = update.diagnostics_disposition; policy.diagnostics_disposition = update.diagnostics_disposition;
policy.diagnostics_retention_seconds = None; policy.diagnostics_retention_seconds = None;
} else if preserve_subject_history && policy.session_disposition == SessionDisposition::Purge && c.query_row(
"SELECT EXISTS(SELECT 1 FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND substr(singleton_key,1,10)='subjektiv:')",
params![w, worker.runtime_id, worker.worker_id],
|r| r.get::<_, bool>(0),
)? {
// Removing a Subject body releases occupancy, not its original history.
// A configured Purge must not silently erase an unfinished Journal. Explicit
// Session erasure uses the separate, proof-bound branch above.
let update = WorkerRetentionPolicyUpdate {
policy_id: "backend-subject-removal-archive-v1".into(),
session_disposition: SessionDisposition::Archive,
metadata_disposition: policy.metadata_disposition,
archive_retention: policy.archive_retention,
diagnostics_disposition: policy.diagnostics_disposition,
diagnostics_retention_seconds: policy.diagnostics_retention_seconds,
};
policy.policy_id = update.policy_id.clone();
policy.policy_digest = worker_retention_policy_digest(&update);
policy.session_disposition = update.session_disposition;
} }
Ok(Some(policy)) Ok(Some(policy))
} }
+34 -3
View File
@@ -2327,7 +2327,9 @@ impl WorkerRemovalService {
)); ));
} }
}; };
match self.worker_projection.publish_ordered(|| { let commit_result =
match subjektiv_journal::finish_worker_removal(self, target, &prepared, &result, || {
self.worker_projection.publish_ordered(|| {
self.store self.store
.commit_worker_removal( .commit_worker_removal(
&self.workspace_id, &self.workspace_id,
@@ -2336,7 +2338,18 @@ impl WorkerRemovalService {
&result, &result,
) )
.map(|commit| ((), commit.catalog)) .map(|commit| ((), commit.catalog))
})
}) { }) {
Ok(result) => result,
Err(error) => {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"subject_removal_completion_unconfirmed",
&error.to_string(),
));
}
};
match commit_result {
Ok(()) => { Ok(()) => {
subjektiv_journal::after_worker_removal(self, &result); subjektiv_journal::after_worker_removal(self, &result);
Ok(worker_remove_success_response(target)) Ok(worker_remove_success_response(target))
@@ -2722,7 +2735,13 @@ impl WorkerRemovalService {
)); ));
} }
}; };
match self.worker_projection.publish_ordered(|| { let commit_result = match subjektiv_journal::finish_worker_removal(
self,
target,
&prepared,
&retention_result,
|| {
self.worker_projection.publish_ordered(|| {
self.store self.store
.commit_worker_removal( .commit_worker_removal(
&self.workspace_id, &self.workspace_id,
@@ -2731,7 +2750,19 @@ impl WorkerRemovalService {
&retention_result, &retention_result,
) )
.map(|commit| ((), commit.catalog)) .map(|commit| ((), commit.catalog))
}) { })
},
) {
Ok(result) => result,
Err(error) => {
return Ok(worker_remove_error_response(
StatusCode::CONFLICT,
"subject_removal_completion_unconfirmed",
&error.to_string(),
));
}
};
match commit_result {
Ok(()) => {} Ok(()) => {}
Err(error) => { Err(error) => {
let _ = self.store.fail_worker_removal( let _ = self.store.fail_worker_removal(
@@ -92,6 +92,20 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) -
}; };
let store = store(api)?; let store = store(api)?;
let host = host(api, &store, grant)?; let host = host(api, &store, grant)?;
let attr = store
.session_attribution(&grant.session_id)
.map_err(|e| Error::Store(e.to_string()))?
.ok_or_else(|| {
Error::WorkspacePermissionDenied("Journal Session attribution missing".into())
})?;
if store
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
.map_err(domain)?
{
return Err(Error::WorkspacePermissionDenied(
"released Journal body cannot dispatch another attempt".into(),
));
}
let input = saved_input(&reservation.job.request, grant)?; let input = saved_input(&reservation.job.request, grant)?;
let g = store let g = store
.prepare_journal_input( .prepare_journal_input(
@@ -584,6 +598,18 @@ fn dispatch(
.lock() .lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?; .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
let store = store(api)?; let store = store(api)?;
if store
.journal_worker_was_released(
&attribution.subject_id,
&attribution.runtime_id,
&attribution.worker_id,
)
.map_err(domain)?
{
// Deletion can retain an unfinished prefix. It must not mint attempts or
// transfer the removed body's write capability to its successor.
return Ok(());
}
let host = let host =
JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?; JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?;
if let Some(g) = store if let Some(g) = store
@@ -1740,8 +1766,8 @@ fn finish_erased_release(
.release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation) .release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation)
} }
/// Normal deletion cannot substitute for Subject release, even for a stopped or disabled body. /// Ordinary removal owns writer revocation and occupancy release. Before execution
/// Explicit erasure remains a separate monotonic source/diagnostics destruction path. /// only preserve/erase source requirements are checked; an active writer is not a blocker.
pub(super) fn prepare_worker_removal( pub(super) fn prepare_worker_removal(
service: &WorkerRemovalService, service: &WorkerRemovalService,
runtime: &RuntimeRegistry, runtime: &RuntimeRegistry,
@@ -1762,7 +1788,6 @@ pub(super) fn prepare_worker_removal(
&service.journal_registration, &service.journal_registration,
) )
.map_err(|e| Error::Store(e.to_string()))?; .map_err(|e| Error::Store(e.to_string()))?;
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
let source = runtime_api::SessionPublicSource::Retained { let source = runtime_api::SessionPublicSource::Retained {
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id) worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?, .ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
@@ -1779,25 +1804,9 @@ pub(super) fn prepare_worker_removal(
finish_erased_release(&api, &attr)?; finish_erased_release(&api, &attr)?;
return Ok(Some(proof)); return Ok(Some(proof));
} }
let Some(binding_id) = host.writer_binding_id() else { // Do not require a live Controller, complete Journal, or prior release here.
return Err(Error::RepositoryConflict( // Runtime removal is the physical fence; finish_worker_removal revokes the
"Subject writer evidence is missing; ordinary deletion cannot bypass release".into(), // exact old body only after its successful, matching retention receipt.
));
};
let writer = JournalWriterBinding {
subject_id: attr.subject_id.clone(),
runtime_id: attr.runtime_id.clone(),
worker_id: attr.worker_id.clone(),
binding_id: binding_id.into(),
};
if store.journal_writer_active(&writer).map_err(domain)? {
return Err(Error::RepositoryConflict("Subject remains occupied: restore to actual Idle, finish Journal recording, then explicitly release before ordinary deletion".into()));
}
if store.journal_retention_required(&host).map_err(domain)? {
return Err(Error::RepositoryConflict(
"Released Subject has unresolved Journal material; retain occupancy and source".into(),
));
}
Ok(None) Ok(None)
} }
@@ -1886,20 +1895,6 @@ pub(super) fn validate_removal_retention(
// its retained permanent fence, including recovery after the aggregate is gone. // its retained permanent fence, including recovery after the aggregate is gone.
return Ok(()); return Ok(());
} }
let binding_id = host.writer_binding_id().ok_or_else(|| {
Error::RepositoryConflict("Subject writer evidence missing during removal recovery".into())
})?;
let binding = JournalWriterBinding {
subject_id: attr.subject_id.clone(),
runtime_id: attr.runtime_id.clone(),
worker_id: attr.worker_id.clone(),
binding_id: binding_id.into(),
};
if store.journal_writer_active(&binding).map_err(domain)? {
return Err(Error::RepositoryConflict(
"Prepared removal cannot bypass explicit Subject release".into(),
));
}
let source = runtime_api::SessionPublicSource::Retained { let source = runtime_api::SessionPublicSource::Retained {
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id) worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?, .ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
@@ -1907,19 +1902,42 @@ pub(super) fn validate_removal_retention(
let runtime = service.runtime.upgrade().ok_or_else(|| { let runtime = service.runtime.upgrade().ok_or_else(|| {
Error::Store("Runtime unavailable for source retention confirmation".into()) Error::Store("Runtime unavailable for source retention confirmation".into())
})?; })?;
if store.journal_retention_required(&host).map_err(domain)? { let retention_required = store.journal_retention_required(&host).map_err(domain)?;
if prepared.runtime_request.session_disposition
== worker_runtime::retention::SessionDisposition::Purge
&& prepared.runtime_request.archive_id.is_none()
&& !retention_required
&& store
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
.map_err(domain)?
{
// Compatibility for an already issued, history-complete ordinary Purge
// plan. New Subject plans always Archive. Keep the old fingerprint and
// disposition; never turn this into an erasure or allow an active writer.
// A missing Runtime Worker is NOT success: execute_worker_retention still
// has to return the exact persisted receipt before anything can commit.
// A pending receipt may outlive the physical aggregate while Runtime's
// in-memory catalog still contains the Worker. Pin release is best effort
// in this historical branch; exact Runtime retention replay still enforces
// pins if the source exists, and requires a receipt if it is already gone.
if let Err(error) = retain_source(&runtime, &service.workspace_id, &attr, source, true) {
tracing::debug!(%error, "Historical Purge source unavailable; reconcile exact Runtime receipt");
}
return Ok(());
}
if prepared.runtime_request.session_disposition if prepared.runtime_request.session_disposition
!= worker_runtime::retention::SessionDisposition::Archive != worker_runtime::retention::SessionDisposition::Archive
|| prepared.runtime_request.archive_id.is_none() || prepared.runtime_request.archive_id.is_none()
{ {
return Err(Error::RepositoryConflict( return Err(Error::RepositoryConflict(
"unreflected Journal requires a retained original Session archive".into(), "Subject removal retains its original Session; explicit erasure is required for Purge"
.into(),
)); ));
} }
if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, false) let release = !retention_required;
{ if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, release) {
// Recovery may already have transferred the physical source before its Backend // A lost reply may follow the physical transfer. Reconcile only the exact
// catalog commit; authenticate the exact reserved archive, never another Session. // reserved archive and Session, whether or not an admitted remainder exists.
let archive = runtime_api::SessionPublicSource::Archived { let archive = runtime_api::SessionPublicSource::Archived {
archive_id: prepared archive_id: prepared
.runtime_request .runtime_request
@@ -1927,17 +1945,97 @@ pub(super) fn validate_removal_retention(
.clone() .clone()
.expect("checked"), .expect("checked"),
}; };
retain_source(&runtime, &service.workspace_id, &attr, archive, false) retain_source(&runtime, &service.workspace_id, &attr, archive, release)
.map_err(|_| original)?; .map_err(|_| original)?;
} }
} else {
// This stopped/removal-fenced Session has no admitted remainder; purge need not
// force a generation. A failed release conservatively blocks ordinary deletion.
retain_source(&runtime, &service.workspace_id, &attr, source, true)?;
}
Ok(()) Ok(())
} }
/// A successful Runtime receipt, not catalog liveness, authorizes ending the old
/// writer. The ordinary removal commit releases occupancy atomically with catalog
/// deletion. Until then a crash leaves the singleton occupied and the same receipt
/// can safely repeat this step; no Controller restore or Journal generation is needed.
pub(super) fn finish_worker_removal<T>(
service: &WorkerRemovalService,
target: &RuntimeWorkerRef,
prepared: &crate::retention::PreparedWorkerRemoval,
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
commit: impl FnOnce() -> T,
) -> Result<T> {
let Some(attr) = removal_attribution(service, target)? else {
return Ok(commit());
};
let request = &prepared.runtime_request;
if prepared.plan.worker != *target
|| request.workspace_id != service.workspace_id
|| request.source_runtime_id != target.runtime_id
|| request.worker_id.to_string() != target.worker_id
|| !result.source_removed
|| result.worker_id != request.worker_id
|| result.operation_id != request.operation_id
|| result.input_fingerprint != request.input_fingerprint
|| result.session_disposition != request.session_disposition
|| result.diagnostics_disposition != request.diagnostics_disposition
|| result.diagnostics_retained
!= (request.diagnostics_disposition
== worker_runtime::retention::DiagnosticsDisposition::Retain)
{
return Err(Error::RepositoryConflict(
"Subject removal receipt does not match its plan".into(),
));
}
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
let archive = result.archive.as_ref().ok_or_else(|| {
Error::RepositoryConflict("Subject removal archive receipt is missing".into())
})?;
if Some(&archive.archive_id) != request.archive_id.as_ref()
|| archive.workspace_id != request.workspace_id
|| archive.source_runtime_id != request.source_runtime_id
|| archive.source_worker_id != request.worker_id
|| archive.source_session_id != attr.session_id
|| archive.operation_id != request.operation_id
|| archive.input_fingerprint != request.input_fingerprint
|| archive.policy_id != request.policy_id
{
return Err(Error::RepositoryConflict(
"Subject removal archive identity mismatch".into(),
));
}
}
let api = service.journal_dispatch_api.upgrade().ok_or_else(|| {
Error::Store("Workspace API unavailable for Subject removal completion".into())
})?;
let gate = lifecycle_lock(&api, &attr.subject_id)?;
let _guard = gate
.lock()
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
let store = store(&api)?;
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
if store.journal_session_erased(&host).map_err(domain)? {
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
// Erasure may win after Runtime archive transfer. Do not publish that
// archive or skip writer revocation: the same removal retry must take
// erased_archive_removal_proof and its physical-erasure commit path.
return Err(Error::RepositoryConflict(
"Session erased during removal; reconcile the same removal operation".into(),
));
}
return Ok(commit());
}
store.release_removed_journal_body(&host).map_err(domain)?;
for session in store
.subject_session_attributions(&attr.subject_id)
.map_err(|e| Error::Store(e.to_string()))?
.into_iter()
.filter(|a| a.runtime_id == target.runtime_id && a.worker_id == target.worker_id)
{
reconcile_session_jobs(&api, &session, false)?;
}
// Keep erasure excluded through the catalog commit: otherwise a tombstone
// could win between revocation and publication of the retained archive.
Ok(commit())
}
#[derive(Deserialize)] #[derive(Deserialize)]
#[serde(deny_unknown_fields)] #[serde(deny_unknown_fields)]
pub(super) struct Empty {} pub(super) struct Empty {}
@@ -2089,7 +2187,28 @@ pub(super) fn reconcile_unknown(
.as_ref() .as_ref()
.ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?; .ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?;
let store = store(api)?; let store = store(api)?;
let host = host(api, &store, grant)?; let attr = store
.session_attribution(&grant.session_id)
.map_err(|e| Error::Store(e.to_string()))?
.filter(|a| a.subject_id == grant.subject_id)
.ok_or_else(|| Error::WorkspacePermissionDenied("Journal attribution mismatch".into()))?;
let released = store
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
.map_err(domain)?;
let host = if released {
// Recovery reads immutable evidence, never renews a legacy writer grant.
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
if !grant.binding_id.is_empty()
&& host.writer_binding_id() != Some(grant.binding_id.as_str())
{
return Err(Error::WorkspacePermissionDenied(
"Journal recovery binding mismatch".into(),
));
}
host
} else {
host(api, &store, grant)?
};
let erased = store.journal_session_erased(&host).map_err(domain)?; let erased = store.journal_session_erased(&host).map_err(domain)?;
if !erased { if !erased {
job.load_executable_request()?; job.load_executable_request()?;
@@ -2097,8 +2216,12 @@ pub(super) fn reconcile_unknown(
job_id: job.request.job_id.clone(), job_id: job.request.job_id.clone(),
attempt_id: attempt.attempt_id.clone(), attempt_id: attempt.attempt_id.clone(),
}; };
let g = if legacy_descriptor(&job.request) { let generation = if released {
// Only reconcile already-saved legacy receipts; never redispatch descriptors. store
.retained_journal_generation(&host, &grant.journal_id, &binding)
.map_err(domain)?
} else if legacy_descriptor(&job.request) {
Some(
store store
.prepare_journal_generation( .prepare_journal_generation(
&host, &host,
@@ -2106,15 +2229,28 @@ pub(super) fn reconcile_unknown(
binding, binding,
grant.selected_ranges.clone(), grant.selected_ranges.clone(),
) )
.map_err(domain)? .map_err(domain)?,
)
} else { } else {
let input = saved_input(&job.request, grant)?; // Preserve active-body recovery: its saved intent can still establish
let generation = store // the original generation fence. A released body must only read it.
.prepare_journal_input(&host, &input, binding) Some(
.map_err(domain)?; store
input.validate_generation(&generation).map_err(domain)?; .prepare_journal_input(&host, &saved_input(&job.request, grant)?, binding)
generation .map_err(domain)?,
)
}; };
if let Some(g) = generation {
if g.selected_ranges != grant.selected_ranges {
return Err(Error::WorkspacePermissionDenied(
"Journal recovery selection mismatch".into(),
));
}
if !legacy_descriptor(&job.request) {
saved_input(&job.request, grant)?
.validate_generation(&g)
.map_err(domain)?;
}
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone()) let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
.map_err(domain)?; .map_err(domain)?;
match journal::service::finish(&store, &ctx) { match journal::service::finish(&store, &ctx) {
@@ -2144,9 +2280,14 @@ pub(super) fn reconcile_unknown(
replayed: false, replayed: false,
}, },
); );
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) { if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed)
{
if !released {
drain_after_completion(api, &job.request); drain_after_completion(api, &job.request);
if job.input_disposition == crate::backend_job::JobInputDisposition::Available { }
if job.input_disposition
== crate::backend_job::JobInputDisposition::Available
{
api.store.reclaim_backend_job_input( api.store.reclaim_backend_job_input(
api.workspace_id(), api.workspace_id(),
&job.request.job_id, &job.request.job_id,
@@ -2163,6 +2304,7 @@ pub(super) fn reconcile_unknown(
Err(error) => return Err(domain(error)), Err(error) => return Err(domain(error)),
} }
} }
}
if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) { if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) {
return Ok(()); return Ok(());
} }
@@ -2179,11 +2321,17 @@ pub(super) fn reconcile_unknown(
false false
}; };
if !confirmed_purge { if !confirmed_purge {
if released {
// Source deletion already fenced this generation. An unbound Unknown
// still lacks execution-cleanup proof: preserve Unknown, but it no
// longer owns the Subject or blocks removal of the original body.
return Ok(());
}
return Err(Error::RepositoryConflict( return Err(Error::RepositoryConflict(
"Unknown Journal attempt has no committed stop/Purge fence; not retried".into(), "Unknown Journal attempt has no committed stop/Purge fence; not retried".into(),
)); ));
} }
if !erased { if !erased && !released {
close(api, &job.request, attempt)?; close(api, &job.request, attempt)?;
} }
api.store.reconcile_backend_journal_attempt( api.store.reconcile_backend_journal_attempt(
@@ -2192,7 +2340,7 @@ pub(super) fn reconcile_unknown(
None, None,
&Utc::now().to_rfc3339(), &Utc::now().to_rfc3339(),
)?; )?;
if !erased { if !erased && !released {
// Resume the same saved body input only after canonical execution/domain fences. // Resume the same saved body input only after canonical execution/domain fences.
// dispatch selects its retained intent before it can capture any newer source. // dispatch selects its retained intent before it can capture any newer source.
let attr = store let attr = store
@@ -2209,6 +2357,10 @@ fn reconcile_session_jobs(
attr: &crate::subjektiv::SubjectSessionAttribution, attr: &crate::subjektiv::SubjectSessionAttribution,
erased: bool, erased: bool,
) -> Result<()> { ) -> Result<()> {
let released = !erased
&& store(api)?
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
.map_err(domain)?;
let mut after: Option<String> = None; let mut after: Option<String> = None;
loop { loop {
let jobs = api.store.list_backend_journal_jobs( let jobs = api.store.list_backend_journal_jobs(
@@ -2223,6 +2375,29 @@ fn reconcile_session_jobs(
.store .store
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?; .ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
if released {
// Writer revocation has already fenced all Journal mutations. End
// pending/Unknown attempts without claiming their input was reflected;
// saved input, operations and original source remain retained.
if !attempt.state.terminal() {
attempt = api.store.finish_backend_job_attempt(
api.workspace_id(),
&job.request.job_id,
&id,
BackendJobAttemptState::Failed,
"journal_body_released",
"Source Worker was removed; unfinished Journal history retained",
&Utc::now().to_rfc3339(),
)?;
}
api.schedule_backend_job_worker_cleanup(&attempt);
if attempt.state == BackendJobAttemptState::Unknown {
// A saved checkpoint may settle success; otherwise retain Unknown
// until canonical Job-worker cleanup has supplied its Purge fence.
reconcile_unknown(api, job, &attempt)?;
}
continue;
}
if !erased if !erased
&& legacy_descriptor(&job.request) && legacy_descriptor(&job.request)
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available && job.input_disposition == crate::backend_job::JobInputDisposition::Available
File diff suppressed because it is too large Load Diff
+3 -3
View File
@@ -3112,8 +3112,8 @@ impl SqliteWorkspaceStore {
}, },
) )
.optional()?; .optional()?;
// Subject occupancy survives removal until an explicit release // An unconfirmed legacy removal cannot release Subject occupancy.
// proof. It must not resurrect the removed creation reservation: // Completed removal or explicit release does; never resurrect a reservation:
// Runtime callbacks require a live reservation or catalog member. // Runtime callbacks require a live reservation or catalog member.
if stored.as_ref().is_some_and(|row| row.6 == "removed") { if stored.as_ref().is_some_and(|row| row.6 == "removed") {
return Err(Error::RepositoryConflict(format!( return Err(Error::RepositoryConflict(format!(
@@ -16223,7 +16223,7 @@ fn current_worker_singleton_owner(
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \ CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \ AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \ AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
THEN 'created' ELSE reservation.state END \ AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.state='succeeded') THEN 'created' ELSE reservation.state END \
FROM worker_singleton_owners owner \ FROM worker_singleton_owners owner \
LEFT JOIN worker_create_reservations reservation \ LEFT JOIN worker_create_reservations reservation \
ON reservation.workspace_id = owner.workspace_id \ ON reservation.workspace_id = owner.workspace_id \