fix: release removed Subject bodies while preserving journal history
This commit is contained in:
@@ -194,6 +194,29 @@ impl SubjektivStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// End a body after the Host has confirmed its Runtime removal. This is not a
|
||||||
|
/// completed Journal handoff: source coverage, positions and saved receipts stay
|
||||||
|
/// unchanged. Revocation and closing generations are atomic, so no retained Job
|
||||||
|
/// can write after deletion even while Backend Job cleanup is being retried.
|
||||||
|
pub fn release_removed_journal_body(&self, host: &JournalHostContext) -> JournalResult<()> {
|
||||||
|
self.database.try_transaction(|tx| {
|
||||||
|
super::store::check_session_host(self, tx, host)?;
|
||||||
|
let attr = host.attribution.as_ref().expect("checked Session authority");
|
||||||
|
let binding: Option<String> = tx.query_row(
|
||||||
|
"SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 ORDER BY rowid DESC LIMIT 1",
|
||||||
|
params![host.subject_id, attr.runtime_id, attr.worker_id], |r| r.get(0),
|
||||||
|
).optional()?;
|
||||||
|
let binding = binding.unwrap_or_else(|| crate::issued_id("journal-removed-body"));
|
||||||
|
tx.execute("INSERT OR IGNORE INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,0)",params![binding,host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
// Bind legacy attributed Sessions to an inactive lifetime, never to the
|
||||||
|
// successor. Retained unfinished history is not an active writer.
|
||||||
|
tx.execute("INSERT OR IGNORE INTO journal_session_bindings(session_id,binding_id) SELECT session_id,?4 FROM subject_session_attributions WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id,binding])?;
|
||||||
|
tx.execute("UPDATE journal_generations SET active=0 WHERE journal_id IN (SELECT j.id FROM journals j JOIN subject_session_attributions a ON a.session_id=j.source_session_id WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3)",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
||||||
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
||||||
/// This transaction rejects retained writers, pending source bytes and active generations.
|
/// This transaction rejects retained writers, pending source bytes and active generations.
|
||||||
|
|||||||
@@ -651,6 +651,32 @@ impl SubjektivStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
/// Read the exact retained Job/attempt snapshot without granting writer
|
||||||
|
/// authority or creating a generation. Used for outcome recovery after removal.
|
||||||
|
pub fn retained_journal_generation(
|
||||||
|
&self,
|
||||||
|
host: &JournalHostContext,
|
||||||
|
id: &str,
|
||||||
|
binding: &JobAttemptBinding,
|
||||||
|
) -> JournalResult<Option<JournalGeneration>> {
|
||||||
|
self.database.try_with_connection(|tx| {
|
||||||
|
check_host(self, tx, host, false)?;
|
||||||
|
check_session_host(self, tx, host)?;
|
||||||
|
journal(tx, host, id, true)?;
|
||||||
|
let raw: Option<String> = tx.query_row(
|
||||||
|
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
|
||||||
|
params![id, binding.job_id, binding.attempt_id], |r| r.get(0),
|
||||||
|
).optional()?;
|
||||||
|
raw.map(|raw| {
|
||||||
|
let generation: JournalGeneration = serde_json::from_str(&raw)?;
|
||||||
|
if generation.journal_id != id || &generation.job != binding {
|
||||||
|
return Err(conflict("retained generation identity mismatch"));
|
||||||
|
}
|
||||||
|
Ok(generation)
|
||||||
|
}).transpose()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn read_journal_generation(
|
pub fn read_journal_generation(
|
||||||
&self,
|
&self,
|
||||||
host: &JournalHostContext,
|
host: &JournalHostContext,
|
||||||
|
|||||||
@@ -325,7 +325,7 @@ impl SqliteWorkspaceStore {
|
|||||||
let now = Utc::now().to_rfc3339();
|
let now = Utc::now().to_rfc3339();
|
||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
|
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal,true)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
|
||||||
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
|
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
|
||||||
Some(v)=>v,
|
Some(v)=>v,
|
||||||
None=>{
|
None=>{
|
||||||
@@ -371,7 +371,7 @@ impl SqliteWorkspaceStore {
|
|||||||
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
|
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
|
||||||
}
|
}
|
||||||
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
|
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
|
||||||
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
|
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID,plan.policy_id=="backend-subject-removal-archive-v1")?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
|
||||||
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
|
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
|
||||||
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
|
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
|
||||||
return Err(stale_error(&plan,"policy content changed"));
|
return Err(stale_error(&plan,"policy content changed"));
|
||||||
@@ -1020,6 +1020,7 @@ fn load_removal_policy(
|
|||||||
w: &str,
|
w: &str,
|
||||||
worker: &RuntimeWorkerRef,
|
worker: &RuntimeWorkerRef,
|
||||||
erased_journal: bool,
|
erased_journal: bool,
|
||||||
|
preserve_subject_history: bool,
|
||||||
) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
||||||
let Some(mut policy) = load_policy(c, w)? else {
|
let Some(mut policy) = load_policy(c, w)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -1044,6 +1045,25 @@ fn load_removal_policy(
|
|||||||
policy.archive_retention = update.archive_retention;
|
policy.archive_retention = update.archive_retention;
|
||||||
policy.diagnostics_disposition = update.diagnostics_disposition;
|
policy.diagnostics_disposition = update.diagnostics_disposition;
|
||||||
policy.diagnostics_retention_seconds = None;
|
policy.diagnostics_retention_seconds = None;
|
||||||
|
} else if preserve_subject_history && policy.session_disposition == SessionDisposition::Purge && c.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND substr(singleton_key,1,10)='subjektiv:')",
|
||||||
|
params![w, worker.runtime_id, worker.worker_id],
|
||||||
|
|r| r.get::<_, bool>(0),
|
||||||
|
)? {
|
||||||
|
// Removing a Subject body releases occupancy, not its original history.
|
||||||
|
// A configured Purge must not silently erase an unfinished Journal. Explicit
|
||||||
|
// Session erasure uses the separate, proof-bound branch above.
|
||||||
|
let update = WorkerRetentionPolicyUpdate {
|
||||||
|
policy_id: "backend-subject-removal-archive-v1".into(),
|
||||||
|
session_disposition: SessionDisposition::Archive,
|
||||||
|
metadata_disposition: policy.metadata_disposition,
|
||||||
|
archive_retention: policy.archive_retention,
|
||||||
|
diagnostics_disposition: policy.diagnostics_disposition,
|
||||||
|
diagnostics_retention_seconds: policy.diagnostics_retention_seconds,
|
||||||
|
};
|
||||||
|
policy.policy_id = update.policy_id.clone();
|
||||||
|
policy.policy_digest = worker_retention_policy_digest(&update);
|
||||||
|
policy.session_disposition = update.session_disposition;
|
||||||
}
|
}
|
||||||
Ok(Some(policy))
|
Ok(Some(policy))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2327,7 +2327,9 @@ impl WorkerRemovalService {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
match self.worker_projection.publish_ordered(|| {
|
let commit_result =
|
||||||
|
match subjektiv_journal::finish_worker_removal(self, target, &prepared, &result, || {
|
||||||
|
self.worker_projection.publish_ordered(|| {
|
||||||
self.store
|
self.store
|
||||||
.commit_worker_removal(
|
.commit_worker_removal(
|
||||||
&self.workspace_id,
|
&self.workspace_id,
|
||||||
@@ -2336,7 +2338,18 @@ impl WorkerRemovalService {
|
|||||||
&result,
|
&result,
|
||||||
)
|
)
|
||||||
.map(|commit| ((), commit.catalog))
|
.map(|commit| ((), commit.catalog))
|
||||||
|
})
|
||||||
}) {
|
}) {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(error) => {
|
||||||
|
return Ok(worker_remove_error_response(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"subject_removal_completion_unconfirmed",
|
||||||
|
&error.to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match commit_result {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
subjektiv_journal::after_worker_removal(self, &result);
|
subjektiv_journal::after_worker_removal(self, &result);
|
||||||
Ok(worker_remove_success_response(target))
|
Ok(worker_remove_success_response(target))
|
||||||
@@ -2722,7 +2735,13 @@ impl WorkerRemovalService {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
match self.worker_projection.publish_ordered(|| {
|
let commit_result = match subjektiv_journal::finish_worker_removal(
|
||||||
|
self,
|
||||||
|
target,
|
||||||
|
&prepared,
|
||||||
|
&retention_result,
|
||||||
|
|| {
|
||||||
|
self.worker_projection.publish_ordered(|| {
|
||||||
self.store
|
self.store
|
||||||
.commit_worker_removal(
|
.commit_worker_removal(
|
||||||
&self.workspace_id,
|
&self.workspace_id,
|
||||||
@@ -2731,7 +2750,19 @@ impl WorkerRemovalService {
|
|||||||
&retention_result,
|
&retention_result,
|
||||||
)
|
)
|
||||||
.map(|commit| ((), commit.catalog))
|
.map(|commit| ((), commit.catalog))
|
||||||
}) {
|
})
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(error) => {
|
||||||
|
return Ok(worker_remove_error_response(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"subject_removal_completion_unconfirmed",
|
||||||
|
&error.to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match commit_result {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
let _ = self.store.fail_worker_removal(
|
let _ = self.store.fail_worker_removal(
|
||||||
|
|||||||
@@ -92,6 +92,20 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) -
|
|||||||
};
|
};
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
let host = host(api, &store, grant)?;
|
let host = host(api, &store, grant)?;
|
||||||
|
let attr = store
|
||||||
|
.session_attribution(&grant.session_id)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.ok_or_else(|| {
|
||||||
|
Error::WorkspacePermissionDenied("Journal Session attribution missing".into())
|
||||||
|
})?;
|
||||||
|
if store
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?
|
||||||
|
{
|
||||||
|
return Err(Error::WorkspacePermissionDenied(
|
||||||
|
"released Journal body cannot dispatch another attempt".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let input = saved_input(&reservation.job.request, grant)?;
|
let input = saved_input(&reservation.job.request, grant)?;
|
||||||
let g = store
|
let g = store
|
||||||
.prepare_journal_input(
|
.prepare_journal_input(
|
||||||
@@ -584,6 +598,18 @@ fn dispatch(
|
|||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
|
if store
|
||||||
|
.journal_worker_was_released(
|
||||||
|
&attribution.subject_id,
|
||||||
|
&attribution.runtime_id,
|
||||||
|
&attribution.worker_id,
|
||||||
|
)
|
||||||
|
.map_err(domain)?
|
||||||
|
{
|
||||||
|
// Deletion can retain an unfinished prefix. It must not mint attempts or
|
||||||
|
// transfer the removed body's write capability to its successor.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let host =
|
let host =
|
||||||
JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?;
|
JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?;
|
||||||
if let Some(g) = store
|
if let Some(g) = store
|
||||||
@@ -1740,8 +1766,8 @@ fn finish_erased_release(
|
|||||||
.release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation)
|
.release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Normal deletion cannot substitute for Subject release, even for a stopped or disabled body.
|
/// Ordinary removal owns writer revocation and occupancy release. Before execution
|
||||||
/// Explicit erasure remains a separate monotonic source/diagnostics destruction path.
|
/// only preserve/erase source requirements are checked; an active writer is not a blocker.
|
||||||
pub(super) fn prepare_worker_removal(
|
pub(super) fn prepare_worker_removal(
|
||||||
service: &WorkerRemovalService,
|
service: &WorkerRemovalService,
|
||||||
runtime: &RuntimeRegistry,
|
runtime: &RuntimeRegistry,
|
||||||
@@ -1762,7 +1788,6 @@ pub(super) fn prepare_worker_removal(
|
|||||||
&service.journal_registration,
|
&service.journal_registration,
|
||||||
)
|
)
|
||||||
.map_err(|e| Error::Store(e.to_string()))?;
|
.map_err(|e| Error::Store(e.to_string()))?;
|
||||||
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
|
||||||
let source = runtime_api::SessionPublicSource::Retained {
|
let source = runtime_api::SessionPublicSource::Retained {
|
||||||
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
||||||
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
||||||
@@ -1779,25 +1804,9 @@ pub(super) fn prepare_worker_removal(
|
|||||||
finish_erased_release(&api, &attr)?;
|
finish_erased_release(&api, &attr)?;
|
||||||
return Ok(Some(proof));
|
return Ok(Some(proof));
|
||||||
}
|
}
|
||||||
let Some(binding_id) = host.writer_binding_id() else {
|
// Do not require a live Controller, complete Journal, or prior release here.
|
||||||
return Err(Error::RepositoryConflict(
|
// Runtime removal is the physical fence; finish_worker_removal revokes the
|
||||||
"Subject writer evidence is missing; ordinary deletion cannot bypass release".into(),
|
// exact old body only after its successful, matching retention receipt.
|
||||||
));
|
|
||||||
};
|
|
||||||
let writer = JournalWriterBinding {
|
|
||||||
subject_id: attr.subject_id.clone(),
|
|
||||||
runtime_id: attr.runtime_id.clone(),
|
|
||||||
worker_id: attr.worker_id.clone(),
|
|
||||||
binding_id: binding_id.into(),
|
|
||||||
};
|
|
||||||
if store.journal_writer_active(&writer).map_err(domain)? {
|
|
||||||
return Err(Error::RepositoryConflict("Subject remains occupied: restore to actual Idle, finish Journal recording, then explicitly release before ordinary deletion".into()));
|
|
||||||
}
|
|
||||||
if store.journal_retention_required(&host).map_err(domain)? {
|
|
||||||
return Err(Error::RepositoryConflict(
|
|
||||||
"Released Subject has unresolved Journal material; retain occupancy and source".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1886,20 +1895,6 @@ pub(super) fn validate_removal_retention(
|
|||||||
// its retained permanent fence, including recovery after the aggregate is gone.
|
// its retained permanent fence, including recovery after the aggregate is gone.
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let binding_id = host.writer_binding_id().ok_or_else(|| {
|
|
||||||
Error::RepositoryConflict("Subject writer evidence missing during removal recovery".into())
|
|
||||||
})?;
|
|
||||||
let binding = JournalWriterBinding {
|
|
||||||
subject_id: attr.subject_id.clone(),
|
|
||||||
runtime_id: attr.runtime_id.clone(),
|
|
||||||
worker_id: attr.worker_id.clone(),
|
|
||||||
binding_id: binding_id.into(),
|
|
||||||
};
|
|
||||||
if store.journal_writer_active(&binding).map_err(domain)? {
|
|
||||||
return Err(Error::RepositoryConflict(
|
|
||||||
"Prepared removal cannot bypass explicit Subject release".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let source = runtime_api::SessionPublicSource::Retained {
|
let source = runtime_api::SessionPublicSource::Retained {
|
||||||
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
||||||
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
||||||
@@ -1907,19 +1902,42 @@ pub(super) fn validate_removal_retention(
|
|||||||
let runtime = service.runtime.upgrade().ok_or_else(|| {
|
let runtime = service.runtime.upgrade().ok_or_else(|| {
|
||||||
Error::Store("Runtime unavailable for source retention confirmation".into())
|
Error::Store("Runtime unavailable for source retention confirmation".into())
|
||||||
})?;
|
})?;
|
||||||
if store.journal_retention_required(&host).map_err(domain)? {
|
let retention_required = store.journal_retention_required(&host).map_err(domain)?;
|
||||||
|
if prepared.runtime_request.session_disposition
|
||||||
|
== worker_runtime::retention::SessionDisposition::Purge
|
||||||
|
&& prepared.runtime_request.archive_id.is_none()
|
||||||
|
&& !retention_required
|
||||||
|
&& store
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?
|
||||||
|
{
|
||||||
|
// Compatibility for an already issued, history-complete ordinary Purge
|
||||||
|
// plan. New Subject plans always Archive. Keep the old fingerprint and
|
||||||
|
// disposition; never turn this into an erasure or allow an active writer.
|
||||||
|
// A missing Runtime Worker is NOT success: execute_worker_retention still
|
||||||
|
// has to return the exact persisted receipt before anything can commit.
|
||||||
|
// A pending receipt may outlive the physical aggregate while Runtime's
|
||||||
|
// in-memory catalog still contains the Worker. Pin release is best effort
|
||||||
|
// in this historical branch; exact Runtime retention replay still enforces
|
||||||
|
// pins if the source exists, and requires a receipt if it is already gone.
|
||||||
|
if let Err(error) = retain_source(&runtime, &service.workspace_id, &attr, source, true) {
|
||||||
|
tracing::debug!(%error, "Historical Purge source unavailable; reconcile exact Runtime receipt");
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
if prepared.runtime_request.session_disposition
|
if prepared.runtime_request.session_disposition
|
||||||
!= worker_runtime::retention::SessionDisposition::Archive
|
!= worker_runtime::retention::SessionDisposition::Archive
|
||||||
|| prepared.runtime_request.archive_id.is_none()
|
|| prepared.runtime_request.archive_id.is_none()
|
||||||
{
|
{
|
||||||
return Err(Error::RepositoryConflict(
|
return Err(Error::RepositoryConflict(
|
||||||
"unreflected Journal requires a retained original Session archive".into(),
|
"Subject removal retains its original Session; explicit erasure is required for Purge"
|
||||||
|
.into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, false)
|
let release = !retention_required;
|
||||||
{
|
if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, release) {
|
||||||
// Recovery may already have transferred the physical source before its Backend
|
// A lost reply may follow the physical transfer. Reconcile only the exact
|
||||||
// catalog commit; authenticate the exact reserved archive, never another Session.
|
// reserved archive and Session, whether or not an admitted remainder exists.
|
||||||
let archive = runtime_api::SessionPublicSource::Archived {
|
let archive = runtime_api::SessionPublicSource::Archived {
|
||||||
archive_id: prepared
|
archive_id: prepared
|
||||||
.runtime_request
|
.runtime_request
|
||||||
@@ -1927,17 +1945,97 @@ pub(super) fn validate_removal_retention(
|
|||||||
.clone()
|
.clone()
|
||||||
.expect("checked"),
|
.expect("checked"),
|
||||||
};
|
};
|
||||||
retain_source(&runtime, &service.workspace_id, &attr, archive, false)
|
retain_source(&runtime, &service.workspace_id, &attr, archive, release)
|
||||||
.map_err(|_| original)?;
|
.map_err(|_| original)?;
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
// This stopped/removal-fenced Session has no admitted remainder; purge need not
|
|
||||||
// force a generation. A failed release conservatively blocks ordinary deletion.
|
|
||||||
retain_source(&runtime, &service.workspace_id, &attr, source, true)?;
|
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A successful Runtime receipt, not catalog liveness, authorizes ending the old
|
||||||
|
/// writer. The ordinary removal commit releases occupancy atomically with catalog
|
||||||
|
/// deletion. Until then a crash leaves the singleton occupied and the same receipt
|
||||||
|
/// can safely repeat this step; no Controller restore or Journal generation is needed.
|
||||||
|
pub(super) fn finish_worker_removal<T>(
|
||||||
|
service: &WorkerRemovalService,
|
||||||
|
target: &RuntimeWorkerRef,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||||
|
commit: impl FnOnce() -> T,
|
||||||
|
) -> Result<T> {
|
||||||
|
let Some(attr) = removal_attribution(service, target)? else {
|
||||||
|
return Ok(commit());
|
||||||
|
};
|
||||||
|
let request = &prepared.runtime_request;
|
||||||
|
if prepared.plan.worker != *target
|
||||||
|
|| request.workspace_id != service.workspace_id
|
||||||
|
|| request.source_runtime_id != target.runtime_id
|
||||||
|
|| request.worker_id.to_string() != target.worker_id
|
||||||
|
|| !result.source_removed
|
||||||
|
|| result.worker_id != request.worker_id
|
||||||
|
|| result.operation_id != request.operation_id
|
||||||
|
|| result.input_fingerprint != request.input_fingerprint
|
||||||
|
|| result.session_disposition != request.session_disposition
|
||||||
|
|| result.diagnostics_disposition != request.diagnostics_disposition
|
||||||
|
|| result.diagnostics_retained
|
||||||
|
!= (request.diagnostics_disposition
|
||||||
|
== worker_runtime::retention::DiagnosticsDisposition::Retain)
|
||||||
|
{
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject removal receipt does not match its plan".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
|
||||||
|
let archive = result.archive.as_ref().ok_or_else(|| {
|
||||||
|
Error::RepositoryConflict("Subject removal archive receipt is missing".into())
|
||||||
|
})?;
|
||||||
|
if Some(&archive.archive_id) != request.archive_id.as_ref()
|
||||||
|
|| archive.workspace_id != request.workspace_id
|
||||||
|
|| archive.source_runtime_id != request.source_runtime_id
|
||||||
|
|| archive.source_worker_id != request.worker_id
|
||||||
|
|| archive.source_session_id != attr.session_id
|
||||||
|
|| archive.operation_id != request.operation_id
|
||||||
|
|| archive.input_fingerprint != request.input_fingerprint
|
||||||
|
|| archive.policy_id != request.policy_id
|
||||||
|
{
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject removal archive identity mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let api = service.journal_dispatch_api.upgrade().ok_or_else(|| {
|
||||||
|
Error::Store("Workspace API unavailable for Subject removal completion".into())
|
||||||
|
})?;
|
||||||
|
let gate = lifecycle_lock(&api, &attr.subject_id)?;
|
||||||
|
let _guard = gate
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
||||||
|
let store = store(&api)?;
|
||||||
|
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
||||||
|
if store.journal_session_erased(&host).map_err(domain)? {
|
||||||
|
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
|
||||||
|
// Erasure may win after Runtime archive transfer. Do not publish that
|
||||||
|
// archive or skip writer revocation: the same removal retry must take
|
||||||
|
// erased_archive_removal_proof and its physical-erasure commit path.
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Session erased during removal; reconcile the same removal operation".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
return Ok(commit());
|
||||||
|
}
|
||||||
|
store.release_removed_journal_body(&host).map_err(domain)?;
|
||||||
|
for session in store
|
||||||
|
.subject_session_attributions(&attr.subject_id)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.into_iter()
|
||||||
|
.filter(|a| a.runtime_id == target.runtime_id && a.worker_id == target.worker_id)
|
||||||
|
{
|
||||||
|
reconcile_session_jobs(&api, &session, false)?;
|
||||||
|
}
|
||||||
|
// Keep erasure excluded through the catalog commit: otherwise a tombstone
|
||||||
|
// could win between revocation and publication of the retained archive.
|
||||||
|
Ok(commit())
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub(super) struct Empty {}
|
pub(super) struct Empty {}
|
||||||
@@ -2089,7 +2187,28 @@ pub(super) fn reconcile_unknown(
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?;
|
.ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?;
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
let host = host(api, &store, grant)?;
|
let attr = store
|
||||||
|
.session_attribution(&grant.session_id)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.filter(|a| a.subject_id == grant.subject_id)
|
||||||
|
.ok_or_else(|| Error::WorkspacePermissionDenied("Journal attribution mismatch".into()))?;
|
||||||
|
let released = store
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?;
|
||||||
|
let host = if released {
|
||||||
|
// Recovery reads immutable evidence, never renews a legacy writer grant.
|
||||||
|
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
||||||
|
if !grant.binding_id.is_empty()
|
||||||
|
&& host.writer_binding_id() != Some(grant.binding_id.as_str())
|
||||||
|
{
|
||||||
|
return Err(Error::WorkspacePermissionDenied(
|
||||||
|
"Journal recovery binding mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
host
|
||||||
|
} else {
|
||||||
|
host(api, &store, grant)?
|
||||||
|
};
|
||||||
let erased = store.journal_session_erased(&host).map_err(domain)?;
|
let erased = store.journal_session_erased(&host).map_err(domain)?;
|
||||||
if !erased {
|
if !erased {
|
||||||
job.load_executable_request()?;
|
job.load_executable_request()?;
|
||||||
@@ -2097,8 +2216,12 @@ pub(super) fn reconcile_unknown(
|
|||||||
job_id: job.request.job_id.clone(),
|
job_id: job.request.job_id.clone(),
|
||||||
attempt_id: attempt.attempt_id.clone(),
|
attempt_id: attempt.attempt_id.clone(),
|
||||||
};
|
};
|
||||||
let g = if legacy_descriptor(&job.request) {
|
let generation = if released {
|
||||||
// Only reconcile already-saved legacy receipts; never redispatch descriptors.
|
store
|
||||||
|
.retained_journal_generation(&host, &grant.journal_id, &binding)
|
||||||
|
.map_err(domain)?
|
||||||
|
} else if legacy_descriptor(&job.request) {
|
||||||
|
Some(
|
||||||
store
|
store
|
||||||
.prepare_journal_generation(
|
.prepare_journal_generation(
|
||||||
&host,
|
&host,
|
||||||
@@ -2106,15 +2229,28 @@ pub(super) fn reconcile_unknown(
|
|||||||
binding,
|
binding,
|
||||||
grant.selected_ranges.clone(),
|
grant.selected_ranges.clone(),
|
||||||
)
|
)
|
||||||
.map_err(domain)?
|
.map_err(domain)?,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
let input = saved_input(&job.request, grant)?;
|
// Preserve active-body recovery: its saved intent can still establish
|
||||||
let generation = store
|
// the original generation fence. A released body must only read it.
|
||||||
.prepare_journal_input(&host, &input, binding)
|
Some(
|
||||||
.map_err(domain)?;
|
store
|
||||||
input.validate_generation(&generation).map_err(domain)?;
|
.prepare_journal_input(&host, &saved_input(&job.request, grant)?, binding)
|
||||||
generation
|
.map_err(domain)?,
|
||||||
|
)
|
||||||
};
|
};
|
||||||
|
if let Some(g) = generation {
|
||||||
|
if g.selected_ranges != grant.selected_ranges {
|
||||||
|
return Err(Error::WorkspacePermissionDenied(
|
||||||
|
"Journal recovery selection mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !legacy_descriptor(&job.request) {
|
||||||
|
saved_input(&job.request, grant)?
|
||||||
|
.validate_generation(&g)
|
||||||
|
.map_err(domain)?;
|
||||||
|
}
|
||||||
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
|
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
|
||||||
.map_err(domain)?;
|
.map_err(domain)?;
|
||||||
match journal::service::finish(&store, &ctx) {
|
match journal::service::finish(&store, &ctx) {
|
||||||
@@ -2144,9 +2280,14 @@ pub(super) fn reconcile_unknown(
|
|||||||
replayed: false,
|
replayed: false,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) {
|
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed)
|
||||||
|
{
|
||||||
|
if !released {
|
||||||
drain_after_completion(api, &job.request);
|
drain_after_completion(api, &job.request);
|
||||||
if job.input_disposition == crate::backend_job::JobInputDisposition::Available {
|
}
|
||||||
|
if job.input_disposition
|
||||||
|
== crate::backend_job::JobInputDisposition::Available
|
||||||
|
{
|
||||||
api.store.reclaim_backend_job_input(
|
api.store.reclaim_backend_job_input(
|
||||||
api.workspace_id(),
|
api.workspace_id(),
|
||||||
&job.request.job_id,
|
&job.request.job_id,
|
||||||
@@ -2163,6 +2304,7 @@ pub(super) fn reconcile_unknown(
|
|||||||
Err(error) => return Err(domain(error)),
|
Err(error) => return Err(domain(error)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) {
|
if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
@@ -2179,11 +2321,17 @@ pub(super) fn reconcile_unknown(
|
|||||||
false
|
false
|
||||||
};
|
};
|
||||||
if !confirmed_purge {
|
if !confirmed_purge {
|
||||||
|
if released {
|
||||||
|
// Source deletion already fenced this generation. An unbound Unknown
|
||||||
|
// still lacks execution-cleanup proof: preserve Unknown, but it no
|
||||||
|
// longer owns the Subject or blocks removal of the original body.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
return Err(Error::RepositoryConflict(
|
return Err(Error::RepositoryConflict(
|
||||||
"Unknown Journal attempt has no committed stop/Purge fence; not retried".into(),
|
"Unknown Journal attempt has no committed stop/Purge fence; not retried".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if !erased {
|
if !erased && !released {
|
||||||
close(api, &job.request, attempt)?;
|
close(api, &job.request, attempt)?;
|
||||||
}
|
}
|
||||||
api.store.reconcile_backend_journal_attempt(
|
api.store.reconcile_backend_journal_attempt(
|
||||||
@@ -2192,7 +2340,7 @@ pub(super) fn reconcile_unknown(
|
|||||||
None,
|
None,
|
||||||
&Utc::now().to_rfc3339(),
|
&Utc::now().to_rfc3339(),
|
||||||
)?;
|
)?;
|
||||||
if !erased {
|
if !erased && !released {
|
||||||
// Resume the same saved body input only after canonical execution/domain fences.
|
// Resume the same saved body input only after canonical execution/domain fences.
|
||||||
// dispatch selects its retained intent before it can capture any newer source.
|
// dispatch selects its retained intent before it can capture any newer source.
|
||||||
let attr = store
|
let attr = store
|
||||||
@@ -2209,6 +2357,10 @@ fn reconcile_session_jobs(
|
|||||||
attr: &crate::subjektiv::SubjectSessionAttribution,
|
attr: &crate::subjektiv::SubjectSessionAttribution,
|
||||||
erased: bool,
|
erased: bool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
let released = !erased
|
||||||
|
&& store(api)?
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?;
|
||||||
let mut after: Option<String> = None;
|
let mut after: Option<String> = None;
|
||||||
loop {
|
loop {
|
||||||
let jobs = api.store.list_backend_journal_jobs(
|
let jobs = api.store.list_backend_journal_jobs(
|
||||||
@@ -2223,6 +2375,29 @@ fn reconcile_session_jobs(
|
|||||||
.store
|
.store
|
||||||
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
|
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
|
||||||
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
|
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
|
||||||
|
if released {
|
||||||
|
// Writer revocation has already fenced all Journal mutations. End
|
||||||
|
// pending/Unknown attempts without claiming their input was reflected;
|
||||||
|
// saved input, operations and original source remain retained.
|
||||||
|
if !attempt.state.terminal() {
|
||||||
|
attempt = api.store.finish_backend_job_attempt(
|
||||||
|
api.workspace_id(),
|
||||||
|
&job.request.job_id,
|
||||||
|
&id,
|
||||||
|
BackendJobAttemptState::Failed,
|
||||||
|
"journal_body_released",
|
||||||
|
"Source Worker was removed; unfinished Journal history retained",
|
||||||
|
&Utc::now().to_rfc3339(),
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
api.schedule_backend_job_worker_cleanup(&attempt);
|
||||||
|
if attempt.state == BackendJobAttemptState::Unknown {
|
||||||
|
// A saved checkpoint may settle success; otherwise retain Unknown
|
||||||
|
// until canonical Job-worker cleanup has supplied its Purge fence.
|
||||||
|
reconcile_unknown(api, job, &attempt)?;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if !erased
|
if !erased
|
||||||
&& legacy_descriptor(&job.request)
|
&& legacy_descriptor(&job.request)
|
||||||
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available
|
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -3112,8 +3112,8 @@ impl SqliteWorkspaceStore {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
// Subject occupancy survives removal until an explicit release
|
// An unconfirmed legacy removal cannot release Subject occupancy.
|
||||||
// proof. It must not resurrect the removed creation reservation:
|
// Completed removal or explicit release does; never resurrect a reservation:
|
||||||
// Runtime callbacks require a live reservation or catalog member.
|
// Runtime callbacks require a live reservation or catalog member.
|
||||||
if stored.as_ref().is_some_and(|row| row.6 == "removed") {
|
if stored.as_ref().is_some_and(|row| row.6 == "removed") {
|
||||||
return Err(Error::RepositoryConflict(format!(
|
return Err(Error::RepositoryConflict(format!(
|
||||||
@@ -16223,7 +16223,7 @@ fn current_worker_singleton_owner(
|
|||||||
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
||||||
THEN 'created' ELSE reservation.state END \
|
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.state='succeeded') THEN 'created' ELSE reservation.state END \
|
||||||
FROM worker_singleton_owners owner \
|
FROM worker_singleton_owners owner \
|
||||||
LEFT JOIN worker_create_reservations reservation \
|
LEFT JOIN worker_create_reservations reservation \
|
||||||
ON reservation.workspace_id = owner.workspace_id \
|
ON reservation.workspace_id = owner.workspace_id \
|
||||||
|
|||||||
Reference in New Issue
Block a user