fix: reconcile erased Archive removals and exercise live compaction recording
Reconcile immutable Runtime removal input under a confirmed permanent erase fence. A confirmed source_removed receipt is audited without republishing erased archives; unknown receipt plus attested remaining aggregate permits an audited stale plan and proof-bound purge replacement. Unknown receipt and inventory remain unresolved. Distinguish domain tombstones from scoped Runtime physical erasure acknowledgements. Erase diagnostic originals and interrupted staging bound to the same archived Session/removal operation, validating Workspace/Worker/fingerprint and saved receipts without rewriting their historical retained flags. Finalize Backend removal only after physical acknowledgement, including previous Retain disposition. Connect public standalone protocol, real Worker request-threshold compaction twice within one turn, cancellation and real pre-AI input rollback, original canonical readers, existing Jobs and sequential Journal tools. Gate generation while compaction completes; verify all durable originals were served, summary/rolled-back input exclusion, exact final cursor, no extra input for successor drain, and one ongoing open entry. External model streams/usage are scripted, not semantic evaluation. Validation: Backend1112lib+11bin (existing ignore1), Runtime429lib+38bin, standalone72lib+27integration (existing model ignore1), runtime-api10, root check/doc/fmt/diff/naming, Server/Runtime builds, isolated worker_environment2 passed. Earlier broad suites remain recorded on the prior source. Fixtures and injected boundaries are not arbitrary crash/remote-transport-loss coverage; worker_environment only verifies startup/connection/cleanup. Acceptance clarification: T-734 assignment answer event22 explicitly defers real-model semantic condition6 to user use, excluding it from this completion scope. It remains unexecuted and unevaluated, not passed. Technical review findings still require a fresh independent verdict on this source.
This commit is contained in:
@@ -110,6 +110,50 @@ impl ErasedJournalSession {
|
||||
}
|
||||
}
|
||||
|
||||
/// Physical Runtime erasure acknowledgement, not merely a domain tombstone.
|
||||
/// Only the scoped erase request and a matching Session response construct it.
|
||||
/// The Runtime acknowledgement includes associated archived diagnostic originals;
|
||||
/// historical retention receipts retain their original values.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RuntimeJournalErasureAck {
|
||||
domain: ErasedJournalSession,
|
||||
source: runtime_api::SessionPublicSource,
|
||||
}
|
||||
|
||||
impl ErasedJournalSession {
|
||||
pub(crate) fn erase_runtime_source(
|
||||
self,
|
||||
runtime: &crate::hosts::RuntimeRegistry,
|
||||
source: runtime_api::SessionPublicSource,
|
||||
) -> crate::Result<RuntimeJournalErasureAck> {
|
||||
let response = runtime
|
||||
.session_material_erase(
|
||||
&self.worker.runtime_id,
|
||||
runtime_api::SessionMaterialEraseRequest {
|
||||
workspace_id: self.workspace_id.clone(),
|
||||
source: source.clone(),
|
||||
expected_session_id: self.session_id.clone(),
|
||||
},
|
||||
)
|
||||
.map_err(|e| e.into_error())?;
|
||||
if response.session_id != self.session_id {
|
||||
return Err(StoreError::RepositoryConflict(
|
||||
"Session erase receipt differs from the requested identity".into(),
|
||||
));
|
||||
}
|
||||
Ok(RuntimeJournalErasureAck {
|
||||
domain: self,
|
||||
source,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl RuntimeJournalErasureAck {
|
||||
pub(crate) fn domain_proof(&self) -> &ErasedJournalSession {
|
||||
&self.domain
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum WorkerRemovalBlocker {
|
||||
@@ -490,12 +534,125 @@ impl SqliteWorkspaceStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// An unknown acceptance is not failure. Supersede only after physical erasure
|
||||
/// fenced Archive effects and Runtime attested the same remaining aggregate.
|
||||
pub(crate) fn supersede_erased_worker_removal(
|
||||
&self,
|
||||
prepared: &PreparedWorkerRemoval,
|
||||
ack: &RuntimeJournalErasureAck,
|
||||
inventory: &WorkerRetentionInventory,
|
||||
) -> Result<(), WorkerRetentionError> {
|
||||
let proof = ack.domain_proof();
|
||||
if proof.workspace_id != prepared.plan.workspace_id
|
||||
|| proof.worker != prepared.plan.worker
|
||||
|| prepared.plan.session_disposition != SessionDisposition::Archive
|
||||
|| inventory.workspace_id != proof.workspace_id
|
||||
|| inventory.runtime_id != proof.worker.runtime_id
|
||||
|| inventory.worker_id.to_string() != proof.worker.worker_id
|
||||
|| inventory
|
||||
.session_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| id != &proof.session_id)
|
||||
{
|
||||
return Err(WorkerRetentionError::Invalid(
|
||||
"Erasure reconciliation source mismatch".into(),
|
||||
));
|
||||
}
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let plan = load_plan_op(&tx, &prepared.plan.operation_id)?
|
||||
.ok_or_else(|| StoreError::InvalidInput("operation missing".into()))?;
|
||||
if plan != prepared.plan
|
||||
|| !matches!(
|
||||
plan.state,
|
||||
WorkerRemovalPlanState::Planned
|
||||
| WorkerRemovalPlanState::Executing
|
||||
| WorkerRemovalPlanState::Failed
|
||||
)
|
||||
{
|
||||
return Err(StoreError::InvalidInput("active operation mismatch".into()));
|
||||
}
|
||||
insert_erasure_reconciliation_audit(&tx, &plan, ack, None)?;
|
||||
mark_stale(&tx, &plan, "journal_session_erased_archive_fenced")?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn audit_erased_worker_removal_receipt(
|
||||
&self,
|
||||
prepared: &PreparedWorkerRemoval,
|
||||
ack: &RuntimeJournalErasureAck,
|
||||
result: &WorkerRetentionExecutionResult,
|
||||
) -> Result<(), WorkerRetentionError> {
|
||||
let proof = ack.domain_proof();
|
||||
if proof.workspace_id != prepared.plan.workspace_id
|
||||
|| proof.worker != prepared.plan.worker
|
||||
|| result.operation_id != prepared.plan.operation_id
|
||||
|| result.input_fingerprint != prepared.plan.input_fingerprint
|
||||
|| result.worker_id.to_string() != proof.worker.worker_id
|
||||
|| !result.source_removed
|
||||
|| result.session_disposition != prepared.plan.session_disposition
|
||||
|| result.diagnostics_disposition != prepared.plan.diagnostics_disposition
|
||||
|| result.archive.as_ref().is_none_or(|a| {
|
||||
a.source_session_id != proof.session_id
|
||||
|| Some(&a.archive_id) != prepared.plan.archive_id.as_ref()
|
||||
|| a.workspace_id != proof.workspace_id
|
||||
|| a.source_runtime_id != proof.worker.runtime_id
|
||||
|| a.source_worker_id.to_string() != proof.worker.worker_id
|
||||
|| a.operation_id != result.operation_id
|
||||
|| a.input_fingerprint != result.input_fingerprint
|
||||
})
|
||||
{
|
||||
return Err(WorkerRetentionError::Invalid(
|
||||
"Erasure receipt mismatch".into(),
|
||||
));
|
||||
}
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
insert_erasure_reconciliation_audit(&tx, &prepared.plan, ack, Some(result))?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn commit_worker_removal(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
operation_id: &str,
|
||||
fp: &str,
|
||||
result: &WorkerRetentionExecutionResult,
|
||||
) -> Result<WorkerRemovalCommit, WorkerRetentionError> {
|
||||
self.commit_worker_removal_inner(workspace_id, operation_id, fp, result, None)
|
||||
}
|
||||
|
||||
/// Receipt-confirmed removal after erasure must not republish the erased Archive.
|
||||
/// The original operation, fingerprint and Runtime result remain unchanged.
|
||||
pub(crate) fn commit_erased_worker_removal(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
prepared: &PreparedWorkerRemoval,
|
||||
result: &WorkerRetentionExecutionResult,
|
||||
ack: &RuntimeJournalErasureAck,
|
||||
) -> Result<WorkerRemovalCommit, WorkerRetentionError> {
|
||||
self.commit_worker_removal_inner(
|
||||
workspace_id,
|
||||
&prepared.plan.operation_id,
|
||||
&prepared.plan.input_fingerprint,
|
||||
result,
|
||||
Some(ack),
|
||||
)
|
||||
}
|
||||
|
||||
fn commit_worker_removal_inner(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
operation_id: &str,
|
||||
fp: &str,
|
||||
result: &WorkerRetentionExecutionResult,
|
||||
erasure: Option<&RuntimeJournalErasureAck>,
|
||||
) -> Result<WorkerRemovalCommit, WorkerRetentionError> {
|
||||
self.with_conn_mut(|conn|{
|
||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
@@ -519,6 +676,13 @@ impl SqliteWorkspaceStore {
|
||||
return Err(StoreError::InvalidInput("Runtime retention result does not match removal plan".into()));
|
||||
}
|
||||
if !result.source_removed{return Err(StoreError::InvalidInput("Runtime source was not removed".into()));}
|
||||
if let Some(ack) = erasure {
|
||||
let proof = ack.domain_proof();
|
||||
if proof.workspace_id != workspace_id || proof.worker != plan.worker
|
||||
|| result.archive.as_ref().is_none_or(|a| a.source_session_id != proof.session_id) {
|
||||
return Err(StoreError::InvalidInput("Erased removal receipt requires matching source and physical Runtime erasure acknowledgement".into()));
|
||||
}
|
||||
}
|
||||
let worker=load_worker(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput("Worker missing before commit".into()))?;
|
||||
if worker.updated_at!=plan.worker_updated_at{return Err(StoreError::InvalidInput(format!("stale:{}:Worker updated_at changed",plan.plan_id)));}
|
||||
if worker.retention_state=="pinned" { return Err(StoreError::InvalidInput(format!("stale:{}:hold added",plan.plan_id))); }
|
||||
@@ -527,6 +691,7 @@ impl SqliteWorkspaceStore {
|
||||
let now=Utc::now().to_rfc3339();
|
||||
if let Some(a)=&result.archive{
|
||||
if plan.archive_id.as_deref()!=Some(&a.archive_id)||a.workspace_id!=workspace_id||a.source_runtime_id!=plan.worker.runtime_id||a.source_worker_id.to_string()!=plan.worker.worker_id||a.policy_id!=plan.policy_id||a.operation_id!=operation_id||a.input_fingerprint!=fp{return Err(StoreError::InvalidInput("archive manifest mismatch".into()));}
|
||||
if erasure.is_none() {
|
||||
let expires_at=match plan.archive_retention { ArchiveRetention::Forever=>None, ArchiveRetention::ForSeconds{seconds}=>{let seconds=i64::try_from(seconds).map_err(|_|StoreError::InvalidInput("archive retention deadline overflow".into()))?;Some((Utc::now()+chrono::Duration::seconds(seconds)).to_rfc3339())} };
|
||||
tx.execute("INSERT OR IGNORE INTO worker_session_archives(archive_id,workspace_id,runtime_id,worker_id,session_id,checksum_sha256,content_bytes,policy_id,policy_digest,operation_id,committed_at,expires_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",params![a.archive_id,workspace_id,plan.worker.runtime_id,plan.worker.worker_id,a.source_session_id,a.content_checksum_sha256,a.content_bytes,plan.policy_id,plan.policy_digest,operation_id,now,expires_at])?;
|
||||
let observe_grants = {
|
||||
@@ -574,20 +739,21 @@ impl SqliteWorkspaceStore {
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}else if plan.session_disposition==SessionDisposition::Archive{return Err(StoreError::InvalidInput("archive manifest missing".into()));}
|
||||
match plan.diagnostics_disposition {
|
||||
DiagnosticsDisposition::Purge if result.diagnostics_retained => return Err(StoreError::InvalidInput("Runtime retained diagnostics for purge disposition".into())),
|
||||
DiagnosticsDisposition::Retain if !result.diagnostics_retained => return Err(StoreError::InvalidInput("Runtime did not retain diagnostics".into())),
|
||||
DiagnosticsDisposition::Retain => {
|
||||
DiagnosticsDisposition::Retain if erasure.is_none() => {
|
||||
let seconds=plan.diagnostics_retention_seconds.ok_or_else(||StoreError::InvalidInput("diagnostics retention deadline missing".into()))?;
|
||||
let seconds=i64::try_from(seconds).map_err(|_|StoreError::InvalidInput("diagnostics retention deadline overflow".into()))?;
|
||||
let expires_at=(Utc::now()+chrono::Duration::seconds(seconds)).to_rfc3339();
|
||||
tx.execute("INSERT OR IGNORE INTO worker_diagnostics_archives(operation_id,workspace_id,runtime_id,worker_id,policy_id,policy_digest,committed_at,expires_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8)",params![operation_id,workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.policy_id,plan.policy_digest,now,expires_at])?;
|
||||
}
|
||||
DiagnosticsDisposition::Purge => {}
|
||||
DiagnosticsDisposition::Purge | DiagnosticsDisposition::Retain => {}
|
||||
}
|
||||
if plan.metadata_disposition==MetadataDisposition::Tombstone{
|
||||
tx.execute("INSERT OR IGNORE INTO worker_tombstones(workspace_id,runtime_id,worker_id,display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_digest,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,worker.display_name,worker.profile,worker.created_at,now,plan.archive_id,plan.policy_id,plan.policy_digest,operation_id])?;
|
||||
tx.execute("INSERT OR IGNORE INTO worker_tombstones(workspace_id,runtime_id,worker_id,display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_digest,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,worker.display_name,worker.profile,worker.created_at,now,if erasure.is_some() { None } else { plan.archive_id.clone() },plan.policy_id,plan.policy_digest,operation_id])?;
|
||||
}
|
||||
tx.execute("UPDATE worker_workdir_links SET unlinked_at=?4 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND unlinked_at IS NULL",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,now])?;
|
||||
let deleted=tx.execute("DELETE FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND updated_at=?4",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.worker_updated_at])?;
|
||||
@@ -595,6 +761,9 @@ impl SqliteWorkspaceStore {
|
||||
if deleted!=1{return Err(StoreError::InvalidInput(format!("stale:{}:removal fence changed",plan.plan_id)));}
|
||||
let catalog=commit_worker_catalog_removal(&tx,workspace_id,&plan.worker)?;
|
||||
tx.execute("UPDATE worker_removal_operations SET state='succeeded',failure_category=NULL,updated_at=?1 WHERE operation_id=?2",params![now,operation_id])?;
|
||||
if let Some(ack) = erasure {
|
||||
insert_erasure_reconciliation_audit(&tx, &plan, ack, Some(result))?;
|
||||
}
|
||||
tx.execute("INSERT OR IGNORE INTO worker_retention_audit_events(event_id,operation_id,workspace_id,event_kind,detail,created_at) VALUES(?1,?2,?3,'worker_removed',?4,?5)",params![stable("wre",operation_id),operation_id,workspace_id,format!("runtime_id={} worker_id={} session={} metadata={} diagnostics={}",plan.worker.runtime_id,plan.worker.worker_id,sess(plan.session_disposition),meta(plan.metadata_disposition),diag(plan.diagnostics_disposition)),now])?;
|
||||
tx.commit()?;plan.state=WorkerRemovalPlanState::Succeeded;plan.updated_at=now;Ok(WorkerRemovalCommit { plan, catalog })
|
||||
}).map_err(map_error)
|
||||
@@ -969,6 +1138,32 @@ fn load_plan_q(c: &Connection, key: &str, id: &str) -> crate::Result<Option<Work
|
||||
.optional()
|
||||
.map_err(StoreError::from)
|
||||
}
|
||||
fn insert_erasure_reconciliation_audit(
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
plan: &WorkerRemovalPlan,
|
||||
ack: &RuntimeJournalErasureAck,
|
||||
result: Option<&WorkerRetentionExecutionResult>,
|
||||
) -> crate::Result<()> {
|
||||
let proof = ack.domain_proof();
|
||||
let kind = if result.is_some() {
|
||||
"erased_removal_receipt"
|
||||
} else {
|
||||
"erased_archive_superseded"
|
||||
};
|
||||
let detail = serde_json::json!({
|
||||
"input_fingerprint": plan.input_fingerprint,
|
||||
"session_id": proof.session_id,
|
||||
"archive_id": plan.archive_id,
|
||||
"source_removed": result.map(|r| r.source_removed),
|
||||
"old_diagnostics_retained": result.map(|r| r.diagnostics_retained),
|
||||
"runtime_erasure_ack": { "session_id": proof.session_id, "source": ack.source },
|
||||
"old_acceptance": if result.is_some() { "receipt_confirmed" } else { "unknown_archive_fenced" },
|
||||
});
|
||||
tx.execute("INSERT OR IGNORE INTO worker_retention_audit_events(event_id,operation_id,workspace_id,event_kind,detail,created_at) VALUES(?1,?2,?3,?4,?5,?6)",
|
||||
params![stable("wre", &format!("{}:{kind}", plan.operation_id)), plan.operation_id, plan.workspace_id, kind, detail.to_string(), Utc::now().to_rfc3339()])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mark_stale(
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
plan: &WorkerRemovalPlan,
|
||||
|
||||
@@ -1889,8 +1889,98 @@ impl WorkerRemovalService {
|
||||
&self,
|
||||
runtime: &RuntimeRegistry,
|
||||
target: &RuntimeWorkerRef,
|
||||
prepared: crate::retention::PreparedWorkerRemoval,
|
||||
mut prepared: crate::retention::PreparedWorkerRemoval,
|
||||
parent_workspace_deletion_operation_id: Option<&str>,
|
||||
) -> std::result::Result<worker::WorkspaceResponse, String> {
|
||||
if prepared.plan.session_disposition
|
||||
== worker_runtime::retention::SessionDisposition::Archive
|
||||
{
|
||||
let erasure_ack = match subjektiv_journal::erased_archive_removal_proof(
|
||||
self, runtime, target, &prepared,
|
||||
) {
|
||||
Ok(ack) => ack,
|
||||
Err(error) => {
|
||||
return Ok(worker_remove_error_response(
|
||||
StatusCode::CONFLICT,
|
||||
"erased_removal_reconciliation_unconfirmed",
|
||||
&error.to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
if let Some(ack) = erasure_ack {
|
||||
// Replay EXACT old input under the acknowledged permanent fence.
|
||||
// Runtime can return an existing receipt, but cannot re-Archive.
|
||||
match runtime.execute_worker_retention(target, prepared.runtime_request.clone()) {
|
||||
Ok(result) => {
|
||||
if let Err(error) = self
|
||||
.store
|
||||
.audit_erased_worker_removal_receipt(&prepared, &ack, &result)
|
||||
{
|
||||
return Ok(worker_retention_error_response(error));
|
||||
}
|
||||
return match self.worker_projection.publish_ordered(|| {
|
||||
self.store
|
||||
.commit_erased_worker_removal(
|
||||
&self.workspace_id,
|
||||
&prepared,
|
||||
&result,
|
||||
&ack,
|
||||
)
|
||||
.map(|commit| ((), commit.catalog))
|
||||
}) {
|
||||
Ok(()) => Ok(worker_remove_success_response(target)),
|
||||
Err(error) => Ok(worker_retention_error_response(error)),
|
||||
};
|
||||
}
|
||||
Err(_) => {
|
||||
// Errors are unknown acceptance, not failure. A missing
|
||||
// aggregate without a receipt remains unresolved.
|
||||
let inventory = match runtime.worker_retention_inventory(target) {
|
||||
Ok(inventory) => inventory,
|
||||
Err(_) => {
|
||||
return Ok(worker_remove_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"erased_removal_outcome_unknown",
|
||||
"Old Runtime outcome cannot be reconciled; removal remains recoverable",
|
||||
));
|
||||
}
|
||||
};
|
||||
if let Err(error) = self
|
||||
.store
|
||||
.supersede_erased_worker_removal(&prepared, &ack, &inventory)
|
||||
{
|
||||
return Ok(worker_retention_error_response(error));
|
||||
}
|
||||
let request = crate::retention::WorkerRemovalPlanRequest {
|
||||
workspace_id: self.workspace_id.clone(),
|
||||
worker: target.clone(),
|
||||
reason: prepared.plan.reason.clone(),
|
||||
journal_erasure: Some(ack.domain_proof().clone()),
|
||||
};
|
||||
let plan = match self.store.plan_worker_removal(&request, &inventory) {
|
||||
Ok(plan) => plan,
|
||||
Err(error) => return Ok(worker_retention_error_response(error)),
|
||||
};
|
||||
prepared = match self.store.prepare_worker_removal_execution(
|
||||
&self.workspace_id,
|
||||
&plan.plan_id,
|
||||
&plan.input_fingerprint,
|
||||
) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return Ok(worker_retention_error_response(error)),
|
||||
};
|
||||
if let Some(parent) = parent_workspace_deletion_operation_id {
|
||||
self.store
|
||||
.append_workspace_deletion_child_operation(
|
||||
parent,
|
||||
&prepared.plan.operation_id,
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Err(error) = subjektiv_journal::validate_removal_retention(self, target, &prepared) {
|
||||
return Ok(worker_remove_error_response(
|
||||
StatusCode::CONFLICT,
|
||||
@@ -2168,7 +2258,12 @@ impl WorkerRemovalService {
|
||||
));
|
||||
}
|
||||
return self
|
||||
.resume_worker_retention(runtime, &target, prepared)
|
||||
.resume_worker_retention(
|
||||
runtime,
|
||||
&target,
|
||||
prepared,
|
||||
parent_workspace_deletion_operation_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
|
||||
@@ -878,6 +878,51 @@ pub(super) fn prepare_worker_removal(
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// A successful erase response proves the Runtime's permanent fence. The reserved
|
||||
/// archive covers transfer-before-Backend-commit crashes too. Never admit or generate.
|
||||
pub(super) fn erased_archive_removal_proof(
|
||||
service: &WorkerRemovalService,
|
||||
runtime: &RuntimeRegistry,
|
||||
target: &RuntimeWorkerRef,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
) -> Result<Option<crate::retention::RuntimeJournalErasureAck>> {
|
||||
let Some(attr) = removal_attribution(service, target)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let store = crate::subjektiv::SubjektivStore::open(
|
||||
&service.journal_storage,
|
||||
&service.journal_registration,
|
||||
)
|
||||
.map_err(|e| Error::Store(e.to_string()))?;
|
||||
let Some(proof) =
|
||||
crate::retention::ErasedJournalSession::validated(&service.workspace_id, &store, &attr)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let api = service.journal_dispatch_api.upgrade().ok_or_else(|| {
|
||||
Error::Store("Workspace API unavailable for erased removal recovery".into())
|
||||
})?;
|
||||
reconcile_session_jobs(&api, &attr, true)?;
|
||||
let source = runtime_api::SessionPublicSource::Retained {
|
||||
worker_id: prepared.runtime_request.worker_id,
|
||||
};
|
||||
let ack = match proof.clone().erase_runtime_source(runtime, source) {
|
||||
Ok(ack) => ack,
|
||||
Err(original) => {
|
||||
let Some(archive_id) = prepared.runtime_request.archive_id.clone() else {
|
||||
return Err(original);
|
||||
};
|
||||
proof
|
||||
.erase_runtime_source(
|
||||
runtime,
|
||||
runtime_api::SessionPublicSource::Archived { archive_id },
|
||||
)
|
||||
.map_err(|_| original)?
|
||||
}
|
||||
};
|
||||
Ok(Some(ack))
|
||||
}
|
||||
|
||||
pub(super) fn validate_removal_retention(
|
||||
service: &WorkerRemovalService,
|
||||
target: &RuntimeWorkerRef,
|
||||
|
||||
@@ -2141,3 +2141,503 @@ async fn serialized_removal_input_cannot_supply_journal_erasure_authority() {
|
||||
assert!(!f.store.journal_session_erased(&f.host).unwrap());
|
||||
assert!(session_root(f.dir.path(), &f.target).exists());
|
||||
}
|
||||
|
||||
impl SubjectRemovalFixture {
|
||||
async fn reopen(self) -> Self {
|
||||
let Self {
|
||||
dir,
|
||||
api,
|
||||
dispatcher,
|
||||
controller,
|
||||
target,
|
||||
attr,
|
||||
host,
|
||||
store,
|
||||
} = self;
|
||||
let config = api.config.clone();
|
||||
drop(api);
|
||||
drop(dispatcher);
|
||||
drop(store);
|
||||
let control = Arc::new(test_control_store(&config));
|
||||
let dispatcher = Arc::new(
|
||||
crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new(
|
||||
config.clone(),
|
||||
control.clone(),
|
||||
),
|
||||
);
|
||||
let api = WorkspaceApi::new_with_execution_backend_and_broker(
|
||||
config,
|
||||
control,
|
||||
Arc::new(DeterministicExecutionBackend::default()),
|
||||
BackendResourceBroker::default(),
|
||||
HostExecutors::default(),
|
||||
Some(dispatcher.clone()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
Self {
|
||||
dir,
|
||||
api,
|
||||
dispatcher,
|
||||
controller,
|
||||
target,
|
||||
attr,
|
||||
host,
|
||||
store,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_erased_removal(&self) {
|
||||
assert!(
|
||||
self.api
|
||||
.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &self.target)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
self.api
|
||||
.config_store
|
||||
.worker_session_archive(TEST_WORKSPACE_ID, &self.target, &self.attr.session_id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
self.api
|
||||
.store
|
||||
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &self.attr.session_id, None, 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(!session_root(self.dir.path(), &self.target).exists());
|
||||
assert!(matches!(
|
||||
source_retention(self.dir.path()).ensure_live(self.attr.session_id.parse().unwrap()),
|
||||
Err(session_store::StoreError::SessionErased(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erased_archive_plan_resend_and_reopen_replan_without_republishing_sources() {
|
||||
for stage in ["planned", "executing", "failed"] {
|
||||
for restart in [false, true] {
|
||||
let mut f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
let policy = f
|
||||
.api
|
||||
.config_store
|
||||
.worker_retention_policy(TEST_WORKSPACE_ID)
|
||||
.unwrap();
|
||||
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
let archive_parent = f
|
||||
.dir
|
||||
.path()
|
||||
.join(".test-embedded-runtime-store/archives/workers");
|
||||
if stage == "planned" {
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE UPDATE OF state ON worker_removal_operations WHEN NEW.state='executing' BEGIN SELECT RAISE(ABORT,'injected execution interruption'); END;").unwrap();
|
||||
} else {
|
||||
// Fail Archive publication before any receipt. Suppress recording
|
||||
// failure too to model the executing-at-Host-stop durable boundary.
|
||||
std::fs::create_dir_all(archive_parent.parent().unwrap()).unwrap();
|
||||
std::fs::write(&archive_parent, "publication blocked").unwrap();
|
||||
if stage == "executing" {
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE UPDATE OF state ON worker_removal_operations WHEN NEW.state='failed' BEGIN SELECT RAISE(ABORT,'injected failure-save interruption'); END;").unwrap();
|
||||
}
|
||||
}
|
||||
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||
conn.execute_batch("DROP TRIGGER IF EXISTS interrupt_removal")
|
||||
.unwrap();
|
||||
if stage != "planned" {
|
||||
std::fs::remove_file(&archive_parent).unwrap();
|
||||
}
|
||||
let old = f
|
||||
.api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(serde_json::to_value(old.plan.state).unwrap(), stage);
|
||||
assert_eq!(
|
||||
old.plan.session_disposition,
|
||||
worker_runtime::retention::SessionDisposition::Archive
|
||||
);
|
||||
f.api
|
||||
.erase_subjektiv_journal_session(
|
||||
&f.attr.session_id,
|
||||
"erase-interrupted-archive-plan",
|
||||
)
|
||||
.unwrap();
|
||||
if restart {
|
||||
f = f.reopen().await;
|
||||
}
|
||||
let response = f.remove();
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::OK.as_u16(),
|
||||
"{stage}/{restart}: {}",
|
||||
response.body
|
||||
);
|
||||
let saved = f
|
||||
.api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
saved.plan.session_disposition,
|
||||
worker_runtime::retention::SessionDisposition::Purge
|
||||
);
|
||||
assert_eq!(saved.plan.reason, old.plan.reason);
|
||||
assert_ne!(saved.plan.operation_id, old.plan.operation_id);
|
||||
let (state, fingerprint, detail): (String, String, String) = conn.query_row(
|
||||
"SELECT o.state,o.input_fingerprint,a.detail FROM worker_removal_operations o JOIN worker_retention_audit_events a USING(operation_id) WHERE o.operation_id=?1 AND a.event_kind='erased_archive_superseded'",
|
||||
[&old.plan.operation_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||
assert_eq!(state, "stale");
|
||||
assert_eq!(fingerprint, old.plan.input_fingerprint);
|
||||
let detail: Value = serde_json::from_str(&detail).unwrap();
|
||||
assert_eq!(detail["old_acceptance"], "unknown_archive_fenced");
|
||||
assert!(detail["source_removed"].is_null());
|
||||
f.assert_erased_removal();
|
||||
assert_eq!(
|
||||
f.api
|
||||
.config_store
|
||||
.worker_retention_policy(TEST_WORKSPACE_ID)
|
||||
.unwrap(),
|
||||
policy
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erased_completed_archive_receipt_is_audited_without_republishing_on_resend_or_reopen() {
|
||||
for restart in [false, true] {
|
||||
let mut f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
// Runtime has committed source_removed, but Backend commit/response is lost.
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected metadata commit interruption'); END;").unwrap();
|
||||
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||
conn.execute_batch("DROP TRIGGER interrupt_removal")
|
||||
.unwrap();
|
||||
let old = f
|
||||
.api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
old.plan.state,
|
||||
crate::retention::WorkerRemovalPlanState::Failed
|
||||
);
|
||||
let receipt = f
|
||||
.api
|
||||
.runtime
|
||||
.execute_worker_retention(&f.target, old.runtime_request.clone())
|
||||
.unwrap();
|
||||
assert!(receipt.source_removed);
|
||||
let archive_id = old.plan.archive_id.as_ref().unwrap();
|
||||
let archive_source = f
|
||||
.dir
|
||||
.path()
|
||||
.join(".test-embedded-runtime-store/archives/workers")
|
||||
.join(archive_id)
|
||||
.join("session");
|
||||
assert!(archive_source.exists());
|
||||
// The domain tombstone commits even though retained-source resolution fails.
|
||||
// Recovery must locate only this plan's exact reserved archive, not publish it.
|
||||
assert!(
|
||||
f.api
|
||||
.erase_subjektiv_journal_session(&f.attr.session_id, "erase-transferred-original")
|
||||
.is_err()
|
||||
);
|
||||
assert!(f.store.journal_session_erased(&f.host).unwrap());
|
||||
if restart {
|
||||
f = f.reopen().await;
|
||||
}
|
||||
let response = f.remove();
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::OK.as_u16(),
|
||||
"{restart}: {}",
|
||||
response.body
|
||||
);
|
||||
let saved = f
|
||||
.api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(saved.plan.operation_id, old.plan.operation_id);
|
||||
assert_eq!(saved.plan.input_fingerprint, old.plan.input_fingerprint);
|
||||
assert_eq!(saved.plan.session_disposition, old.plan.session_disposition);
|
||||
let detail: String = conn.query_row("SELECT detail FROM worker_retention_audit_events WHERE operation_id=?1 AND event_kind='erased_removal_receipt'", [&old.plan.operation_id], |r| r.get(0)).unwrap();
|
||||
let detail: Value = serde_json::from_str(&detail).unwrap();
|
||||
assert_eq!(detail["source_removed"], true);
|
||||
assert_eq!(detail["old_acceptance"], "receipt_confirmed");
|
||||
assert!(!archive_source.exists());
|
||||
f.assert_erased_removal();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erased_archive_without_receipt_or_inventory_remains_unknown_without_replanning() {
|
||||
let f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE UPDATE OF state ON worker_removal_operations WHEN NEW.state='executing' BEGIN SELECT RAISE(ABORT,'injected interruption'); END;").unwrap();
|
||||
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||
conn.execute_batch("DROP TRIGGER interrupt_removal")
|
||||
.unwrap();
|
||||
let old = f
|
||||
.api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
f.api
|
||||
.erase_subjektiv_journal_session(&f.attr.session_id, "erase-with-unknown-outcome")
|
||||
.unwrap();
|
||||
// Corrupt/missing aggregate after an unacknowledged operation is NOT evidence
|
||||
// that source_removed completed. Runtime has no matching receipt here.
|
||||
std::fs::remove_file(
|
||||
session_root(f.dir.path(), &f.target)
|
||||
.parent()
|
||||
.unwrap()
|
||||
.join("worker.json"),
|
||||
)
|
||||
.unwrap();
|
||||
let response = f.remove();
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::SERVICE_UNAVAILABLE.as_u16(),
|
||||
"{}",
|
||||
response.body
|
||||
);
|
||||
let body: Value = serde_json::from_str(&response.body).unwrap();
|
||||
assert_eq!(body["code"], "erased_removal_outcome_unknown");
|
||||
let saved = f
|
||||
.api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(saved.plan.operation_id, old.plan.operation_id);
|
||||
assert_eq!(
|
||||
saved.plan.state,
|
||||
crate::retention::WorkerRemovalPlanState::Executing
|
||||
);
|
||||
let count: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM worker_removal_operations WHERE worker_id=?1",
|
||||
[&f.target.worker_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erased_archive_retained_diagnostics_receipt_finalizes_only_after_physical_ack_on_resend_or_reopen()
|
||||
{
|
||||
for restart in [false, true] {
|
||||
for interrupt_erase in [false, true] {
|
||||
let mut f = SubjectRemovalFixture::new().await;
|
||||
f.disable();
|
||||
let policy = f
|
||||
.api
|
||||
.config_store
|
||||
.worker_retention_policy(TEST_WORKSPACE_ID)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let policy = f
|
||||
.api
|
||||
.config_store
|
||||
.update_worker_retention_policy(
|
||||
TEST_WORKSPACE_ID,
|
||||
&policy.policy_digest,
|
||||
&crate::retention::WorkerRetentionPolicyUpdate {
|
||||
policy_id: "fixture-retained-diagnostics".into(),
|
||||
session_disposition: policy.session_disposition,
|
||||
metadata_disposition: policy.metadata_disposition,
|
||||
archive_retention: policy.archive_retention,
|
||||
diagnostics_disposition:
|
||||
worker_runtime::retention::DiagnosticsDisposition::Retain,
|
||||
diagnostics_retention_seconds: Some(3600),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let worker_root = session_root(f.dir.path(), &f.target)
|
||||
.parent()
|
||||
.unwrap()
|
||||
.to_path_buf();
|
||||
std::fs::create_dir_all(worker_root.join("runs/fixture-run")).unwrap();
|
||||
std::fs::write(
|
||||
worker_root.join("runs/fixture-run/worker.out.log"),
|
||||
"diagnostic original",
|
||||
)
|
||||
.unwrap();
|
||||
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||
// Real Runtime receipt exists; Backend commit/response is interrupted.
|
||||
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected metadata commit interruption'); END;").unwrap();
|
||||
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||
conn.execute_batch("DROP TRIGGER interrupt_removal")
|
||||
.unwrap();
|
||||
let old = f
|
||||
.api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let runtime_root = f.dir.path().join(".test-embedded-runtime-store");
|
||||
let receipt_path = runtime_root
|
||||
.join("retention/operations")
|
||||
.join(format!("{}.json", old.plan.operation_id));
|
||||
let receipt_bytes = std::fs::read(&receipt_path).unwrap();
|
||||
let receipt = f
|
||||
.api
|
||||
.runtime
|
||||
.execute_worker_retention(&f.target, old.runtime_request.clone())
|
||||
.unwrap();
|
||||
assert!(receipt.source_removed);
|
||||
assert!(receipt.diagnostics_retained);
|
||||
let diagnostics_root = runtime_root
|
||||
.join("archives/diagnostics")
|
||||
.join(&old.plan.operation_id);
|
||||
let diagnostics_manifest = diagnostics_root.join("manifest.json");
|
||||
let diagnostics_bytes = std::fs::read(&diagnostics_manifest).unwrap();
|
||||
let archive_source = runtime_root
|
||||
.join("archives/workers")
|
||||
.join(old.plan.archive_id.as_ref().unwrap())
|
||||
.join("session");
|
||||
assert!(diagnostics_root.exists());
|
||||
assert!(archive_source.exists());
|
||||
// Domain erasure succeeds, but no physical acknowledgement can be
|
||||
// claimed from the old retained locator after aggregate removal.
|
||||
assert!(
|
||||
f.api
|
||||
.erase_subjektiv_journal_session(
|
||||
&f.attr.session_id,
|
||||
"erase-retained-diagnostics"
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
if interrupt_erase {
|
||||
// Runtime must reject mismatched diagnostics identity rather than
|
||||
// acknowledge removal of an unrelated operation. No fake Host ack.
|
||||
let mut damaged: Value = serde_json::from_slice(&diagnostics_bytes).unwrap();
|
||||
damaged["input_fingerprint"] = json!("unrelated-operation");
|
||||
std::fs::write(&diagnostics_manifest, serde_json::to_vec(&damaged).unwrap())
|
||||
.unwrap();
|
||||
let response = f.remove();
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::CONFLICT.as_u16(),
|
||||
"{}",
|
||||
response.body
|
||||
);
|
||||
let body: Value = serde_json::from_str(&response.body).unwrap();
|
||||
assert_eq!(body["code"], "erased_removal_reconciliation_unconfirmed");
|
||||
assert!(
|
||||
f.api
|
||||
.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
let count: i64 = conn.query_row("SELECT COUNT(*) FROM worker_retention_audit_events WHERE operation_id=?1 AND event_kind='erased_removal_receipt'", [&old.plan.operation_id], |r| r.get(0)).unwrap();
|
||||
assert_eq!(
|
||||
count, 0,
|
||||
"failed acknowledgement cannot authorize finalization/audit"
|
||||
);
|
||||
assert_eq!(std::fs::read(&receipt_path).unwrap(), receipt_bytes);
|
||||
let waiting = f
|
||||
.api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(waiting.plan.operation_id, old.plan.operation_id);
|
||||
assert_eq!(waiting.plan.input_fingerprint, old.plan.input_fingerprint);
|
||||
assert_eq!(
|
||||
waiting.plan.state,
|
||||
crate::retention::WorkerRemovalPlanState::Executing
|
||||
);
|
||||
let plans: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM worker_removal_operations WHERE worker_id=?1",
|
||||
[&f.target.worker_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
plans, 1,
|
||||
"missing physical ack cannot authorize a replacement plan"
|
||||
);
|
||||
std::fs::write(&diagnostics_manifest, &diagnostics_bytes).unwrap();
|
||||
}
|
||||
if restart {
|
||||
f = f.reopen().await;
|
||||
}
|
||||
let response = f.remove();
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::OK.as_u16(),
|
||||
"restart={restart}, interrupted={interrupt_erase}: {}",
|
||||
response.body
|
||||
);
|
||||
assert!(!diagnostics_root.exists());
|
||||
assert!(!archive_source.exists());
|
||||
assert_eq!(
|
||||
std::fs::read(&receipt_path).unwrap(),
|
||||
receipt_bytes,
|
||||
"historical Runtime receipt must not be rewritten"
|
||||
);
|
||||
let saved = f
|
||||
.api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(saved.plan.operation_id, old.plan.operation_id);
|
||||
assert_eq!(saved.plan.input_fingerprint, old.plan.input_fingerprint);
|
||||
assert_eq!(
|
||||
saved.plan.diagnostics_disposition,
|
||||
old.plan.diagnostics_disposition
|
||||
);
|
||||
let detail: String = conn.query_row("SELECT detail FROM worker_retention_audit_events WHERE operation_id=?1 AND event_kind='erased_removal_receipt'", [&old.plan.operation_id], |r| r.get(0)).unwrap();
|
||||
let detail: Value = serde_json::from_str(&detail).unwrap();
|
||||
assert_eq!(detail["source_removed"], true);
|
||||
assert_eq!(detail["old_diagnostics_retained"], true);
|
||||
assert_eq!(
|
||||
detail["runtime_erasure_ack"]["session_id"],
|
||||
f.attr.session_id
|
||||
);
|
||||
let diagnostics_catalog: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM worker_diagnostics_archives WHERE operation_id=?1",
|
||||
[&old.plan.operation_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(diagnostics_catalog, 0);
|
||||
let tombstone_archive: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT archive_id FROM worker_tombstones WHERE operation_id=?1",
|
||||
[&old.plan.operation_id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(tombstone_archive.is_none());
|
||||
f.assert_erased_removal();
|
||||
assert_eq!(
|
||||
f.api
|
||||
.config_store
|
||||
.worker_retention_policy(TEST_WORKSPACE_ID)
|
||||
.unwrap(),
|
||||
Some(policy)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1451,6 +1451,40 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
||||
"Worker retention authority is unavailable".to_string(),
|
||||
))
|
||||
}
|
||||
fn supersede_erased_worker_removal(
|
||||
&self,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||
inventory: &worker_runtime::retention::WorkerRetentionInventory,
|
||||
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
|
||||
let _ = (prepared, proof, inventory);
|
||||
Err(crate::retention::WorkerRetentionError::Invalid(
|
||||
"Erased removal reconciliation authority is unavailable".into(),
|
||||
))
|
||||
}
|
||||
fn audit_erased_worker_removal_receipt(
|
||||
&self,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
|
||||
let _ = (prepared, proof, result);
|
||||
Err(crate::retention::WorkerRetentionError::Invalid(
|
||||
"Erased removal reconciliation authority is unavailable".into(),
|
||||
))
|
||||
}
|
||||
fn commit_erased_worker_removal(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||
) -> std::result::Result<WorkerRemovalCommit, crate::retention::WorkerRetentionError> {
|
||||
let _ = (workspace_id, prepared, result, proof);
|
||||
Err(crate::retention::WorkerRetentionError::Invalid(
|
||||
"Erased removal reconciliation authority is unavailable".into(),
|
||||
))
|
||||
}
|
||||
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>>;
|
||||
fn upsert_repository(&self, record: &RepositoryRecord) -> Result<()>;
|
||||
fn insert_repository(&self, record: &RepositoryRecord) -> Result<RepositoryInsertOutcome>;
|
||||
@@ -5438,6 +5472,37 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
)
|
||||
}
|
||||
|
||||
fn supersede_erased_worker_removal(
|
||||
&self,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||
inventory: &worker_runtime::retention::WorkerRetentionInventory,
|
||||
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
|
||||
SqliteWorkspaceStore::supersede_erased_worker_removal(self, prepared, proof, inventory)
|
||||
}
|
||||
fn audit_erased_worker_removal_receipt(
|
||||
&self,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
|
||||
SqliteWorkspaceStore::audit_erased_worker_removal_receipt(self, prepared, proof, result)
|
||||
}
|
||||
fn commit_erased_worker_removal(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||
) -> std::result::Result<WorkerRemovalCommit, crate::retention::WorkerRetentionError> {
|
||||
SqliteWorkspaceStore::commit_erased_worker_removal(
|
||||
self,
|
||||
workspace_id,
|
||||
prepared,
|
||||
result,
|
||||
proof,
|
||||
)
|
||||
}
|
||||
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>> {
|
||||
self.with_conn(|conn| {
|
||||
let mut stmt = conn.prepare(
|
||||
|
||||
Reference in New Issue
Block a user