fix: reconcile unfinished retention before Subject deletion
This commit is contained in:
@@ -1219,8 +1219,34 @@ pub struct SubjectDeleteWorkerResponse {
|
|||||||
pub originals_absent: bool,
|
pub originals_absent: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Additional erasure of a Subject Job's newly reconciled removal copies.
|
||||||
|
/// Both identities are exact saved requests; this never changes the old receipt.
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct SubjectJobRemovalPurgeRequest {
|
||||||
|
pub deletion: SubjectDeleteWorkerRequest,
|
||||||
|
pub removal: WorkerRetentionExecutionRequest,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Confirms current physical erasure under the permanent deletion fence.
|
||||||
|
/// Operation identity belongs to `removal`, not the Subject deletion operation.
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct SubjectJobRemovalPurgeResponse {
|
||||||
|
pub workspace_id: String,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub operation_id: String,
|
||||||
|
pub input_fingerprint: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[api(reqwest, axum)]
|
#[api(reqwest, axum)]
|
||||||
pub trait RuntimeApi {
|
pub trait RuntimeApi {
|
||||||
|
#[post("/v1/subject-job-removal-purge", status = 200, error_status = 400)]
|
||||||
|
async fn purge_subject_job_removal(
|
||||||
|
&self,
|
||||||
|
#[body] request: SubjectJobRemovalPurgeRequest,
|
||||||
|
) -> Result<SubjectJobRemovalPurgeResponse, RuntimeApiError>;
|
||||||
|
|
||||||
#[post("/v1/subject-delete-worker", status = 200, error_status = 400)]
|
#[post("/v1/subject-delete-worker", status = 200, error_status = 400)]
|
||||||
async fn subject_delete_worker(
|
async fn subject_delete_worker(
|
||||||
&self,
|
&self,
|
||||||
@@ -1787,6 +1813,18 @@ mod tests {
|
|||||||
) -> Result<SessionMaterialPinResponse, RuntimeApiError> {
|
) -> Result<SessionMaterialPinResponse, RuntimeApiError> {
|
||||||
Err(test_error(501))
|
Err(test_error(501))
|
||||||
}
|
}
|
||||||
|
async fn purge_subject_job_removal(
|
||||||
|
&self,
|
||||||
|
request: SubjectJobRemovalPurgeRequest,
|
||||||
|
) -> Result<SubjectJobRemovalPurgeResponse, RuntimeApiError> {
|
||||||
|
Ok(SubjectJobRemovalPurgeResponse {
|
||||||
|
workspace_id: request.removal.workspace_id,
|
||||||
|
worker_id: request.removal.worker_id,
|
||||||
|
operation_id: request.removal.operation_id,
|
||||||
|
input_fingerprint: request.removal.input_fingerprint,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn subject_delete_worker(
|
async fn subject_delete_worker(
|
||||||
&self,
|
&self,
|
||||||
request: SubjectDeleteWorkerRequest,
|
request: SubjectDeleteWorkerRequest,
|
||||||
@@ -1989,9 +2027,35 @@ mod tests {
|
|||||||
.subject_delete_worker(deletion.clone())
|
.subject_delete_worker(deletion.clone())
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let mut expected = serde_json::to_value(deletion).unwrap();
|
let mut expected = serde_json::to_value(&deletion).unwrap();
|
||||||
expected["originals_absent"] = true.into();
|
expected["originals_absent"] = true.into();
|
||||||
assert_eq!(serde_json::to_value(receipt).unwrap(), expected);
|
assert_eq!(serde_json::to_value(receipt).unwrap(), expected);
|
||||||
|
let purge = SubjectJobRemovalPurgeRequest {
|
||||||
|
removal: WorkerRetentionExecutionRequest {
|
||||||
|
operation_id: "old-removal".into(),
|
||||||
|
input_fingerprint: "old-removal-fingerprint".into(),
|
||||||
|
archive_id: Some("old-archive".into()),
|
||||||
|
workspace_id: deletion.workspace_id.clone(),
|
||||||
|
source_runtime_id: "runtime-a".into(),
|
||||||
|
worker_id: deletion.worker_id,
|
||||||
|
source_created_at: "2026-01-01T00:00:00Z".into(),
|
||||||
|
removed_at: "2026-01-02T00:00:00Z".into(),
|
||||||
|
effective_profile: None,
|
||||||
|
retention_class: None,
|
||||||
|
policy_id: "old-policy".into(),
|
||||||
|
session_disposition: SessionDisposition::Archive,
|
||||||
|
diagnostics_disposition: DiagnosticsDisposition::Retain,
|
||||||
|
},
|
||||||
|
deletion,
|
||||||
|
};
|
||||||
|
let receipt = client
|
||||||
|
.purge_subject_job_removal(purge.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(receipt.workspace_id, purge.removal.workspace_id);
|
||||||
|
assert_eq!(receipt.worker_id, purge.removal.worker_id);
|
||||||
|
assert_eq!(receipt.operation_id, purge.removal.operation_id);
|
||||||
|
assert_eq!(receipt.input_fingerprint, purge.removal.input_fingerprint);
|
||||||
let summary = client.runtime_summary().await.unwrap();
|
let summary = client.runtime_summary().await.unwrap();
|
||||||
assert_eq!(summary.runtime.display_name.as_deref(), Some("round-trip"));
|
assert_eq!(summary.runtime.display_name.as_deref(), Some("round-trip"));
|
||||||
let workers = client
|
let workers = client
|
||||||
|
|||||||
@@ -35,6 +35,14 @@ originals and pre-existing archives have independent retention lifecycles: the W
|
|||||||
Host archives source originals rather than erasing them. Those archives may contain
|
Host archives source originals rather than erasing them. Those archives may contain
|
||||||
previously injected Memory; Workdirs, Tickets, exports and external provider copies are
|
previously injected Memory; Workdirs, Tickets, exports and external provider copies are
|
||||||
also outside Subject content erasure. These boundaries must be explained to the user.
|
also outside Subject content erasure. These boundaries must be explained to the user.
|
||||||
|
An unfinished, previously authorized Worker retention operation is snapshotted separately
|
||||||
|
from prior successful removals. Its exact original request is reconciled under the permanent
|
||||||
|
fence; its disposition, fingerprint and canonical result are never rewritten. Body originals
|
||||||
|
follow that earlier independent request. For a Subject Job, any archive/diagnostic copies
|
||||||
|
produced by the unfinished operation require a distinct Runtime physical-erasure acknowledgement
|
||||||
|
before deletion can complete. Lost responses or failed receipt commits retain the same request
|
||||||
|
and incomplete state across restart. Successes that already existed at deletion start remain
|
||||||
|
under their independent archive lifecycle; a later success is not inserted into that snapshot.
|
||||||
The domain rechecks after out-of-DB capture callbacks and before returning Job read
|
The domain rechecks after out-of-DB capture callbacks and before returning Job read
|
||||||
responses; already returned DTOs cannot be revoked by SQLite. Host lifecycle locking
|
responses; already returned DTOs cannot be revoked by SQLite. Host lifecycle locking
|
||||||
must cover delivery/reinjection of previously fetched content as well as finish.
|
must cover delivery/reinjection of previously fetched content as well as finish.
|
||||||
|
|||||||
@@ -2104,7 +2104,9 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
|||||||
}
|
}
|
||||||
if matches!(
|
if matches!(
|
||||||
path,
|
path,
|
||||||
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
|
"/v1/subject-create-recovery"
|
||||||
|
| "/v1/subject-delete-worker"
|
||||||
|
| "/v1/subject-job-removal-purge"
|
||||||
) && *method == Method::POST
|
) && *method == Method::POST
|
||||||
{
|
{
|
||||||
return Some("workers:delete");
|
return Some("workers:delete");
|
||||||
|
|||||||
@@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn purge_subject_job_removal(
|
||||||
|
&self,
|
||||||
|
request: runtime_api::SubjectJobRemovalPurgeRequest,
|
||||||
|
) -> Result<runtime_api::SubjectJobRemovalPurgeResponse, runtime_api::RuntimeApiError> {
|
||||||
|
let scope = material_scope(&self.state, &request.deletion.workspace_id)?;
|
||||||
|
self.state
|
||||||
|
.runtime
|
||||||
|
.purge_subject_job_removal_scoped(&scope, request)
|
||||||
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
|
.map_err(api_error)
|
||||||
|
}
|
||||||
|
|
||||||
async fn subject_delete_worker(
|
async fn subject_delete_worker(
|
||||||
&self,
|
&self,
|
||||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
//! verifier are connected. Only controller execution is replaced by AcceptingBackend;
|
//! verifier are connected. Only controller execution is replaced by AcceptingBackend;
|
||||||
//! no model, external server, user credentials or running Runtime is used.
|
//! no model, external server, user credentials or running Runtime is used.
|
||||||
use super::*;
|
use super::*;
|
||||||
|
#[path = "subject_job_removal_purge_tests.rs"]
|
||||||
|
mod subject_job_removal_purge_tests;
|
||||||
use crate::retention::{DiagnosticsDisposition, SessionDisposition, runtime_session_retention};
|
use crate::retention::{DiagnosticsDisposition, SessionDisposition, runtime_session_retention};
|
||||||
use runtime_api::{
|
use runtime_api::{
|
||||||
SessionMaterialCatalogAvailability as Catalog, SessionMaterialReadAvailability as Read,
|
SessionMaterialCatalogAvailability as Catalog, SessionMaterialReadAvailability as Read,
|
||||||
|
|||||||
+117
@@ -0,0 +1,117 @@
|
|||||||
|
//! Real filesystem failures and reopened durable fences for the additional purge.
|
||||||
|
//! Uses the same production stores/router fixture as the copy-erasure suite.
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_job_removal_purge_partial_physical_failure_keeps_fence_and_retry_erases_after_reopen() {
|
||||||
|
let f = Fixture::new();
|
||||||
|
let mut request = f.removal_purge_request(SessionDisposition::Archive);
|
||||||
|
request.removal.diagnostics_disposition = runtime_api::DiagnosticsDisposition::Retain;
|
||||||
|
f.complete_removal(&request);
|
||||||
|
let receipt_path = f.receipt_path();
|
||||||
|
let receipt = std::fs::read(&receipt_path).unwrap();
|
||||||
|
let session_root = f
|
||||||
|
.temp
|
||||||
|
.path()
|
||||||
|
.join("archives/workers/canonical-archive/session");
|
||||||
|
let parked = f.temp.path().join("interrupted-session-fixture");
|
||||||
|
std::fs::rename(&session_root, &parked).unwrap();
|
||||||
|
std::fs::write(&session_root, b"deterministic directory removal failure").unwrap();
|
||||||
|
assert!(
|
||||||
|
f.runtime
|
||||||
|
.purge_subject_job_removal_scoped(&f.scope, request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
runtime_session_retention(f.temp.path()).ensure_live(f.session),
|
||||||
|
Err(session_store::StoreError::SessionErased(_))
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
!f.temp
|
||||||
|
.path()
|
||||||
|
.join("archives/diagnostics/remove-source")
|
||||||
|
.exists()
|
||||||
|
);
|
||||||
|
assert_eq!(std::fs::read(&receipt_path).unwrap(), receipt);
|
||||||
|
std::fs::remove_file(&session_root).unwrap();
|
||||||
|
std::fs::rename(parked, &session_root).unwrap();
|
||||||
|
let Fixture {
|
||||||
|
temp,
|
||||||
|
runtime,
|
||||||
|
scope,
|
||||||
|
store,
|
||||||
|
session,
|
||||||
|
..
|
||||||
|
} = f;
|
||||||
|
drop(store);
|
||||||
|
drop(runtime);
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
crate::FsRuntimeStoreOptions::new(temp.path()).with_runtime_id("runtime-test"),
|
||||||
|
Arc::new(AcceptingBackend),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("runtime-test").unwrap();
|
||||||
|
runtime
|
||||||
|
.purge_subject_job_removal_scoped(&scope, request)
|
||||||
|
.unwrap();
|
||||||
|
assert!(!session_root.exists());
|
||||||
|
assert!(matches!(
|
||||||
|
runtime_session_retention(temp.path()).lock_live(session),
|
||||||
|
Err(session_store::StoreError::SessionErased(_))
|
||||||
|
));
|
||||||
|
assert_eq!(std::fs::read(receipt_path).unwrap(), receipt);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_job_removal_purge_unconfirmed_durable_deletion_fence_preserves_copies_until_reconciled()
|
||||||
|
{
|
||||||
|
let f = Fixture::new();
|
||||||
|
let request = f.removal_purge_request(SessionDisposition::Archive);
|
||||||
|
f.complete_removal(&request);
|
||||||
|
let copies = f.copies();
|
||||||
|
let receipt_path = f.receipt_path();
|
||||||
|
let receipt = std::fs::read(&receipt_path).unwrap();
|
||||||
|
let Fixture {
|
||||||
|
temp,
|
||||||
|
runtime,
|
||||||
|
scope,
|
||||||
|
store,
|
||||||
|
..
|
||||||
|
} = f;
|
||||||
|
drop(store);
|
||||||
|
drop(runtime);
|
||||||
|
// Frozen durable crash-window fixture: permanent admission exclusion was
|
||||||
|
// saved, but no confirmed execution-shutdown response was committed.
|
||||||
|
let snapshot_path = temp.path().join("runtime.json");
|
||||||
|
let mut snapshot: serde_json::Value =
|
||||||
|
serde_json::from_slice(&std::fs::read(&snapshot_path).unwrap()).unwrap();
|
||||||
|
snapshot["subject_delete_worker_fences"][request.deletion.worker_id.to_string()]["response"] =
|
||||||
|
serde_json::Value::Null;
|
||||||
|
std::fs::write(&snapshot_path, serde_json::to_vec(&snapshot).unwrap()).unwrap();
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
crate::FsRuntimeStoreOptions::new(temp.path()).with_runtime_id("runtime-test"),
|
||||||
|
Arc::new(AcceptingBackend),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("runtime-test").unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.purge_subject_job_removal_scoped(&scope, request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(tree(&temp.path().join("archives")), copies);
|
||||||
|
assert_eq!(std::fs::read(&receipt_path).unwrap(), receipt);
|
||||||
|
runtime
|
||||||
|
.subject_delete_worker_scoped(&scope, request.deletion.clone())
|
||||||
|
.unwrap();
|
||||||
|
runtime
|
||||||
|
.purge_subject_job_removal_scoped(&scope, request)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!temp
|
||||||
|
.path()
|
||||||
|
.join("archives/workers/canonical-archive/session")
|
||||||
|
.exists()
|
||||||
|
);
|
||||||
|
assert_eq!(std::fs::read(receipt_path).unwrap(), receipt);
|
||||||
|
}
|
||||||
@@ -0,0 +1,486 @@
|
|||||||
|
//! Additional removal-copy erasure uses the real Runtime, retention provider,
|
||||||
|
//! Session tombstone, filesystem and signed HTTP router. Only execution is fake
|
||||||
|
//! (the parent fixture's AcceptingBackend); no Backend recovery is simulated here.
|
||||||
|
use super::*;
|
||||||
|
#[path = "subject_job_removal_purge_recovery_tests.rs"]
|
||||||
|
mod recovery;
|
||||||
|
use runtime_api::{SubjectDeleteWorkerRequest, SubjectJobRemovalPurgeRequest};
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
fn tree(root: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
|
||||||
|
fn walk(root: &Path, at: &Path, files: &mut BTreeMap<PathBuf, Vec<u8>>) {
|
||||||
|
if !at.exists() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for entry in std::fs::read_dir(at).unwrap() {
|
||||||
|
let entry = entry.unwrap();
|
||||||
|
if entry.file_type().unwrap().is_dir() {
|
||||||
|
walk(root, &entry.path(), files);
|
||||||
|
} else {
|
||||||
|
files.insert(
|
||||||
|
entry.path().strip_prefix(root).unwrap().into(),
|
||||||
|
std::fs::read(entry.path()).unwrap(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut files = BTreeMap::new();
|
||||||
|
walk(root, root, &mut files);
|
||||||
|
files
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Fixture {
|
||||||
|
fn deletion_request(&self) -> SubjectDeleteWorkerRequest {
|
||||||
|
SubjectDeleteWorkerRequest {
|
||||||
|
workspace_id: self.scope.workspace_id.clone(),
|
||||||
|
worker_id: self.worker.worker_id,
|
||||||
|
create_fingerprint: task_request("identity only").create_fingerprint,
|
||||||
|
operation_id: "subject-deletion".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn removal_purge_request(
|
||||||
|
&self,
|
||||||
|
disposition: SessionDisposition,
|
||||||
|
) -> SubjectJobRemovalPurgeRequest {
|
||||||
|
let removal = self.retention_request(disposition);
|
||||||
|
SubjectJobRemovalPurgeRequest {
|
||||||
|
deletion: self.deletion_request(),
|
||||||
|
removal: serde_json::from_value(serde_json::to_value(removal).unwrap()).unwrap(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn complete_removal(
|
||||||
|
&self,
|
||||||
|
request: &SubjectJobRemovalPurgeRequest,
|
||||||
|
) -> crate::retention::WorkerRetentionExecutionResult {
|
||||||
|
let diagnostics = self.root().parent().unwrap().join("runs/attempt");
|
||||||
|
std::fs::create_dir_all(&diagnostics).unwrap();
|
||||||
|
std::fs::write(diagnostics.join("worker.out.log"), b"Subject Job output").unwrap();
|
||||||
|
self.runtime
|
||||||
|
.subject_delete_worker_scoped(&self.scope, request.deletion.clone())
|
||||||
|
.unwrap();
|
||||||
|
let removal =
|
||||||
|
serde_json::from_value(serde_json::to_value(&request.removal).unwrap()).unwrap();
|
||||||
|
self.runtime.execute_worker_retention(&removal).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn receipt_path(&self) -> PathBuf {
|
||||||
|
self.temp
|
||||||
|
.path()
|
||||||
|
.join("retention/operations/remove-source.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copies(&self) -> BTreeMap<PathBuf, Vec<u8>> {
|
||||||
|
tree(&self.temp.path().join("archives"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn independent_worker(&self) -> WorkerRef {
|
||||||
|
let mut request = task_request("independent Worker");
|
||||||
|
request.config_bundle = None;
|
||||||
|
request.memory_settings.as_mut().unwrap().workspace_id = self.scope.workspace_id.clone();
|
||||||
|
self.runtime
|
||||||
|
.create_worker_scoped(&self.scope, request)
|
||||||
|
.unwrap()
|
||||||
|
.worker_ref
|
||||||
|
}
|
||||||
|
|
||||||
|
// Independent successful snapshot made before the Subject deletion starts.
|
||||||
|
fn prior_archive(&self) -> WorkerRef {
|
||||||
|
let worker = self.independent_worker();
|
||||||
|
let aggregate = self
|
||||||
|
.temp
|
||||||
|
.path()
|
||||||
|
.join("workers")
|
||||||
|
.join(worker.worker_id.to_string());
|
||||||
|
let session = session_store::new_session_id();
|
||||||
|
let segment = session_store::new_segment_id();
|
||||||
|
WorkerSessionStore::new(aggregate.join("session"))
|
||||||
|
.unwrap()
|
||||||
|
.create_segment(
|
||||||
|
session,
|
||||||
|
segment,
|
||||||
|
&[LogEntry::AnnotatedSegmentStart {
|
||||||
|
ts: 1,
|
||||||
|
session_id: session,
|
||||||
|
system_prompt: None,
|
||||||
|
config: Default::default(),
|
||||||
|
history: vec![],
|
||||||
|
forked_from: None,
|
||||||
|
compacted_from: None,
|
||||||
|
}],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
WorkerAggregateStore::new(&aggregate, format!("worker-runtime-{}", worker.worker_id))
|
||||||
|
.unwrap()
|
||||||
|
.write(
|
||||||
|
&session_store::WorkerMetadata::new(
|
||||||
|
format!("worker-runtime-{}", worker.worker_id),
|
||||||
|
Some(session_store::WorkerActiveSegmentRef::active_segment(
|
||||||
|
session, segment,
|
||||||
|
)),
|
||||||
|
)
|
||||||
|
.with_workspace_id(&self.scope.workspace_id),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
self.runtime
|
||||||
|
.stop_worker_scoped(&self.scope, &worker, None)
|
||||||
|
.unwrap();
|
||||||
|
let mut removal = self.retention_request(SessionDisposition::Archive);
|
||||||
|
removal.worker_id = worker.worker_id;
|
||||||
|
removal.operation_id = "prior-removal".into();
|
||||||
|
removal.archive_id = Some("prior-archive".into());
|
||||||
|
removal.input_fingerprint = "prior-fingerprint".into();
|
||||||
|
self.runtime.execute_worker_retention(&removal).unwrap();
|
||||||
|
worker
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_job_removal_purge_erases_copies_preserves_receipt_and_prior_success_after_reopen() {
|
||||||
|
for disposition in [SessionDisposition::Archive, SessionDisposition::Purge] {
|
||||||
|
let f = Fixture::new();
|
||||||
|
f.prior_archive();
|
||||||
|
let prior = tree(&f.temp.path().join("archives/workers/prior-archive"));
|
||||||
|
assert!(!prior.is_empty());
|
||||||
|
let unrelated = f.independent_worker();
|
||||||
|
let unrelated_root = f
|
||||||
|
.temp
|
||||||
|
.path()
|
||||||
|
.join("workers")
|
||||||
|
.join(unrelated.worker_id.to_string());
|
||||||
|
let unrelated_before = tree(&unrelated_root);
|
||||||
|
let mut request = f.removal_purge_request(disposition);
|
||||||
|
request.removal.diagnostics_disposition = runtime_api::DiagnosticsDisposition::Retain;
|
||||||
|
let original_result = f.complete_removal(&request);
|
||||||
|
let receipt = std::fs::read(f.receipt_path()).unwrap();
|
||||||
|
let archive = f.temp.path().join("archives/workers/canonical-archive");
|
||||||
|
let diagnostics = f.temp.path().join("archives/diagnostics/remove-source");
|
||||||
|
assert!(diagnostics.is_dir());
|
||||||
|
let manifest = (disposition == SessionDisposition::Archive)
|
||||||
|
.then(|| std::fs::read(archive.join("manifest.json")).unwrap());
|
||||||
|
let archive_staging = f
|
||||||
|
.temp
|
||||||
|
.path()
|
||||||
|
.join("archives/workers/.canonical-archive.tmp-123-1");
|
||||||
|
if let Some(manifest) = &manifest {
|
||||||
|
std::fs::create_dir_all(archive_staging.join("session/segments")).unwrap();
|
||||||
|
std::fs::write(archive_staging.join("manifest.json"), manifest).unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
archive_staging.join("session/segments/interrupted.jsonl"),
|
||||||
|
b"interrupted Subject Job copy",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
// Diagnostics copying precedes publication of its manifest. Interrupted
|
||||||
|
// staging is still bound to the exact removal operation name.
|
||||||
|
let diagnostics_staging = f
|
||||||
|
.temp
|
||||||
|
.path()
|
||||||
|
.join("archives/diagnostics/.remove-source.tmp-123-2");
|
||||||
|
std::fs::create_dir_all(&diagnostics_staging).unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
diagnostics_staging.join("worker.out.log"),
|
||||||
|
b"interrupted diagnostic copy",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for _ in 0..2 {
|
||||||
|
let response = f
|
||||||
|
.runtime
|
||||||
|
.purge_subject_job_removal_scoped(&f.scope, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.operation_id, request.removal.operation_id);
|
||||||
|
assert_eq!(
|
||||||
|
response.input_fingerprint,
|
||||||
|
request.removal.input_fingerprint
|
||||||
|
);
|
||||||
|
assert_eq!(response.worker_id, request.removal.worker_id);
|
||||||
|
assert_eq!(response.workspace_id, request.removal.workspace_id);
|
||||||
|
assert!(!diagnostics.exists());
|
||||||
|
assert!(!diagnostics_staging.exists());
|
||||||
|
assert!(!archive_staging.join("session").exists());
|
||||||
|
assert!(!archive.join("session").exists());
|
||||||
|
assert_eq!(std::fs::read(f.receipt_path()).unwrap(), receipt);
|
||||||
|
if let Some(manifest) = &manifest {
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(archive.join("manifest.json")).unwrap(),
|
||||||
|
*manifest
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
runtime_session_retention(f.temp.path()).ensure_live(f.session),
|
||||||
|
Err(session_store::StoreError::SessionErased(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
tree(&f.temp.path().join("archives/workers/prior-archive")),
|
||||||
|
prior
|
||||||
|
);
|
||||||
|
assert_eq!(tree(&unrelated_root), unrelated_before);
|
||||||
|
let Fixture {
|
||||||
|
temp,
|
||||||
|
runtime,
|
||||||
|
scope,
|
||||||
|
store,
|
||||||
|
session,
|
||||||
|
..
|
||||||
|
} = f;
|
||||||
|
drop(store);
|
||||||
|
drop(runtime);
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
crate::FsRuntimeStoreOptions::new(temp.path()).with_runtime_id("runtime-test"),
|
||||||
|
Arc::new(AcceptingBackend),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("runtime-test").unwrap();
|
||||||
|
// Lost response/restart is an exact retry, not a new erasure operation.
|
||||||
|
runtime
|
||||||
|
.purge_subject_job_removal_scoped(&scope, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
let removal =
|
||||||
|
serde_json::from_value(serde_json::to_value(&request.removal).unwrap()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
runtime.execute_worker_retention(&removal).unwrap(),
|
||||||
|
original_result
|
||||||
|
);
|
||||||
|
assert!(!archive.join("session").exists());
|
||||||
|
assert!(!diagnostics.exists());
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(temp.path().join("retention/operations/remove-source.json")).unwrap(),
|
||||||
|
receipt
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
tree(&temp.path().join("archives/workers/prior-archive")),
|
||||||
|
prior
|
||||||
|
);
|
||||||
|
assert_eq!(tree(&unrelated_root), unrelated_before);
|
||||||
|
if disposition == SessionDisposition::Archive {
|
||||||
|
assert!(matches!(
|
||||||
|
runtime_session_retention(temp.path()).lock_live(session),
|
||||||
|
Err(session_store::StoreError::SessionErased(_))
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.session_public_material_reader_scoped(
|
||||||
|
&scope,
|
||||||
|
&Source::Archived {
|
||||||
|
archive_id: "canonical-archive".into()
|
||||||
|
},
|
||||||
|
&session.to_string()
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_job_removal_purge_requires_complete_old_request_and_confirmed_exact_fence() {
|
||||||
|
let f = Fixture::new();
|
||||||
|
let request = f.removal_purge_request(SessionDisposition::Archive);
|
||||||
|
let copies = f.copies();
|
||||||
|
assert!(
|
||||||
|
f.runtime
|
||||||
|
.purge_subject_job_removal_scoped(&f.scope, request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(f.copies(), copies);
|
||||||
|
f.complete_removal(&request);
|
||||||
|
let copies = f.copies();
|
||||||
|
let receipt = std::fs::read(f.receipt_path()).unwrap();
|
||||||
|
let original = serde_json::to_value(&request).unwrap();
|
||||||
|
// Every saved request field is bound, not just operation/fingerprint. These
|
||||||
|
// are external serialized requests, including the pre-policy-change values.
|
||||||
|
for (section, field, value) in [
|
||||||
|
(
|
||||||
|
"deletion",
|
||||||
|
"workspace_id",
|
||||||
|
serde_json::json!("foreign-workspace"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"deletion",
|
||||||
|
"worker_id",
|
||||||
|
serde_json::json!(WorkerId::now_v7()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"deletion",
|
||||||
|
"create_fingerprint",
|
||||||
|
serde_json::json!("foreign-create"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"deletion",
|
||||||
|
"operation_id",
|
||||||
|
serde_json::json!("foreign-deletion"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"removal",
|
||||||
|
"operation_id",
|
||||||
|
serde_json::json!("unknown-removal"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"removal",
|
||||||
|
"input_fingerprint",
|
||||||
|
serde_json::json!("foreign-fingerprint"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"removal",
|
||||||
|
"archive_id",
|
||||||
|
serde_json::json!("foreign-archive"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"removal",
|
||||||
|
"workspace_id",
|
||||||
|
serde_json::json!("foreign-workspace"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"removal",
|
||||||
|
"source_runtime_id",
|
||||||
|
serde_json::json!("foreign-runtime"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"removal",
|
||||||
|
"worker_id",
|
||||||
|
serde_json::json!(WorkerId::now_v7()),
|
||||||
|
),
|
||||||
|
("removal", "source_created_at", serde_json::json!("changed")),
|
||||||
|
("removal", "removed_at", serde_json::json!("changed")),
|
||||||
|
("removal", "effective_profile", serde_json::json!("changed")),
|
||||||
|
("removal", "retention_class", serde_json::json!("changed")),
|
||||||
|
("removal", "policy_id", serde_json::json!("new-policy")),
|
||||||
|
("removal", "session_disposition", serde_json::json!("purge")),
|
||||||
|
(
|
||||||
|
"removal",
|
||||||
|
"diagnostics_disposition",
|
||||||
|
serde_json::json!("retain"),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let mut changed = original.clone();
|
||||||
|
changed[section][field] = value;
|
||||||
|
let changed = serde_json::from_value(changed).unwrap();
|
||||||
|
assert!(
|
||||||
|
f.runtime
|
||||||
|
.purge_subject_job_removal_scoped(&f.scope, changed)
|
||||||
|
.is_err(),
|
||||||
|
"{section}.{field}"
|
||||||
|
);
|
||||||
|
assert_eq!(f.copies(), copies, "{section}.{field}");
|
||||||
|
assert_eq!(std::fs::read(f.receipt_path()).unwrap(), receipt);
|
||||||
|
}
|
||||||
|
for scope in [
|
||||||
|
RuntimeWorkspaceScope::new("workspace-b", "https://backend.test"),
|
||||||
|
RuntimeWorkspaceScope::new("workspace-a", "https://other-backend.test"),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
f.runtime
|
||||||
|
.purge_subject_job_removal_scoped(&scope, request.clone())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(f.copies(), copies);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn subject_job_removal_purge_missing_pending_or_foreign_receipt_preserves_copies() {
|
||||||
|
for kind in [
|
||||||
|
"missing",
|
||||||
|
"pending",
|
||||||
|
"foreign-worker",
|
||||||
|
"foreign-disposition",
|
||||||
|
] {
|
||||||
|
let f = Fixture::new();
|
||||||
|
let mut request = f.removal_purge_request(SessionDisposition::Archive);
|
||||||
|
request.removal.diagnostics_disposition = runtime_api::DiagnosticsDisposition::Retain;
|
||||||
|
f.complete_removal(&request);
|
||||||
|
let path = f.receipt_path();
|
||||||
|
let mut receipt: serde_json::Value =
|
||||||
|
serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
|
||||||
|
match kind {
|
||||||
|
"missing" => std::fs::remove_file(&path).unwrap(),
|
||||||
|
_ => {
|
||||||
|
match kind {
|
||||||
|
"pending" => receipt["result"]["source_removed"] = false.into(),
|
||||||
|
"foreign-worker" => {
|
||||||
|
receipt["result"]["worker_id"] = serde_json::json!(WorkerId::now_v7())
|
||||||
|
}
|
||||||
|
_ => receipt["result"]["session_disposition"] = "purge".into(),
|
||||||
|
}
|
||||||
|
std::fs::write(&path, serde_json::to_vec(&receipt).unwrap()).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let copies = f.copies();
|
||||||
|
let retained_receipt = std::fs::read(&path).ok();
|
||||||
|
assert!(
|
||||||
|
f.runtime
|
||||||
|
.purge_subject_job_removal_scoped(&f.scope, request)
|
||||||
|
.is_err(),
|
||||||
|
"{kind}"
|
||||||
|
);
|
||||||
|
assert_eq!(f.copies(), copies, "{kind}");
|
||||||
|
assert_eq!(std::fs::read(path).ok(), retained_receipt);
|
||||||
|
runtime_session_retention(f.temp.path())
|
||||||
|
.ensure_live(f.session)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn subject_job_removal_purge_signed_http_requires_delete_and_matching_scope() {
|
||||||
|
let f = Fixture::new();
|
||||||
|
let mut request = f.removal_purge_request(SessionDisposition::Archive);
|
||||||
|
request.removal.diagnostics_disposition = runtime_api::DiagnosticsDisposition::Retain;
|
||||||
|
f.complete_removal(&request);
|
||||||
|
let identity = RuntimeIdentityMaterial::generate("purge-key").unwrap();
|
||||||
|
let trust = workspace_trust(&identity);
|
||||||
|
let app = runtime_http_router_with_workspace_auth(
|
||||||
|
f.runtime.clone(),
|
||||||
|
None,
|
||||||
|
workspace_test_auth(vec![trust.clone()]),
|
||||||
|
);
|
||||||
|
let path = "/v1/subject-job-removal-purge";
|
||||||
|
let copies = f.copies();
|
||||||
|
for operation in ["workers:read", "workers:stop", "workers:create"] {
|
||||||
|
let body = serde_json::to_vec(&request).unwrap();
|
||||||
|
let mut claims = signed_claims(&trust, "POST", path, &body);
|
||||||
|
claims.operation = operation.into();
|
||||||
|
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
assert_eq!(f.copies(), copies);
|
||||||
|
}
|
||||||
|
let mut foreign = request.clone();
|
||||||
|
foreign.deletion.workspace_id = "foreign-workspace".into();
|
||||||
|
let body = serde_json::to_vec(&foreign).unwrap();
|
||||||
|
let claims = signed_claims(&trust, "POST", path, &body);
|
||||||
|
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||||
|
assert_eq!(f.copies(), copies);
|
||||||
|
for _ in 0..2 {
|
||||||
|
let body = serde_json::to_vec(&request).unwrap();
|
||||||
|
let claims = signed_claims(&trust, "POST", path, &body);
|
||||||
|
assert_eq!(claims.operation, "workers:delete");
|
||||||
|
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let bytes = axum::body::to_bytes(response.into_body(), 16384)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let response: runtime_api::SubjectJobRemovalPurgeResponse =
|
||||||
|
serde_json::from_slice(&bytes).unwrap();
|
||||||
|
assert_eq!(response.operation_id, request.removal.operation_id);
|
||||||
|
assert_eq!(
|
||||||
|
response.input_fingerprint,
|
||||||
|
request.removal.input_fingerprint
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!f.temp
|
||||||
|
.path()
|
||||||
|
.join("archives/workers/canonical-archive/session")
|
||||||
|
.exists()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!f.temp
|
||||||
|
.path()
|
||||||
|
.join("archives/diagnostics/remove-source")
|
||||||
|
.exists()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -359,28 +359,91 @@ fn erase_archived_diagnostics(
|
|||||||
return Err(invalid());
|
return Err(invalid());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let parent = runtime_root.join("archives/diagnostics");
|
erase_operation_diagnostics(
|
||||||
let entries = match fs::read_dir(&parent) {
|
runtime_root,
|
||||||
Ok(entries) => entries,
|
&archive.workspace_id,
|
||||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
&archive.source_runtime_id,
|
||||||
Err(error) => return Err(error.into()),
|
archive.source_worker_id,
|
||||||
};
|
&archive.operation_id,
|
||||||
let staging_prefix = format!(".{}.tmp-", archive.operation_id);
|
&archive.input_fingerprint,
|
||||||
for entry in entries {
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn erase_removal_session_copies(
|
||||||
|
runtime_root: &Path,
|
||||||
|
archive: &WorkerSessionArchiveManifest,
|
||||||
|
) -> Result<(), session_store::StoreError> {
|
||||||
|
let parent = runtime_root.join("archives/workers");
|
||||||
|
let prefix = format!(".{}.tmp-", archive.archive_id);
|
||||||
|
for entry in fs::read_dir(&parent)? {
|
||||||
let entry = entry?;
|
let entry = entry?;
|
||||||
let name = entry.file_name();
|
let name = entry.file_name();
|
||||||
let Some(name) = name.to_str() else {
|
let Some(name) = name.to_str() else { continue };
|
||||||
|
if name != archive.archive_id && !is_operation_staging(name, &prefix) {
|
||||||
continue;
|
continue;
|
||||||
};
|
}
|
||||||
let staged = name.strip_prefix(&staging_prefix).is_some_and(|suffix| {
|
if !entry.file_type()?.is_dir() {
|
||||||
|
return Err(std::io::Error::other("Removal archive source is not a directory").into());
|
||||||
|
}
|
||||||
|
let mut saved: WorkerSessionArchiveManifest =
|
||||||
|
serde_json::from_slice(&fs::read(entry.path().join("manifest.json"))?)?;
|
||||||
|
// Each interrupted attempt attests the same input, with its own time.
|
||||||
|
saved.archived_at_unix_seconds = archive.archived_at_unix_seconds;
|
||||||
|
if saved != *archive {
|
||||||
|
return Err(std::io::Error::other("Removal archive copy identity mismatch").into());
|
||||||
|
}
|
||||||
|
match fs::remove_dir_all(entry.path().join("session")) {
|
||||||
|
Ok(()) => (),
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => (),
|
||||||
|
Err(error) => return Err(error.into()),
|
||||||
|
}
|
||||||
|
sync_directory(&entry.path(), "erase removal Session")
|
||||||
|
.map_err(|error| std::io::Error::other(error.to_string()))?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_operation_staging(name: &str, prefix: &str) -> bool {
|
||||||
|
name.strip_prefix(prefix).is_some_and(|suffix| {
|
||||||
suffix.split_once('-').is_some_and(|(pid, sequence)| {
|
suffix.split_once('-').is_some_and(|(pid, sequence)| {
|
||||||
!pid.is_empty()
|
!pid.is_empty()
|
||||||
&& !sequence.is_empty()
|
&& !sequence.is_empty()
|
||||||
&& pid.bytes().all(|b| b.is_ascii_digit())
|
&& pid.bytes().all(|b| b.is_ascii_digit())
|
||||||
&& sequence.bytes().all(|b| b.is_ascii_digit())
|
&& sequence.bytes().all(|b| b.is_ascii_digit())
|
||||||
})
|
})
|
||||||
});
|
})
|
||||||
if name != archive.operation_id && !staged {
|
}
|
||||||
|
|
||||||
|
fn erase_operation_diagnostics(
|
||||||
|
runtime_root: &Path,
|
||||||
|
workspace_id: &str,
|
||||||
|
source_runtime_id: &str,
|
||||||
|
worker_id: WorkerId,
|
||||||
|
operation_id: &str,
|
||||||
|
input_fingerprint: &str,
|
||||||
|
) -> Result<(), session_store::StoreError> {
|
||||||
|
let invalid = || {
|
||||||
|
session_store::StoreError::Io(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidData,
|
||||||
|
"Archived diagnostics erasure identity mismatch",
|
||||||
|
))
|
||||||
|
};
|
||||||
|
validate_id("operation_id", operation_id).map_err(|_| invalid())?;
|
||||||
|
let parent = runtime_root.join("archives/diagnostics");
|
||||||
|
let entries = match fs::read_dir(&parent) {
|
||||||
|
Ok(entries) => entries,
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||||
|
Err(error) => return Err(error.into()),
|
||||||
|
};
|
||||||
|
let staging_prefix = format!(".{operation_id}.tmp-");
|
||||||
|
for entry in entries {
|
||||||
|
let entry = entry?;
|
||||||
|
let name = entry.file_name();
|
||||||
|
let Some(name) = name.to_str() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let staged = is_operation_staging(name, &staging_prefix);
|
||||||
|
if name != operation_id && !staged {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if !entry.file_type()?.is_dir() {
|
if !entry.file_type()?.is_dir() {
|
||||||
@@ -395,11 +458,11 @@ fn erase_archived_diagnostics(
|
|||||||
};
|
};
|
||||||
if let Some(manifest) = manifest {
|
if let Some(manifest) = manifest {
|
||||||
if manifest.schema_version != ARCHIVE_SCHEMA_VERSION
|
if manifest.schema_version != ARCHIVE_SCHEMA_VERSION
|
||||||
|| manifest.workspace_id != archive.workspace_id
|
|| manifest.workspace_id != workspace_id
|
||||||
|| manifest.source_runtime_id != archive.source_runtime_id
|
|| manifest.source_runtime_id != source_runtime_id
|
||||||
|| manifest.source_worker_id != archive.source_worker_id
|
|| manifest.source_worker_id != worker_id
|
||||||
|| manifest.operation_id != archive.operation_id
|
|| manifest.operation_id != operation_id
|
||||||
|| manifest.input_fingerprint != archive.input_fingerprint
|
|| manifest.input_fingerprint != input_fingerprint
|
||||||
{
|
{
|
||||||
return Err(invalid());
|
return Err(invalid());
|
||||||
}
|
}
|
||||||
@@ -480,6 +543,109 @@ impl FsWorkerRetentionProvider {
|
|||||||
Ok(receipt.result.source_removed.then_some(receipt.result))
|
Ok(receipt.result.source_removed.then_some(receipt.result))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Caller holds the Worker operation lock and has verified the durable
|
||||||
|
/// Subject deletion fence. Never execute/recover a missing or pending receipt
|
||||||
|
/// here: the original canonical operation must have completed first.
|
||||||
|
pub(crate) fn purge_subject_job_removal(
|
||||||
|
&self,
|
||||||
|
request: &WorkerRetentionExecutionRequest,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
validate_request(request)?;
|
||||||
|
let lock_path = self.runtime_root.join(RETENTION_LOCK);
|
||||||
|
let lock = OpenOptions::new()
|
||||||
|
.create(true)
|
||||||
|
.read(true)
|
||||||
|
.write(true)
|
||||||
|
.open(&lock_path)
|
||||||
|
.map_err(|source| RuntimeError::StoreIo {
|
||||||
|
operation: "lock Subject Job removal purge",
|
||||||
|
path: lock_path.clone(),
|
||||||
|
source,
|
||||||
|
})?;
|
||||||
|
lock.lock().map_err(|source| RuntimeError::StoreIo {
|
||||||
|
operation: "lock Subject Job removal purge",
|
||||||
|
path: lock_path,
|
||||||
|
source,
|
||||||
|
})?;
|
||||||
|
let receipt = read_operation_receipt(&self.operation_path(&request.operation_id)?)?
|
||||||
|
.ok_or_else(|| {
|
||||||
|
RuntimeError::InvalidRequest("Canonical removal receipt missing".into())
|
||||||
|
})?;
|
||||||
|
validate_receipt_request(&receipt, request)?;
|
||||||
|
let result = &receipt.result;
|
||||||
|
if !result.source_removed
|
||||||
|
|| result.worker_id != request.worker_id
|
||||||
|
|| result.session_disposition != request.session_disposition
|
||||||
|
|| result.diagnostics_disposition != request.diagnostics_disposition
|
||||||
|
|| result.diagnostics_retained
|
||||||
|
!= (request.diagnostics_disposition == DiagnosticsDisposition::Retain)
|
||||||
|
|| result.archive.is_some()
|
||||||
|
!= (request.session_disposition == SessionDisposition::Archive)
|
||||||
|
|| self.worker_dir(request.worker_id).exists()
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Canonical removal is not an exact successful receipt".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let archive = if let Some(manifest) = &result.archive {
|
||||||
|
if request.archive_id.as_ref() != Some(&manifest.archive_id)
|
||||||
|
|| manifest.workspace_id != request.workspace_id
|
||||||
|
|| manifest.source_runtime_id != request.source_runtime_id
|
||||||
|
|| manifest.source_worker_id != request.worker_id
|
||||||
|
|| manifest.operation_id != request.operation_id
|
||||||
|
|| manifest.input_fingerprint != request.input_fingerprint
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Removal archive identity mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let (saved, _) = self.session_archive_identity(&manifest.archive_id)?;
|
||||||
|
if saved != *manifest {
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Removal archive receipt mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let session = manifest
|
||||||
|
.source_session_id
|
||||||
|
.parse::<session_store::SessionId>()
|
||||||
|
.map_err(|_| {
|
||||||
|
RuntimeError::InvalidRequest("Invalid removal Session identity".into())
|
||||||
|
})?;
|
||||||
|
Some(session)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let erase = || -> Result<(), session_store::StoreError> {
|
||||||
|
erase_operation_diagnostics(
|
||||||
|
&self.runtime_root,
|
||||||
|
&request.workspace_id,
|
||||||
|
&request.source_runtime_id,
|
||||||
|
request.worker_id,
|
||||||
|
&request.operation_id,
|
||||||
|
&request.input_fingerprint,
|
||||||
|
)?;
|
||||||
|
if let Some(manifest) = &result.archive {
|
||||||
|
erase_removal_session_copies(&self.runtime_root, manifest)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
};
|
||||||
|
if let Some(session) = &archive {
|
||||||
|
runtime_session_retention(&self.runtime_root)
|
||||||
|
.delete(
|
||||||
|
*session,
|
||||||
|
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||||
|
erase,
|
||||||
|
)
|
||||||
|
.map_err(retention_error)?;
|
||||||
|
} else {
|
||||||
|
// Diagnostics-only old Purge operations have no archive Session ID.
|
||||||
|
// The existing permanent Worker fence plus immutable completed receipt
|
||||||
|
// prevents execution and re-publication; hold the retention lock here.
|
||||||
|
erase().map_err(retention_error)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn snapshot(
|
pub(crate) fn snapshot(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
|
|||||||
@@ -10,6 +10,79 @@ pub(crate) struct SubjectDeleteWorkerFence {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Runtime {
|
impl Runtime {
|
||||||
|
/// Erase only copies bound to an exact, completed canonical removal. The
|
||||||
|
/// Subject fence already excludes execution; this does not mint erase authority
|
||||||
|
/// for an arbitrary Worker or reinterpret the historical retention result.
|
||||||
|
pub fn purge_subject_job_removal_scoped(
|
||||||
|
&self,
|
||||||
|
scope: &RuntimeWorkspaceScope,
|
||||||
|
request: runtime_api::SubjectJobRemovalPurgeRequest,
|
||||||
|
) -> Result<runtime_api::SubjectJobRemovalPurgeResponse, RuntimeError> {
|
||||||
|
let deletion = &request.deletion;
|
||||||
|
let removal = &request.removal;
|
||||||
|
if deletion.workspace_id != scope.workspace_id
|
||||||
|
|| removal.workspace_id != deletion.workspace_id
|
||||||
|
|| removal.worker_id != deletion.worker_id
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Subject Job removal scope mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let lease = self.worker_operation_lock(deletion.worker_id)?;
|
||||||
|
let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.ensure_workspace_owner(scope, false)?;
|
||||||
|
if state.runtime_identity.as_deref() != Some(removal.source_runtime_id.as_str()) {
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Subject Job removal Runtime identity mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state
|
||||||
|
.subject_delete_worker_fences
|
||||||
|
.get(&deletion.worker_id)
|
||||||
|
.filter(|fence| fence.request == *deletion && fence.response.is_some())
|
||||||
|
.ok_or_else(|| {
|
||||||
|
RuntimeError::InvalidRequest(
|
||||||
|
"Exact confirmed Subject deletion fence required".into(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
if state.workers.contains_key(&deletion.worker_id)
|
||||||
|
|| state
|
||||||
|
.unreconciled_worker_writes
|
||||||
|
.contains(&deletion.worker_id)
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"Subject Job canonical removal must be reconciled first".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
{
|
||||||
|
let store = state.fs_store().ok_or_else(|| {
|
||||||
|
RuntimeError::InvalidRequest("Subject Job purge requires filesystem storage".into())
|
||||||
|
})?;
|
||||||
|
let removal: WorkerRetentionExecutionRequest = serde_json::from_value(
|
||||||
|
serde_json::to_value(removal)
|
||||||
|
.map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?,
|
||||||
|
)
|
||||||
|
.map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?;
|
||||||
|
// Worker -> Runtime -> retention -> Session is the canonical removal
|
||||||
|
// lock order. The provider validates the receipt under its disk lock.
|
||||||
|
FsWorkerRetentionProvider::new(store.runtime_dir())
|
||||||
|
.purge_subject_job_removal(&removal)?;
|
||||||
|
Ok(runtime_api::SubjectJobRemovalPurgeResponse {
|
||||||
|
workspace_id: removal.workspace_id,
|
||||||
|
worker_id: removal.worker_id,
|
||||||
|
operation_id: removal.operation_id,
|
||||||
|
input_fingerprint: removal.input_fingerprint,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
Err(RuntimeError::InvalidRequest(
|
||||||
|
"Subject Job purge requires filesystem storage".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn subject_delete_worker_scoped(
|
pub fn subject_delete_worker_scoped(
|
||||||
&self,
|
&self,
|
||||||
scope: &RuntimeWorkspaceScope,
|
scope: &RuntimeWorkspaceScope,
|
||||||
|
|||||||
@@ -1292,6 +1292,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn purge_subject_job_removal(
|
||||||
|
&self,
|
||||||
|
_request: runtime_api::SubjectJobRemovalPurgeRequest,
|
||||||
|
) -> Result<runtime_api::SubjectJobRemovalPurgeResponse, String> {
|
||||||
|
Err("Subject Job removal source erasure is unavailable on this Runtime".into())
|
||||||
|
}
|
||||||
|
|
||||||
fn subject_delete_worker(
|
fn subject_delete_worker(
|
||||||
&self,
|
&self,
|
||||||
_request: runtime_api::SubjectDeleteWorkerRequest,
|
_request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
@@ -2276,6 +2283,32 @@ impl RuntimeRegistry {
|
|||||||
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
|
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn purge_subject_job_removal(
|
||||||
|
&self,
|
||||||
|
runtime_id: &str,
|
||||||
|
deletion: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
removal: WorkerRetentionExecutionRequest,
|
||||||
|
) -> Result<runtime_api::SubjectJobRemovalPurgeResponse, RuntimeRegistryError> {
|
||||||
|
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||||
|
let removal = runtime_contract_convert(removal).map_err(|diagnostic| {
|
||||||
|
RuntimeRegistryError::RuntimeOperationFailed {
|
||||||
|
runtime_id: runtime_id.into(),
|
||||||
|
code: "subject_job_removal_request_invalid".into(),
|
||||||
|
message: diagnostic.message,
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
self.runtime(runtime_id)?
|
||||||
|
.purge_subject_job_removal(runtime_api::SubjectJobRemovalPurgeRequest {
|
||||||
|
deletion,
|
||||||
|
removal,
|
||||||
|
})
|
||||||
|
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
|
||||||
|
runtime_id: runtime_id.into(),
|
||||||
|
code: "subject_job_removal_purge_unconfirmed".into(),
|
||||||
|
message,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn subject_delete_worker(
|
pub fn subject_delete_worker(
|
||||||
&self,
|
&self,
|
||||||
runtime_id: &str,
|
runtime_id: &str,
|
||||||
@@ -3402,6 +3435,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
|||||||
.map_err(|e| e.to_string())
|
.map_err(|e| e.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn purge_subject_job_removal(
|
||||||
|
&self,
|
||||||
|
request: runtime_api::SubjectJobRemovalPurgeRequest,
|
||||||
|
) -> Result<runtime_api::SubjectJobRemovalPurgeResponse, String> {
|
||||||
|
if request.deletion.workspace_id != self.workspace_id {
|
||||||
|
return Err("Subject Job removal erasure Workspace mismatch".into());
|
||||||
|
}
|
||||||
|
let scope = RuntimeWorkspaceScope::new(&request.deletion.workspace_id, "embedded-backend");
|
||||||
|
self.runtime
|
||||||
|
.purge_subject_job_removal_scoped(&scope, request)
|
||||||
|
.map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
fn recover_subject_create(
|
fn recover_subject_create(
|
||||||
&self,
|
&self,
|
||||||
request: runtime_api::SubjectCreateRecoveryRequest,
|
request: runtime_api::SubjectCreateRecoveryRequest,
|
||||||
@@ -5694,6 +5740,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn purge_subject_job_removal(
|
||||||
|
&self,
|
||||||
|
request: runtime_api::SubjectJobRemovalPurgeRequest,
|
||||||
|
) -> Result<runtime_api::SubjectJobRemovalPurgeResponse, String> {
|
||||||
|
self.run_runtime_api(
|
||||||
|
self.request_timeout,
|
||||||
|
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||||
|
move |client| async move { client.purge_subject_job_removal(request).await },
|
||||||
|
)
|
||||||
|
.map_err(|diagnostic| diagnostic.message)
|
||||||
|
}
|
||||||
|
|
||||||
fn subject_delete_worker(
|
fn subject_delete_worker(
|
||||||
&self,
|
&self,
|
||||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||||
|
|||||||
@@ -407,7 +407,8 @@ fn reconstruct_implementation_schema96(conn: &Connection) -> Result<()> {
|
|||||||
}
|
}
|
||||||
// Children before parents; dropping the tables also drops snapshot guards.
|
// Children before parents; dropping the tables also drops snapshot guards.
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
"DROP TABLE subject_deletion_allocations;
|
"DROP TABLE subject_deletion_removals;
|
||||||
|
DROP TABLE subject_deletion_allocations;
|
||||||
DROP TABLE subject_deletion_jobs;
|
DROP TABLE subject_deletion_jobs;
|
||||||
DROP TABLE subject_deletion_workers;
|
DROP TABLE subject_deletion_workers;
|
||||||
DROP TABLE subject_deletions;
|
DROP TABLE subject_deletions;
|
||||||
|
|||||||
@@ -333,6 +333,8 @@ impl SqliteWorkspaceStore {
|
|||||||
return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()}));
|
return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()}));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let pending:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_removals WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND completed=0)",params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|row|row.get(0))?;
|
||||||
|
if pending {return Err(StoreError::InvalidInput("Pinned pre-deletion removal must reconcile before another retention plan".into()));}
|
||||||
if crate::store::subject_deletion_worker_kind(&tx, &req.workspace_id, &req.worker)?.is_some() {
|
if crate::store::subject_deletion_worker_kind(&tx, &req.workspace_id, &req.worker)?.is_some() {
|
||||||
let prior = tx.prepare("SELECT plan_id FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('executing','failed')")?
|
let prior = tx.prepare("SELECT plan_id FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('executing','failed')")?
|
||||||
.query_map(params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|r.get::<_,String>(0))?
|
.query_map(params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|r.get::<_,String>(0))?
|
||||||
@@ -381,23 +383,26 @@ impl SqliteWorkspaceStore {
|
|||||||
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
|
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
|
||||||
}
|
}
|
||||||
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
|
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
|
||||||
|
let reconciliation=crate::store::subject_deletion_removal_matches(&tx,workspace_id,&plan.worker,&plan.operation_id,&plan.input_fingerprint)?;
|
||||||
|
if !reconciliation {
|
||||||
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
|
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
|
||||||
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
|
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
|
||||||
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
|
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
|
||||||
return Err(stale_error(&plan,"policy content changed"));
|
return Err(stale_error(&plan,"policy content changed"));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
let worker=load_worker(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:Worker missing")))?;
|
let worker=load_worker(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:Worker missing")))?;
|
||||||
if worker.updated_at!=plan.worker_updated_at{
|
if !reconciliation && worker.updated_at!=plan.worker_updated_at{
|
||||||
mark_stale(&tx,&plan,"Worker updated_at changed")?;tx.commit()?;
|
mark_stale(&tx,&plan,"Worker updated_at changed")?;tx.commit()?;
|
||||||
return Err(stale_error(&plan,"Worker updated_at changed"));
|
return Err(stale_error(&plan,"Worker updated_at changed"));
|
||||||
}
|
}
|
||||||
if worker.retention_state=="pinned"{
|
if worker.retention_state=="pinned"{
|
||||||
mark_stale(&tx,&plan,"hold added")?;tx.commit()?;
|
if !reconciliation {mark_stale(&tx,&plan,"hold added")?;}tx.commit()?;
|
||||||
return Err(stale_error(&plan,"hold added"));
|
return Err(stale_error(&plan,"hold added"));
|
||||||
}
|
}
|
||||||
let assigned:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3 UNION ALL SELECT 1 FROM ticket_assignment_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND claim_state='pending' AND action IN ('assign','reassign'))",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id],|r|r.get(0))?;
|
let assigned:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3 UNION ALL SELECT 1 FROM ticket_assignment_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND claim_state='pending' AND action IN ('assign','reassign'))",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id],|r|r.get(0))?;
|
||||||
if assigned{
|
if assigned{
|
||||||
mark_stale(&tx,&plan,"unfinished work added")?;tx.commit()?;
|
if !reconciliation {mark_stale(&tx,&plan,"unfinished work added")?;}tx.commit()?;
|
||||||
return Err(stale_error(&plan,"unfinished work added"));
|
return Err(stale_error(&plan,"unfinished work added"));
|
||||||
}
|
}
|
||||||
let now=Utc::now().to_rfc3339();
|
let now=Utc::now().to_rfc3339();
|
||||||
@@ -416,6 +421,15 @@ impl SqliteWorkspaceStore {
|
|||||||
input_fingerprint: &str,
|
input_fingerprint: &str,
|
||||||
) -> Result<PreparedWorkerRemoval, WorkerRetentionError> {
|
) -> Result<PreparedWorkerRemoval, WorkerRetentionError> {
|
||||||
let plan = self.begin_worker_removal(workspace_id, plan_id, input_fingerprint)?;
|
let plan = self.begin_worker_removal(workspace_id, plan_id, input_fingerprint)?;
|
||||||
|
if let Some(reconciliation) = self.subject_deletion_removal(workspace_id, &plan.worker)?
|
||||||
|
&& reconciliation.request.operation_id == plan.operation_id
|
||||||
|
{
|
||||||
|
return Ok(PreparedWorkerRemoval {
|
||||||
|
runtime_request: reconciliation.request,
|
||||||
|
prior_failure_category: plan.failure_category.clone(),
|
||||||
|
plan,
|
||||||
|
});
|
||||||
|
}
|
||||||
let worker = self
|
let worker = self
|
||||||
.with_conn(|conn| load_worker(conn, workspace_id, &plan.worker))?
|
.with_conn(|conn| load_worker(conn, workspace_id, &plan.worker))?
|
||||||
.ok_or_else(|| WorkerRetentionError::StalePlan {
|
.ok_or_else(|| WorkerRetentionError::StalePlan {
|
||||||
@@ -466,12 +480,18 @@ impl SqliteWorkspaceStore {
|
|||||||
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
|
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
|
||||||
AND state IN ('planned','executing','failed','succeeded')
|
AND state IN ('planned','executing','failed','succeeded')
|
||||||
AND (
|
AND (
|
||||||
state='succeeded' OR worker_updated_at=(
|
state='succeeded' OR EXISTS(
|
||||||
|
SELECT 1 FROM subject_deletion_removals d
|
||||||
|
WHERE d.workspace_id=worker_removal_operations.workspace_id
|
||||||
|
AND d.runtime_id=worker_removal_operations.runtime_id
|
||||||
|
AND d.worker_id=worker_removal_operations.worker_id
|
||||||
|
AND d.operation_id=worker_removal_operations.operation_id
|
||||||
|
) OR worker_updated_at=(
|
||||||
SELECT updated_at FROM worker_registry
|
SELECT updated_at FROM worker_registry
|
||||||
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
|
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
ORDER BY CASE WHEN EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=worker_removal_operations.workspace_id AND d.runtime_id=worker_removal_operations.runtime_id AND d.worker_id=worker_removal_operations.worker_id AND d.prior_removal_operation_id=worker_removal_operations.operation_id) THEN 0 WHEN state='succeeded' THEN 1 ELSE 2 END,
|
ORDER BY CASE WHEN EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=worker_removal_operations.workspace_id AND d.runtime_id=worker_removal_operations.runtime_id AND d.worker_id=worker_removal_operations.worker_id AND d.prior_removal_operation_id=worker_removal_operations.operation_id) THEN 0 WHEN EXISTS(SELECT 1 FROM subject_deletion_removals d WHERE d.workspace_id=worker_removal_operations.workspace_id AND d.runtime_id=worker_removal_operations.runtime_id AND d.worker_id=worker_removal_operations.worker_id AND d.operation_id=worker_removal_operations.operation_id) THEN 1 WHEN state='succeeded' THEN 2 ELSE 3 END,
|
||||||
created_at DESC,operation_id DESC LIMIT 1",
|
created_at DESC,operation_id DESC LIMIT 1",
|
||||||
params![workspace_id, worker.runtime_id, worker.worker_id],
|
params![workspace_id, worker.runtime_id, worker.worker_id],
|
||||||
|row| row.get::<_, String>(0),
|
|row| row.get::<_, String>(0),
|
||||||
@@ -489,6 +509,15 @@ impl SqliteWorkspaceStore {
|
|||||||
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
|
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
|
||||||
.map_err(map_error)?;
|
.map_err(map_error)?;
|
||||||
let prior_failure_category = plan.failure_category.clone();
|
let prior_failure_category = plan.failure_category.clone();
|
||||||
|
if let Some(reconciliation) = self.subject_deletion_removal(workspace_id, &plan.worker)?
|
||||||
|
&& reconciliation.request.operation_id == plan.operation_id
|
||||||
|
{
|
||||||
|
return Ok(Some(PreparedWorkerRemoval {
|
||||||
|
runtime_request: reconciliation.request,
|
||||||
|
prior_failure_category,
|
||||||
|
plan,
|
||||||
|
}));
|
||||||
|
}
|
||||||
let worker_id = plan
|
let worker_id = plan
|
||||||
.worker
|
.worker
|
||||||
.worker_id
|
.worker_id
|
||||||
@@ -698,8 +727,12 @@ impl SqliteWorkspaceStore {
|
|||||||
return Err(StoreError::InvalidInput("Erased removal receipt requires matching source and physical Runtime erasure acknowledgement".into()));
|
return Err(StoreError::InvalidInput("Erased removal receipt requires matching source and physical Runtime erasure acknowledgement".into()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let reconciliation=crate::store::subject_deletion_removal_matches(&tx,workspace_id,&plan.worker,&plan.operation_id,&plan.input_fingerprint)?;
|
||||||
let worker=load_worker(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput("Worker missing before commit".into()))?;
|
let worker=load_worker(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput("Worker missing before commit".into()))?;
|
||||||
if worker.updated_at!=plan.worker_updated_at{return Err(StoreError::InvalidInput(format!("stale:{}:Worker updated_at changed",plan.plan_id)));}
|
// A pinned pre-deletion request is reconciled unchanged under the permanent
|
||||||
|
// Subject/Runtime exclusion. Its Worker identity cannot be reused. Metadata
|
||||||
|
// version movement must not discard an already-applied canonical result.
|
||||||
|
if !reconciliation && worker.updated_at!=plan.worker_updated_at{return Err(StoreError::InvalidInput(format!("stale:{}:Worker updated_at changed",plan.plan_id)));}
|
||||||
if worker.retention_state=="pinned" { return Err(StoreError::InvalidInput(format!("stale:{}:hold added",plan.plan_id))); }
|
if worker.retention_state=="pinned" { return Err(StoreError::InvalidInput(format!("stale:{}:hold added",plan.plan_id))); }
|
||||||
let assigned:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3 UNION ALL SELECT 1 FROM ticket_assignment_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND claim_state='pending' AND action IN ('assign','reassign'))",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id],|row|row.get(0))?;
|
let assigned:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3 UNION ALL SELECT 1 FROM ticket_assignment_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND claim_state='pending' AND action IN ('assign','reassign'))",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id],|row|row.get(0))?;
|
||||||
if assigned { return Err(StoreError::InvalidInput(format!("stale:{}:unfinished work added",plan.plan_id))); }
|
if assigned { return Err(StoreError::InvalidInput(format!("stale:{}:unfinished work added",plan.plan_id))); }
|
||||||
@@ -771,7 +804,7 @@ impl SqliteWorkspaceStore {
|
|||||||
tx.execute("INSERT OR IGNORE INTO worker_tombstones(workspace_id,runtime_id,worker_id,display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_digest,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,worker.display_name,worker.profile,worker.created_at,now,if erasure.is_some() { None } else { plan.archive_id.clone() },plan.policy_id,plan.policy_digest,operation_id])?;
|
tx.execute("INSERT OR IGNORE INTO worker_tombstones(workspace_id,runtime_id,worker_id,display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_digest,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,worker.display_name,worker.profile,worker.created_at,now,if erasure.is_some() { None } else { plan.archive_id.clone() },plan.policy_id,plan.policy_digest,operation_id])?;
|
||||||
}
|
}
|
||||||
tx.execute("UPDATE worker_workdir_links SET unlinked_at=?4 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND unlinked_at IS NULL",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,now])?;
|
tx.execute("UPDATE worker_workdir_links SET unlinked_at=?4 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND unlinked_at IS NULL",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,now])?;
|
||||||
let deleted=tx.execute("DELETE FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND updated_at=?4",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.worker_updated_at])?;
|
let deleted=tx.execute("DELETE FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND (updated_at=?4 OR ?5)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.worker_updated_at,reconciliation])?;
|
||||||
tx.execute("UPDATE worker_create_reservations SET state='removed',updated_at=?4 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('reserved','created')",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,now])?;
|
tx.execute("UPDATE worker_create_reservations SET state='removed',updated_at=?4 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('reserved','created')",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,now])?;
|
||||||
if deleted!=1{return Err(StoreError::InvalidInput(format!("stale:{}:removal fence changed",plan.plan_id)));}
|
if deleted!=1{return Err(StoreError::InvalidInput(format!("stale:{}:removal fence changed",plan.plan_id)));}
|
||||||
let catalog=commit_worker_catalog_removal(&tx,workspace_id,&plan.worker)?;
|
let catalog=commit_worker_catalog_removal(&tx,workspace_id,&plan.worker)?;
|
||||||
@@ -1109,6 +1142,18 @@ fn require_current_subject_deletion_plan(
|
|||||||
c: &Connection,
|
c: &Connection,
|
||||||
plan: &WorkerRemovalPlan,
|
plan: &WorkerRemovalPlan,
|
||||||
) -> crate::Result<()> {
|
) -> crate::Result<()> {
|
||||||
|
if crate::store::subject_deletion_removal_matches(
|
||||||
|
c,
|
||||||
|
&plan.workspace_id,
|
||||||
|
&plan.worker,
|
||||||
|
&plan.operation_id,
|
||||||
|
&plan.input_fingerprint,
|
||||||
|
)? {
|
||||||
|
// An unfinished canonical request is pinned separately from successes. Only
|
||||||
|
// its exact original input may reconcile; any required Job copy erasure is
|
||||||
|
// a subsequent physical operation and never changes this historical result.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
if plan.state == WorkerRemovalPlanState::Succeeded
|
if plan.state == WorkerRemovalPlanState::Succeeded
|
||||||
&& crate::store::subject_deletion_prior_removal_matches(
|
&& crate::store::subject_deletion_prior_removal_matches(
|
||||||
c,
|
c,
|
||||||
@@ -2831,7 +2876,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn subject_deletion_snapshot_overrides_policy_without_changing_normal_removal() {
|
fn subject_deletion_snapshot_overrides_policy_without_changing_normal_removal() {
|
||||||
let store = setup();
|
let store = setup();
|
||||||
let ordinary = store.plan_worker_removal(&req(), &inv()).unwrap();
|
let ordinary = setup().plan_worker_removal(&req(), &inv()).unwrap();
|
||||||
store
|
store
|
||||||
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
|
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -2842,12 +2887,8 @@ mod tests {
|
|||||||
assert_eq!(body.diagnostics_disposition, DiagnosticsDisposition::Retain);
|
assert_eq!(body.diagnostics_disposition, DiagnosticsDisposition::Retain);
|
||||||
assert!(body.diagnostics_retention_seconds.is_some());
|
assert!(body.diagnostics_retention_seconds.is_some());
|
||||||
assert_ne!(body.policy_id, ordinary.policy_id);
|
assert_ne!(body.policy_id, ordinary.policy_id);
|
||||||
// A saved plan that would discard diagnostics cannot silently stand in
|
// A new deletion-owned plan uses the archival policy; independently saved
|
||||||
// for the Subject conversation archive plan.
|
// old requests are reconciled separately, without borrowing this policy.
|
||||||
assert!(matches!(
|
|
||||||
store.begin_worker_removal("w", &ordinary.plan_id, &ordinary.input_fingerprint),
|
|
||||||
Err(WorkerRetentionError::StalePlan { .. })
|
|
||||||
));
|
|
||||||
let ordinary_store = setup();
|
let ordinary_store = setup();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ordinary_store
|
ordinary_store
|
||||||
@@ -2859,10 +2900,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn subject_deletion_job_purge_rejects_recovery_of_old_executing_archive() {
|
fn subject_deletion_pins_old_archive_input_without_counting_it_as_job_purge() {
|
||||||
let store = setup();
|
let store = setup();
|
||||||
let old = store.plan_worker_removal(&req(), &inv()).unwrap();
|
let old = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||||
store
|
let original = store
|
||||||
.prepare_worker_removal_execution("w", &old.plan_id, &old.input_fingerprint)
|
.prepare_worker_removal_execution("w", &old.plan_id, &old.input_fingerprint)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
store
|
store
|
||||||
@@ -2877,14 +2918,21 @@ mod tests {
|
|||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(matches!(
|
let recovered = store
|
||||||
store.recover_worker_removal_execution("w", &req().worker),
|
.recover_worker_removal_execution("w", &req().worker)
|
||||||
Err(WorkerRetentionError::StalePlan { .. })
|
.unwrap()
|
||||||
));
|
.unwrap();
|
||||||
assert!(matches!(
|
assert_eq!(recovered.runtime_request, original.runtime_request);
|
||||||
store.plan_worker_removal(&req(), &inv()),
|
let prepared = store
|
||||||
Err(WorkerRetentionError::StalePlan { .. })
|
.prepare_worker_removal_execution("w", &old.plan_id, &old.input_fingerprint)
|
||||||
));
|
.unwrap();
|
||||||
|
assert_eq!(prepared.runtime_request, original.runtime_request);
|
||||||
|
assert!(store.plan_worker_removal(&req(), &inv()).is_err());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.erase_subject_deleted_jobs("w", "subject", "delete")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
let fresh = store
|
let fresh = store
|
||||||
.with_conn(|c| load_removal_policy(c, "w", &req().worker, false))
|
.with_conn(|c| load_removal_policy(c, "w", &req().worker, false))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|||||||
@@ -182,6 +182,45 @@ async fn remove_worker(
|
|||||||
)
|
)
|
||||||
.into());
|
.into());
|
||||||
}
|
}
|
||||||
|
if let Some(reconciliation) = api
|
||||||
|
.config_store
|
||||||
|
.subject_deletion_removal(api.workspace_id(), worker)?
|
||||||
|
{
|
||||||
|
if reconciliation.completed {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
// First reconcile the original canonical result, without changing any old
|
||||||
|
// disposition, fingerprint or success snapshot. A lost Runtime response or
|
||||||
|
// Backend commit remains unknown until that same child succeeds.
|
||||||
|
execute_removal(api, operation, worker).await?;
|
||||||
|
let result = api
|
||||||
|
.runtime
|
||||||
|
.execute_worker_retention(worker, reconciliation.request.clone())
|
||||||
|
.map_err(RuntimeRegistryError::into_error)?;
|
||||||
|
let purge = if reconciliation.is_job {
|
||||||
|
Some(
|
||||||
|
api.runtime
|
||||||
|
.purge_subject_job_removal(
|
||||||
|
&worker.runtime_id,
|
||||||
|
request,
|
||||||
|
reconciliation.request.clone(),
|
||||||
|
)
|
||||||
|
.map_err(RuntimeRegistryError::into_error)?,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
// The extra physical Job erasure is a distinct receipt. An old Archive
|
||||||
|
// success is never counted as a Purge, nor pinned as an older success later.
|
||||||
|
api.config_store.confirm_subject_deletion_removal(
|
||||||
|
api.workspace_id(),
|
||||||
|
worker,
|
||||||
|
&reconciliation.request,
|
||||||
|
&result,
|
||||||
|
purge.as_ref(),
|
||||||
|
)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
// Replay the canonical child even when physical removal already happened:
|
// Replay the canonical child even when physical removal already happened:
|
||||||
// its retained receipt can finish the Backend catalog midpoint. The Runtime
|
// its retained receipt can finish the Backend catalog midpoint. The Runtime
|
||||||
// fence above also prevents an older create from arriving after that removal.
|
// fence above also prevents an older create from arriving after that removal.
|
||||||
|
|||||||
@@ -727,3 +727,473 @@ async fn unmaterialized_body_and_job_reservations_finish_only_with_runtime_stora
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
async fn old_job_removal_reconciles_exact_input_then_erases_copies_across_http_retry_and_restart() {
|
||||||
|
use worker_runtime::retention::{DiagnosticsDisposition, SessionDisposition};
|
||||||
|
for phase in [
|
||||||
|
"planned",
|
||||||
|
"executing",
|
||||||
|
"failed",
|
||||||
|
"response_lost",
|
||||||
|
"catalog_failure",
|
||||||
|
"purge_ack_commit_failure",
|
||||||
|
] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
let subject = store
|
||||||
|
.create_subject(
|
||||||
|
crate::subjektiv::SubjectRole::new(format!("old plan {phase}")).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let other = store
|
||||||
|
.create_subject_with_behavior(
|
||||||
|
crate::subjektiv::SubjectRole::new("keep other").unwrap(),
|
||||||
|
"untouched".into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let candidate = serde_json::from_value(json!({
|
||||||
|
"schema_version": 2, "id": "old-job-candidate", "extract_run_id": "extract",
|
||||||
|
"source": {"segment_id":"segment", "range":[1,2]}, "kind":"constraint",
|
||||||
|
"claim":"private candidate", "why_useful":"erase after explicit deletion",
|
||||||
|
"evidence": [], "source_refs": []
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.stage_candidate(crate::subjektiv::SubjectStagingRecord::attach(
|
||||||
|
&subject.id,
|
||||||
|
candidate,
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
start_subjektiv_staging_consolidation(
|
||||||
|
api.clone(),
|
||||||
|
&subject.id,
|
||||||
|
MemoryConsolidateStagingOperation { force: true },
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let job = api
|
||||||
|
.store
|
||||||
|
.find_active_backend_job_for_resource(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&crate::backend_job::subjektiv_serialization_key(&subject.id),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let attempt = api
|
||||||
|
.store
|
||||||
|
.get_backend_job_attempt(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&job.request.job_id,
|
||||||
|
&crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let worker = attempt.worker.clone().unwrap();
|
||||||
|
seed_worker_session_for_cleanup(dir.path(), &worker);
|
||||||
|
api.runtime
|
||||||
|
.stop_worker(
|
||||||
|
&worker,
|
||||||
|
WorkerLifecycleRequest {
|
||||||
|
reason: Some("Prior independent removal".into()),
|
||||||
|
ticket_assignment: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let diagnostic = dir
|
||||||
|
.path()
|
||||||
|
.join(".test-embedded-runtime-store/workers")
|
||||||
|
.join(&worker.worker_id)
|
||||||
|
.join("runs/old/worker.out.log");
|
||||||
|
std::fs::create_dir_all(diagnostic.parent().unwrap()).unwrap();
|
||||||
|
std::fs::write(&diagnostic, "private old Job diagnostic").unwrap();
|
||||||
|
let policy = api
|
||||||
|
.config_store
|
||||||
|
.worker_retention_policy(TEST_WORKSPACE_ID)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
api.config_store
|
||||||
|
.update_worker_retention_policy(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&policy.policy_digest,
|
||||||
|
&crate::retention::WorkerRetentionPolicyUpdate {
|
||||||
|
policy_id: "prior-job-archive-policy".into(),
|
||||||
|
session_disposition: SessionDisposition::Archive,
|
||||||
|
metadata_disposition: crate::retention::MetadataDisposition::Tombstone,
|
||||||
|
archive_retention: crate::retention::ArchiveRetention::ForSeconds {
|
||||||
|
seconds: 600,
|
||||||
|
},
|
||||||
|
diagnostics_disposition: DiagnosticsDisposition::Retain,
|
||||||
|
diagnostics_retention_seconds: Some(600),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let inventory = api.runtime.worker_retention_inventory(&worker).unwrap();
|
||||||
|
assert!(inventory.diagnostics_bytes > 0);
|
||||||
|
let old = api
|
||||||
|
.config_store
|
||||||
|
.plan_worker_removal(
|
||||||
|
&crate::retention::WorkerRemovalPlanRequest {
|
||||||
|
workspace_id: TEST_WORKSPACE_ID.into(),
|
||||||
|
worker: worker.clone(),
|
||||||
|
reason: "Previously requested retention".into(),
|
||||||
|
journal_erasure: None,
|
||||||
|
},
|
||||||
|
&inventory,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(old.session_disposition, SessionDisposition::Archive);
|
||||||
|
let saved = if phase == "planned" {
|
||||||
|
api.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
} else {
|
||||||
|
api.config_store
|
||||||
|
.prepare_worker_removal_execution(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&old.plan_id,
|
||||||
|
&old.input_fingerprint,
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
if phase == "failed" {
|
||||||
|
api.config_store
|
||||||
|
.fail_worker_removal(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&old.operation_id,
|
||||||
|
&old.input_fingerprint,
|
||||||
|
"old_unknown_outcome",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
if phase == "catalog_failure" {
|
||||||
|
api.config_store.with_conn(|c| { c.execute_batch("CREATE TRIGGER injected_old_catalog_failure BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'catalog failure'); END;")?; Ok(()) }).unwrap();
|
||||||
|
}
|
||||||
|
let old_result = if matches!(phase, "response_lost" | "catalog_failure") {
|
||||||
|
let result = api
|
||||||
|
.runtime
|
||||||
|
.execute_worker_retention(&worker, saved.runtime_request.clone())
|
||||||
|
.unwrap();
|
||||||
|
if phase == "catalog_failure" {
|
||||||
|
assert!(
|
||||||
|
api.config_store
|
||||||
|
.commit_worker_removal(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&old.operation_id,
|
||||||
|
&old.input_fingerprint,
|
||||||
|
&result
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Some(result)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
if phase == "purge_ack_commit_failure" {
|
||||||
|
api.config_store.with_conn(|c| { c.execute_batch("CREATE TRIGGER injected_purge_ack_commit_failure BEFORE UPDATE OF completed ON subject_deletion_removals WHEN NEW.completed=1 BEGIN SELECT RAISE(ABORT,'purge receipt commit failure'); END;")?; Ok(()) }).unwrap();
|
||||||
|
}
|
||||||
|
// Force a midpoint AFTER domain/snapshot commit but BEFORE any deletion-owned
|
||||||
|
// Runtime effect. It must retain exactly the original canonical input.
|
||||||
|
*execution.subject_delete_failure.lock().unwrap() = Some("shutdown outcome unknown".into());
|
||||||
|
let owner =
|
||||||
|
seed_test_workspace_owner_token(api.store.as_ref(), &format!("old-removal-{phase}"))
|
||||||
|
.await;
|
||||||
|
let (status, pending) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"same-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{phase}: {pending}");
|
||||||
|
assert_eq!(pending["completed"], false, "{phase}");
|
||||||
|
let pinned = api
|
||||||
|
.config_store
|
||||||
|
.subject_deletion_removal(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(pinned.request, saved.runtime_request, "{phase}");
|
||||||
|
assert!(!pinned.completed);
|
||||||
|
assert!(pinned.is_job);
|
||||||
|
assert!(
|
||||||
|
api.config_store
|
||||||
|
.erase_subject_deleted_jobs(TEST_WORKSPACE_ID, &subject.id, "same-delete")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
*execution.subject_delete_failure.lock().unwrap() = None;
|
||||||
|
let mut reopened = api.clone();
|
||||||
|
let reopened_store =
|
||||||
|
Arc::new(SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap());
|
||||||
|
reopened.config_store = reopened_store.clone();
|
||||||
|
reopened.store = reopened_store;
|
||||||
|
if matches!(phase, "catalog_failure" | "purge_ack_commit_failure") {
|
||||||
|
let (status, pending) = delete_request(
|
||||||
|
&reopened,
|
||||||
|
&subject.id,
|
||||||
|
"same-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{phase}: {pending}");
|
||||||
|
assert_eq!(pending["completed"], false, "{phase}");
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.subject_deletion(&subject.id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
reopened
|
||||||
|
.store
|
||||||
|
.get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id)
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
reopened
|
||||||
|
.config_store
|
||||||
|
.with_conn(|c| {
|
||||||
|
c.execute_batch(if phase == "catalog_failure" {
|
||||||
|
"DROP TRIGGER injected_old_catalog_failure;"
|
||||||
|
} else {
|
||||||
|
"DROP TRIGGER injected_purge_ack_commit_failure;"
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
// Exercise the same production restart recovery that starts from the saved
|
||||||
|
// domain operation, then HTTP replay of the same full request.
|
||||||
|
let recovery = reopened.clone();
|
||||||
|
let handle = tokio::runtime::Handle::current();
|
||||||
|
reopened
|
||||||
|
.host_executors
|
||||||
|
.host()
|
||||||
|
.unwrap()
|
||||||
|
.run(move || handle.block_on(subjektiv_deletion::recover(&recovery)))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let (status, done) = delete_request(
|
||||||
|
&reopened,
|
||||||
|
&subject.id,
|
||||||
|
"same-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{phase}: {done}");
|
||||||
|
assert_eq!(done["completed"], true, "{phase}: {done}");
|
||||||
|
let reconciled = reopened
|
||||||
|
.config_store
|
||||||
|
.subject_deletion_removal(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(reconciled.completed);
|
||||||
|
assert_eq!(reconciled.request, saved.runtime_request);
|
||||||
|
let canonical = reopened
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
canonical.plan.state,
|
||||||
|
crate::retention::WorkerRemovalPlanState::Succeeded
|
||||||
|
);
|
||||||
|
assert_eq!(canonical.runtime_request, saved.runtime_request);
|
||||||
|
assert_eq!(
|
||||||
|
canonical.plan.session_disposition,
|
||||||
|
SessionDisposition::Archive,
|
||||||
|
"old result was not relabeled Purge"
|
||||||
|
);
|
||||||
|
let result = reopened
|
||||||
|
.runtime
|
||||||
|
.execute_worker_retention(&worker, saved.runtime_request.clone())
|
||||||
|
.unwrap();
|
||||||
|
if let Some(old_result) = old_result {
|
||||||
|
assert_eq!(result, old_result);
|
||||||
|
}
|
||||||
|
assert_eq!(result.session_disposition, SessionDisposition::Archive);
|
||||||
|
assert!(result.source_removed);
|
||||||
|
let archive = result.archive.as_ref().unwrap();
|
||||||
|
let root = dir.path().join(".test-embedded-runtime-store");
|
||||||
|
assert!(
|
||||||
|
!root
|
||||||
|
.join("archives/workers")
|
||||||
|
.join(&archive.archive_id)
|
||||||
|
.join("session")
|
||||||
|
.exists(),
|
||||||
|
"new Job archive copy survives {phase}"
|
||||||
|
);
|
||||||
|
if result.diagnostics_disposition == DiagnosticsDisposition::Retain {
|
||||||
|
assert!(
|
||||||
|
!root
|
||||||
|
.join("archives/diagnostics")
|
||||||
|
.join(&old.operation_id)
|
||||||
|
.exists(),
|
||||||
|
"Job diagnostics survive {phase}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
reopened
|
||||||
|
.store
|
||||||
|
.get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
reopened
|
||||||
|
.store
|
||||||
|
.current_worker_singleton_owner(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&crate::backend_job::subjektiv_serialization_key(&subject.id)
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||||
|
assert_eq!(
|
||||||
|
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||||
|
"untouched"
|
||||||
|
);
|
||||||
|
let (status, replay) = delete_request(
|
||||||
|
&reopened,
|
||||||
|
&subject.id,
|
||||||
|
"same-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(done, replay);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
async fn old_body_archive_reconciles_unchanged_and_remains_retained_after_subject_deletion() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
init_clean_git_workspace(dir.path());
|
||||||
|
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
let subject = store
|
||||||
|
.create_subject(crate::subjektiv::SubjectRole::new("old body archive").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let Json(created) = create_workspace_worker(
|
||||||
|
State(api.clone()),
|
||||||
|
HeaderMap::new(),
|
||||||
|
Json(CreateWorkspaceWorkerRequest {
|
||||||
|
runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(),
|
||||||
|
display_name: "Existing body removal".into(),
|
||||||
|
singleton_key: None,
|
||||||
|
profile: Some("builtin:companion".into()),
|
||||||
|
ticket_assignment: None,
|
||||||
|
initial_submit: vec![],
|
||||||
|
workdir_attachments: vec![],
|
||||||
|
feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest {
|
||||||
|
subjektiv: Some(server_api::SubjektivWorkerConnectionRequest {
|
||||||
|
subject_id: subject.id.clone(),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
control_operation_id: None,
|
||||||
|
execution_grant: None,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id);
|
||||||
|
seed_worker_session_for_cleanup(dir.path(), &worker);
|
||||||
|
api.runtime
|
||||||
|
.stop_worker(
|
||||||
|
&worker,
|
||||||
|
WorkerLifecycleRequest {
|
||||||
|
reason: Some("Earlier removal".into()),
|
||||||
|
ticket_assignment: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let inventory = api.runtime.worker_retention_inventory(&worker).unwrap();
|
||||||
|
let old = api
|
||||||
|
.config_store
|
||||||
|
.plan_worker_removal(
|
||||||
|
&crate::retention::WorkerRemovalPlanRequest {
|
||||||
|
workspace_id: TEST_WORKSPACE_ID.into(),
|
||||||
|
worker: worker.clone(),
|
||||||
|
reason: "Earlier authorized archive".into(),
|
||||||
|
journal_erasure: None,
|
||||||
|
},
|
||||||
|
&inventory,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let saved = api
|
||||||
|
.config_store
|
||||||
|
.prepare_worker_removal_execution(TEST_WORKSPACE_ID, &old.plan_id, &old.input_fingerprint)
|
||||||
|
.unwrap();
|
||||||
|
let result = api
|
||||||
|
.runtime
|
||||||
|
.execute_worker_retention(&worker, saved.runtime_request.clone())
|
||||||
|
.unwrap();
|
||||||
|
// Simulate a lost result before canonical Backend commit. Deletion must pin an
|
||||||
|
// unfinished request, not retrospectively classify its archive as a prior success.
|
||||||
|
*execution.subject_delete_failure.lock().unwrap() = Some("unknown shutdown".into());
|
||||||
|
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "old-body-removal").await;
|
||||||
|
let (status, pending) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"same-body-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{pending}");
|
||||||
|
assert_eq!(pending["completed"], false);
|
||||||
|
let snapshot = api
|
||||||
|
.config_store
|
||||||
|
.subject_deletion_removal(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(!snapshot.is_job && !snapshot.completed);
|
||||||
|
assert_eq!(snapshot.request, saved.runtime_request);
|
||||||
|
*execution.subject_delete_failure.lock().unwrap() = None;
|
||||||
|
let (status, done) = delete_request(
|
||||||
|
&api,
|
||||||
|
&subject.id,
|
||||||
|
"same-body-delete",
|
||||||
|
Some(&owner),
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK, "{done}");
|
||||||
|
assert_eq!(done["completed"], true, "{done}");
|
||||||
|
let completed = api
|
||||||
|
.config_store
|
||||||
|
.subject_deletion_removal(TEST_WORKSPACE_ID, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(completed.completed);
|
||||||
|
assert_eq!(completed.request, saved.runtime_request);
|
||||||
|
let replay = api
|
||||||
|
.runtime
|
||||||
|
.execute_worker_retention(&worker, saved.runtime_request)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(replay, result);
|
||||||
|
let archive = result.archive.unwrap();
|
||||||
|
assert!(
|
||||||
|
dir.path()
|
||||||
|
.join(".test-embedded-runtime-store/archives/workers")
|
||||||
|
.join(&archive.archive_id)
|
||||||
|
.join("session")
|
||||||
|
.is_dir()
|
||||||
|
);
|
||||||
|
assert!(api.config_store.with_conn(|c| Ok(c.query_row("SELECT prior_removal_operation_id IS NULL FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4", rusqlite::params![TEST_WORKSPACE_ID,subject.id,worker.runtime_id,worker.worker_id], |r|r.get::<_,bool>(0))?)).unwrap());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.subject_deletion(&subject.id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -95,7 +95,8 @@ use crate::{Error, Result};
|
|||||||
#[path = "subject_deletion_store.rs"]
|
#[path = "subject_deletion_store.rs"]
|
||||||
mod subject_deletion_store;
|
mod subject_deletion_store;
|
||||||
pub(crate) use subject_deletion_store::{
|
pub(crate) use subject_deletion_store::{
|
||||||
prior_removal_matches as subject_deletion_prior_removal_matches,
|
SubjectDeletionRemoval, prior_removal_matches as subject_deletion_prior_removal_matches,
|
||||||
|
removal_reconciliation_matches as subject_deletion_removal_matches,
|
||||||
worker_kind as subject_deletion_worker_kind,
|
worker_kind as subject_deletion_worker_kind,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -16255,7 +16256,8 @@ fn current_worker_singleton_owner(
|
|||||||
let owner = conn
|
let owner = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT owner.runtime_id, owner.worker_id, owner.generation, \
|
"SELECT owner.runtime_id, owner.worker_id, owner.generation, \
|
||||||
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
CASE WHEN reservation.state='removed' AND EXISTS(SELECT 1 FROM subject_deletion_removals d WHERE d.workspace_id=owner.workspace_id AND d.runtime_id=owner.runtime_id AND d.worker_id=owner.worker_id AND owner.singleton_key IN ('subjektiv:'||d.subject_id,'subjektiv-consolidation:'||d.subject_id)) THEN 'created' \
|
||||||
|
WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
||||||
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r JOIN subject_deletion_workers d ON d.workspace_id=r.workspace_id AND d.runtime_id=r.runtime_id AND d.worker_id=r.worker_id WHERE r.workspace_id=owner.workspace_id AND r.runtime_id=owner.runtime_id AND r.worker_id=owner.worker_id AND r.state='succeeded' AND owner.singleton_key='subjektiv:'||d.subject_id AND (d.prior_removal_operation_id=r.operation_id OR (d.prior_removal_operation_id IS NULL AND d.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.metadata_disposition='tombstone'))) \
|
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r JOIN subject_deletion_workers d ON d.workspace_id=r.workspace_id AND d.runtime_id=r.runtime_id AND d.worker_id=r.worker_id WHERE r.workspace_id=owner.workspace_id AND r.runtime_id=owner.runtime_id AND r.worker_id=owner.worker_id AND r.state='succeeded' AND owner.singleton_key='subjektiv:'||d.subject_id AND (d.prior_removal_operation_id=r.operation_id OR (d.prior_removal_operation_id IS NULL AND d.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.metadata_disposition='tombstone'))) \
|
||||||
@@ -16278,7 +16280,15 @@ fn current_worker_singleton_owner(
|
|||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
owner
|
owner
|
||||||
.map(|(runtime_id, worker_id, generation, state)| {
|
.map(|(runtime_id, worker_id, generation, mut state)| {
|
||||||
|
let worker = RuntimeWorkerRef::new(runtime_id, worker_id);
|
||||||
|
if let Some(subject) = singleton_key.strip_prefix("subjektiv:").or_else(|| singleton_key.strip_prefix("subjektiv-consolidation:"))
|
||||||
|
&& state.as_deref() == Some("created")
|
||||||
|
&& subject_deletion_store::completed_removal_matches(conn, workspace_id, subject, &worker)?
|
||||||
|
{
|
||||||
|
let removed: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='removed')",params![workspace_id,worker.runtime_id,worker.worker_id],|r|r.get(0))?;
|
||||||
|
if removed { state = Some("removed".into()); }
|
||||||
|
}
|
||||||
let generation = generation.try_into().map_err(|_| {
|
let generation = generation.try_into().map_err(|_| {
|
||||||
Error::Store(format!(
|
Error::Store(format!(
|
||||||
"Worker singleton key {singleton_key:?} has an invalid generation"
|
"Worker singleton key {singleton_key:?} has an invalid generation"
|
||||||
@@ -16288,7 +16298,7 @@ fn current_worker_singleton_owner(
|
|||||||
WorkerSingletonLease {
|
WorkerSingletonLease {
|
||||||
key: singleton_key.to_string(),
|
key: singleton_key.to_string(),
|
||||||
generation,
|
generation,
|
||||||
worker: RuntimeWorkerRef::new(runtime_id, worker_id),
|
worker,
|
||||||
},
|
},
|
||||||
state,
|
state,
|
||||||
))
|
))
|
||||||
@@ -26903,7 +26913,7 @@ INSERT INTO worker_registry (
|
|||||||
.query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
|
.query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
for (name,table) in triggers {conn.execute_batch(&format!("DROP TRIGGER {name}; CREATE TRIGGER {name} BEFORE DELETE ON {table} BEGIN SELECT RAISE(ABORT,'Notification evidence is immutable'); END;"))?;}
|
for (name,table) in triggers {conn.execute_batch(&format!("DROP TRIGGER {name}; CREATE TRIGGER {name} BEFORE DELETE ON {table} BEGIN SELECT RAISE(ABORT,'Notification evidence is immutable'); END;"))?;}
|
||||||
for name in ["subject_deleted_job_insert","subject_deleted_job_result","subject_deleted_attempt_insert","subject_deleted_attempt_result","subject_deleted_worker_reservation","subject_deleted_notification_source","subject_deleted_notification_payload","subject_deleted_notification_send","subject_deletion_worker_prior_insert","subject_deletion_worker_identity_immutable"] {conn.execute_batch(&format!("DROP TRIGGER {name}"))?;}
|
for name in ["subject_deleted_job_insert","subject_deleted_job_result","subject_deleted_attempt_insert","subject_deleted_attempt_result","subject_deleted_worker_reservation","subject_deleted_notification_source","subject_deleted_notification_payload","subject_deleted_notification_send","subject_deletion_worker_prior_insert","subject_deletion_worker_identity_immutable"] {conn.execute_batch(&format!("DROP TRIGGER {name}"))?;}
|
||||||
conn.execute_batch("DROP TABLE subject_deletion_allocations; DROP TABLE subject_deletion_jobs; DROP TABLE subject_deletion_workers; DROP TABLE subject_deletions;")?;
|
conn.execute_batch("DROP TABLE subject_deletion_removals; DROP TABLE subject_deletion_allocations; DROP TABLE subject_deletion_jobs; DROP TABLE subject_deletion_workers; DROP TABLE subject_deletions;")?;
|
||||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
DROP TRIGGER passkey_metadata_insert;
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
|
|||||||
@@ -35,6 +35,41 @@ BEGIN SELECT RAISE(ABORT,'Subject deletion Worker snapshot identity is immutable
|
|||||||
CREATE TRIGGER subject_deletion_original_absence_monotonic BEFORE UPDATE OF originals_absent ON subject_deletion_workers
|
CREATE TRIGGER subject_deletion_original_absence_monotonic BEFORE UPDATE OF originals_absent ON subject_deletion_workers
|
||||||
WHEN OLD.originals_absent=1 AND NEW.originals_absent=0
|
WHEN OLD.originals_absent=1 AND NEW.originals_absent=0
|
||||||
BEGIN SELECT RAISE(ABORT,'Subject deletion original absence proof is permanent'); END;
|
BEGIN SELECT RAISE(ABORT,'Subject deletion original absence proof is permanent'); END;
|
||||||
|
-- In-flight removals are distinct from successful prior receipts. Pin the exact
|
||||||
|
-- old Runtime request once; Subject deletion never rewrites its retention policy.
|
||||||
|
CREATE TABLE subject_deletion_removals (
|
||||||
|
workspace_id TEXT NOT NULL,
|
||||||
|
subject_id TEXT NOT NULL,
|
||||||
|
runtime_id TEXT NOT NULL,
|
||||||
|
worker_id TEXT NOT NULL,
|
||||||
|
operation_id TEXT NOT NULL REFERENCES worker_removal_operations(operation_id),
|
||||||
|
input_fingerprint TEXT NOT NULL,
|
||||||
|
request_json TEXT NOT NULL CHECK(json_valid(request_json)),
|
||||||
|
completed INTEGER NOT NULL DEFAULT 0 CHECK(completed IN (0,1)),
|
||||||
|
job_purge_receipt_json TEXT CHECK(job_purge_receipt_json IS NULL OR json_valid(job_purge_receipt_json)),
|
||||||
|
PRIMARY KEY(workspace_id,subject_id,runtime_id,worker_id),
|
||||||
|
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY(workspace_id,subject_id,runtime_id,worker_id)
|
||||||
|
REFERENCES subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE TRIGGER subject_deletion_removal_insert BEFORE INSERT ON subject_deletion_removals
|
||||||
|
WHEN NEW.completed!=0 OR NEW.job_purge_receipt_json IS NOT NULL OR NOT EXISTS(
|
||||||
|
SELECT 1 FROM subject_deletion_workers w JOIN worker_removal_operations r
|
||||||
|
ON r.workspace_id=w.workspace_id AND r.runtime_id=w.runtime_id AND r.worker_id=w.worker_id
|
||||||
|
WHERE w.workspace_id=NEW.workspace_id AND w.subject_id=NEW.subject_id
|
||||||
|
AND w.runtime_id=NEW.runtime_id AND w.worker_id=NEW.worker_id AND w.prior_removal_operation_id IS NULL
|
||||||
|
AND r.operation_id=NEW.operation_id AND r.input_fingerprint=NEW.input_fingerprint
|
||||||
|
AND r.state IN ('planned','executing','failed')
|
||||||
|
AND json_extract(NEW.request_json,'$.operation_id')=r.operation_id
|
||||||
|
AND json_extract(NEW.request_json,'$.input_fingerprint')=r.input_fingerprint)
|
||||||
|
BEGIN SELECT RAISE(ABORT,'Subject deletion reconciliation must pin an unfinished canonical removal'); END;
|
||||||
|
CREATE TRIGGER subject_deletion_removal_identity_immutable BEFORE UPDATE OF
|
||||||
|
workspace_id,subject_id,runtime_id,worker_id,operation_id,input_fingerprint,request_json ON subject_deletion_removals
|
||||||
|
BEGIN SELECT RAISE(ABORT,'Subject deletion removal request is immutable'); END;
|
||||||
|
CREATE TRIGGER subject_deletion_removal_completion_monotonic BEFORE UPDATE ON subject_deletion_removals
|
||||||
|
WHEN (OLD.completed=1 AND NEW.completed!=1)
|
||||||
|
OR (OLD.job_purge_receipt_json IS NOT NULL AND NEW.job_purge_receipt_json IS NOT OLD.job_purge_receipt_json)
|
||||||
|
BEGIN SELECT RAISE(ABORT,'Subject deletion removal receipt is permanent'); END;
|
||||||
-- Job/attempt identities remain after payload erasure, preventing stale allocation
|
-- Job/attempt identities remain after payload erasure, preventing stale allocation
|
||||||
-- or caller-selected Job-id reuse from recreating the deleted Subject's inputs.
|
-- or caller-selected Job-id reuse from recreating the deleted Subject's inputs.
|
||||||
CREATE TABLE subject_deletion_jobs (
|
CREATE TABLE subject_deletion_jobs (
|
||||||
|
|||||||
@@ -5,14 +5,184 @@
|
|||||||
//! All historical Job attempts and dispatch allocations are snapshotted, not just
|
//! All historical Job attempts and dispatch allocations are snapshotted, not just
|
||||||
//! the current attempt. Pending/unknown work becomes failed/subject_deleted (not a
|
//! the current attempt. Pending/unknown work becomes failed/subject_deleted (not a
|
||||||
//! successful Job). Originals in body/delegated Sessions are archived forever;
|
//! successful Job). Originals in body/delegated Sessions are archived forever;
|
||||||
//! Existing canonical Worker removals and archives are outside this deletion:
|
//! Successful canonical Worker removals and archives are outside this deletion:
|
||||||
//! prepare pins their exact successful receipt, regardless of its disposition.
|
//! prepare pins their exact successful receipt, regardless of its disposition.
|
||||||
|
//! Unfinished removals instead pin their immutable old execution request; exact
|
||||||
|
//! reconciliation and (for Jobs) additional physical erasure must complete first.
|
||||||
//! Newly removed Job Sessions/diagnostics need canonical Purge before input,
|
//! Newly removed Job Sessions/diagnostics need canonical Purge before input,
|
||||||
//! result, notification payloads and live Job/cache references can be erased.
|
//! result, notification payloads and live Job/cache references can be erased.
|
||||||
//! This DB has no separate Job content cache. Runtime caches are the canonical
|
//! This DB has no separate Job content cache. Runtime caches are the canonical
|
||||||
//! Worker removal's responsibility. Notification text already delivered into an
|
//! Worker removal's responsibility. Notification text already delivered into an
|
||||||
//! ordinary conversation follows that conversation's archive lifecycle.
|
//! ordinary conversation follows that conversation's archive lifecycle.
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use worker_runtime::retention::{
|
||||||
|
DiagnosticsDisposition, WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub(crate) struct SubjectDeletionRemoval {
|
||||||
|
pub request: WorkerRetentionExecutionRequest,
|
||||||
|
pub completed: bool,
|
||||||
|
pub is_job: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_request(json: &str) -> Result<WorkerRetentionExecutionRequest> {
|
||||||
|
serde_json::from_str(json).map_err(|e| Error::Store(e.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only source metadata comes from the saved Worker snapshot. Everything owned by
|
||||||
|
// the canonical plan is rechecked, even after its success/catalog removal.
|
||||||
|
fn canonical_request(
|
||||||
|
conn: &Connection,
|
||||||
|
operation: &str,
|
||||||
|
source_created_at: &str,
|
||||||
|
profile: Option<&str>,
|
||||||
|
) -> Result<Option<(WorkerRetentionExecutionRequest, String)>> {
|
||||||
|
let row: Option<(String, String)> = conn.query_row(
|
||||||
|
"SELECT json_object('operation_id',operation_id,'input_fingerprint',input_fingerprint,
|
||||||
|
'archive_id',archive_id,'workspace_id',workspace_id,'source_runtime_id',runtime_id,
|
||||||
|
'worker_id',worker_id,'source_created_at',?2,'removed_at',created_at,
|
||||||
|
'effective_profile',?3,'retention_class',NULL,'policy_id',policy_id,
|
||||||
|
'session_disposition',session_disposition,'diagnostics_disposition',diagnostics_disposition),state
|
||||||
|
FROM worker_removal_operations WHERE operation_id=?1",
|
||||||
|
params![operation,source_created_at,profile], |r| Ok((r.get(0)?,r.get(1)?)),
|
||||||
|
).optional()?;
|
||||||
|
row.map(|(json, state)| Ok((decode_request(&json)?, state)))
|
||||||
|
.transpose()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn saved_removal(
|
||||||
|
conn: &Connection,
|
||||||
|
workspace: &str,
|
||||||
|
subject: Option<&str>,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
) -> Result<Option<SubjectDeletionRemoval>> {
|
||||||
|
let rows = conn.prepare("SELECT d.request_json,d.completed,w.is_job FROM subject_deletion_removals d
|
||||||
|
JOIN subject_deletion_workers w USING(workspace_id,subject_id,runtime_id,worker_id)
|
||||||
|
WHERE d.workspace_id=?1 AND d.runtime_id=?2 AND d.worker_id=?3 AND (?4 IS NULL OR d.subject_id=?4)
|
||||||
|
ORDER BY d.subject_id")?
|
||||||
|
.query_map(params![workspace,worker.runtime_id,worker.worker_id,subject], |r| Ok((r.get::<_,String>(0)?,r.get::<_,bool>(1)?,r.get::<_,bool>(2)?)))?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
let mut saved: Option<SubjectDeletionRemoval> = None;
|
||||||
|
for (json, completed, is_job) in rows {
|
||||||
|
let request = decode_request(&json)?;
|
||||||
|
if let Some(saved) = saved.as_mut() {
|
||||||
|
if saved.request != request {
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Conflicting Subject deletion removal snapshots".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
saved.completed &= completed;
|
||||||
|
saved.is_job |= is_job;
|
||||||
|
} else {
|
||||||
|
saved = Some(SubjectDeletionRemoval {
|
||||||
|
request,
|
||||||
|
completed,
|
||||||
|
is_job,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(saved)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn removal_reconciliation_matches(
|
||||||
|
conn: &Connection,
|
||||||
|
workspace: &str,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
operation: &str,
|
||||||
|
fp: &str,
|
||||||
|
) -> Result<bool> {
|
||||||
|
let Some(saved) = saved_removal(conn, workspace, None, worker)? else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
let request = &saved.request;
|
||||||
|
if request.workspace_id != workspace
|
||||||
|
|| request.source_runtime_id != worker.runtime_id
|
||||||
|
|| request.worker_id.to_string() != worker.worker_id
|
||||||
|
|| request.operation_id != operation
|
||||||
|
|| request.input_fingerprint != fp
|
||||||
|
{
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
Ok(canonical_request(
|
||||||
|
conn,
|
||||||
|
operation,
|
||||||
|
&request.source_created_at,
|
||||||
|
request.effective_profile.as_deref(),
|
||||||
|
)?
|
||||||
|
.is_some_and(|(canonical, state)| {
|
||||||
|
canonical == *request
|
||||||
|
&& matches!(
|
||||||
|
state.as_str(),
|
||||||
|
"planned" | "executing" | "failed" | "succeeded"
|
||||||
|
)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn completed_removal_matches(
|
||||||
|
conn: &Connection,
|
||||||
|
workspace: &str,
|
||||||
|
subject: &str,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
) -> Result<bool> {
|
||||||
|
let Some(saved) = saved_removal(conn, workspace, Some(subject), worker)? else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
if !saved.completed
|
||||||
|
|| !removal_reconciliation_matches(
|
||||||
|
conn,
|
||||||
|
workspace,
|
||||||
|
worker,
|
||||||
|
&saved.request.operation_id,
|
||||||
|
&saved.request.input_fingerprint,
|
||||||
|
)?
|
||||||
|
{
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
Ok(conn.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM subject_deletion_removals d
|
||||||
|
JOIN worker_removal_operations r ON r.operation_id=d.operation_id AND r.input_fingerprint=d.input_fingerprint
|
||||||
|
WHERE d.workspace_id=?1 AND d.subject_id=?2 AND d.runtime_id=?3 AND d.worker_id=?4
|
||||||
|
AND r.state='succeeded' AND d.completed=1 AND (?5=0 OR (
|
||||||
|
json_extract(d.job_purge_receipt_json,'$.workspace_id')=d.workspace_id
|
||||||
|
AND json_extract(d.job_purge_receipt_json,'$.worker_id')=d.worker_id
|
||||||
|
AND json_extract(d.job_purge_receipt_json,'$.operation_id')=d.operation_id
|
||||||
|
AND json_extract(d.job_purge_receipt_json,'$.input_fingerprint')=d.input_fingerprint)))",
|
||||||
|
params![workspace,subject,worker.runtime_id,worker.worker_id,saved.is_job],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pin_removal(
|
||||||
|
conn: &Connection,
|
||||||
|
workspace: &str,
|
||||||
|
subject: &str,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
) -> Result<()> {
|
||||||
|
let operation: Option<String> = conn.query_row("SELECT operation_id FROM worker_removal_operations
|
||||||
|
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('planned','executing','failed')
|
||||||
|
AND (state IN ('executing','failed') OR worker_updated_at=(SELECT updated_at FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3))
|
||||||
|
ORDER BY created_at DESC,operation_id DESC LIMIT 1",
|
||||||
|
params![workspace,worker.runtime_id,worker.worker_id],|r|r.get(0)).optional()?;
|
||||||
|
let Some(operation) = operation else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
let source: Option<(String,Option<String>)> = conn.query_row("SELECT created_at,profile FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",
|
||||||
|
params![workspace,worker.runtime_id,worker.worker_id], |r| Ok((r.get(0)?,r.get(1)?))).optional()?;
|
||||||
|
let (created_at, profile) = source.ok_or_else(|| {
|
||||||
|
Error::RepositoryConflict(
|
||||||
|
"Unfinished Worker removal has no source identity to snapshot".into(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let (request, _) = canonical_request(conn, &operation, &created_at, profile.as_deref())?
|
||||||
|
.ok_or_else(|| {
|
||||||
|
Error::RepositoryConflict("Canonical removal disappeared while snapshotting".into())
|
||||||
|
})?;
|
||||||
|
let json = serde_json::to_string(&request).map_err(|e| Error::Store(e.to_string()))?;
|
||||||
|
conn.execute("INSERT INTO subject_deletion_removals(workspace_id,subject_id,runtime_id,worker_id,operation_id,input_fingerprint,request_json)
|
||||||
|
VALUES(?1,?2,?3,?4,?5,?6,?7)",params![workspace,subject,worker.runtime_id,worker.worker_id,operation,request.input_fingerprint,json])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
const MIGRATION_NAME: &str = "Durable Subject deletion snapshots and Job admission fences";
|
const MIGRATION_NAME: &str = "Durable Subject deletion snapshots and Job admission fences";
|
||||||
|
|
||||||
@@ -103,6 +273,10 @@ pub(super) fn verify(conn: &Connection) -> Result<()> {
|
|||||||
for name in [
|
for name in [
|
||||||
"subject_deletions",
|
"subject_deletions",
|
||||||
"subject_deletion_workers",
|
"subject_deletion_workers",
|
||||||
|
"subject_deletion_removals",
|
||||||
|
"subject_deletion_removal_insert",
|
||||||
|
"subject_deletion_removal_identity_immutable",
|
||||||
|
"subject_deletion_removal_completion_monotonic",
|
||||||
"subject_deletion_jobs",
|
"subject_deletion_jobs",
|
||||||
"subject_deletion_allocations",
|
"subject_deletion_allocations",
|
||||||
"subject_deleted_job_insert",
|
"subject_deleted_job_insert",
|
||||||
@@ -251,6 +425,72 @@ fn worker_fence_request(
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SqliteWorkspaceStore {
|
impl SqliteWorkspaceStore {
|
||||||
|
pub(crate) fn subject_deletion_removal(
|
||||||
|
&self,
|
||||||
|
workspace: &str,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
) -> Result<Option<super::SubjectDeletionRemoval>> {
|
||||||
|
self.with_conn(|conn| saved_removal(conn, workspace, None, worker))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A canonical success alone is not proof of additional Job erasure. Keep the
|
||||||
|
/// old receipt/dispositions intact and only complete this separate snapshot
|
||||||
|
/// after the exact Runtime result and (for Jobs) physical purge response.
|
||||||
|
pub(crate) fn confirm_subject_deletion_removal(
|
||||||
|
&self,
|
||||||
|
workspace: &str,
|
||||||
|
worker: &RuntimeWorkerRef,
|
||||||
|
request: &WorkerRetentionExecutionRequest,
|
||||||
|
result: &WorkerRetentionExecutionResult,
|
||||||
|
purge: Option<&runtime_api::SubjectJobRemovalPurgeResponse>,
|
||||||
|
) -> Result<()> {
|
||||||
|
self.with_conn_mut(|conn| {
|
||||||
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
let saved = saved_removal(&tx,workspace,None,worker)?
|
||||||
|
.ok_or_else(|| Error::RepositoryConflict("No saved Subject deletion removal".into()))?;
|
||||||
|
if saved.request != *request
|
||||||
|
|| !removal_reconciliation_matches(&tx,workspace,worker,&request.operation_id,&request.input_fingerprint)?
|
||||||
|
|| !tx.query_row("SELECT state='succeeded' FROM worker_removal_operations WHERE operation_id=?1",[&request.operation_id],|r|r.get::<_,bool>(0))?
|
||||||
|
|| result.operation_id != request.operation_id || result.input_fingerprint != request.input_fingerprint
|
||||||
|
|| result.worker_id != request.worker_id || result.session_disposition != request.session_disposition
|
||||||
|
|| result.diagnostics_disposition != request.diagnostics_disposition || !result.source_removed
|
||||||
|
|| result.diagnostics_retained != (request.diagnostics_disposition == DiagnosticsDisposition::Retain) {
|
||||||
|
return Err(Error::RepositoryConflict("Subject deletion removal requires exact saved request and canonical Runtime success".into()));
|
||||||
|
}
|
||||||
|
let archive_matches = match (&result.archive, request.session_disposition) {
|
||||||
|
(Some(archive), worker_runtime::retention::SessionDisposition::Archive) => {
|
||||||
|
request.archive_id.as_deref() == Some(archive.archive_id.as_str())
|
||||||
|
&& archive.workspace_id == workspace
|
||||||
|
&& archive.source_runtime_id == worker.runtime_id
|
||||||
|
&& archive.source_worker_id == request.worker_id
|
||||||
|
&& archive.operation_id == request.operation_id
|
||||||
|
&& archive.input_fingerprint == request.input_fingerprint
|
||||||
|
&& archive.policy_id == request.policy_id
|
||||||
|
&& archive.source_created_at == request.source_created_at
|
||||||
|
}
|
||||||
|
(None, worker_runtime::retention::SessionDisposition::Purge) => true,
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
if !archive_matches {
|
||||||
|
return Err(Error::RepositoryConflict("Subject deletion removal archive identity mismatch".into()));
|
||||||
|
}
|
||||||
|
if saved.is_job && purge.is_none() {
|
||||||
|
return Err(Error::RepositoryConflict("Subject Job removal requires an additional physical purge receipt".into()));
|
||||||
|
}
|
||||||
|
if let Some(purge) = purge {
|
||||||
|
if purge.workspace_id != workspace || purge.worker_id != request.worker_id
|
||||||
|
|| purge.operation_id != request.operation_id || purge.input_fingerprint != request.input_fingerprint {
|
||||||
|
return Err(Error::RepositoryConflict("Subject Job physical purge receipt identity mismatch".into()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let purge_json = purge.map(serde_json::to_string).transpose().map_err(|e| Error::Store(e.to_string()))?;
|
||||||
|
tx.execute("UPDATE subject_deletion_removals SET completed=1,job_purge_receipt_json=coalesce(job_purge_receipt_json,?4)
|
||||||
|
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",params![workspace,worker.runtime_id,worker.worker_id,purge_json])?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn subject_deletion_worker_request(
|
pub(crate) fn subject_deletion_worker_request(
|
||||||
&self,
|
&self,
|
||||||
workspace: &str,
|
workspace: &str,
|
||||||
@@ -316,6 +556,12 @@ impl SqliteWorkspaceStore {
|
|||||||
tx.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id) VALUES(?1,?2,?3,?4,?5,?6)
|
tx.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id) VALUES(?1,?2,?3,?4,?5,?6)
|
||||||
ON CONFLICT(workspace_id,subject_id,runtime_id,worker_id) DO UPDATE SET is_job=max(is_job,excluded.is_job)",params![workspace,subject,worker.runtime_id,worker.worker_id,is_job,prior])?;
|
ON CONFLICT(workspace_id,subject_id,runtime_id,worker_id) DO UPDATE SET is_job=max(is_job,excluded.is_job)",params![workspace,subject,worker.runtime_id,worker.worker_id,is_job,prior])?;
|
||||||
}
|
}
|
||||||
|
// Deduplicate workers before pinning: an identity can be both a
|
||||||
|
// delegated Worker and a Job attempt; the final is_job wins.
|
||||||
|
let pending = tx.prepare("SELECT runtime_id,worker_id FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND prior_removal_operation_id IS NULL")?
|
||||||
|
.query_map(params![workspace,subject],|r|Ok(RuntimeWorkerRef::new(r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
for worker in pending { pin_removal(&tx,workspace,subject,&worker)?; }
|
||||||
let now = chrono::Utc::now().to_rfc3339();
|
let now = chrono::Utc::now().to_rfc3339();
|
||||||
tx.execute("UPDATE backend_job_attempts SET state='failed',failure_category='subject_deleted',failure_detail=NULL,completed_at=?3,updated_at=?3,
|
tx.execute("UPDATE backend_job_attempts SET state='failed',failure_category='subject_deleted',failure_detail=NULL,completed_at=?3,updated_at=?3,
|
||||||
worker_cleanup_state=CASE WHEN worker_id IS NULL THEN 'completed' ELSE 'pending' END,
|
worker_cleanup_state=CASE WHEN worker_id IS NULL THEN 'completed' ELSE 'pending' END,
|
||||||
@@ -362,6 +608,12 @@ impl SqliteWorkspaceStore {
|
|||||||
|| fence.create_fingerprint != request.create_fingerprint || fence.operation_id != request.operation_id {
|
|| fence.create_fingerprint != request.create_fingerprint || fence.operation_id != request.operation_id {
|
||||||
return Err(Error::RepositoryConflict("Subject deletion absence fence identity mismatch".into()));
|
return Err(Error::RepositoryConflict("Subject deletion absence fence identity mismatch".into()));
|
||||||
}
|
}
|
||||||
|
// An in-flight pin cannot be bypassed by UnknownWorker/absence or
|
||||||
|
// a different later canonical success, even if that success is Purge.
|
||||||
|
if saved_removal(conn,workspace,Some(subject),worker)?.is_some() {
|
||||||
|
if completed_removal_matches(conn,workspace,subject,worker)? { return Ok(()); }
|
||||||
|
return Err(Error::RepositoryConflict("Subject deletion removal reconciliation is incomplete".into()));
|
||||||
|
}
|
||||||
if fence.originals_absent {
|
if fence.originals_absent {
|
||||||
let registered: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3)", params![workspace,worker.runtime_id,worker.worker_id], |r|r.get(0))?;
|
let registered: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3)", params![workspace,worker.runtime_id,worker.worker_id], |r|r.get(0))?;
|
||||||
if registered { return Err(Error::RepositoryConflict("Registered Worker requires canonical original retention".into())); }
|
if registered { return Err(Error::RepositoryConflict("Registered Worker requires canonical original retention".into())); }
|
||||||
@@ -474,7 +726,19 @@ fn require_worker_receipts(
|
|||||||
subject: &str,
|
subject: &str,
|
||||||
jobs_only: bool,
|
jobs_only: bool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
let pinned = conn.prepare("SELECT d.runtime_id,d.worker_id FROM subject_deletion_removals d JOIN subject_deletion_workers w USING(workspace_id,subject_id,runtime_id,worker_id)
|
||||||
|
WHERE d.workspace_id=?1 AND d.subject_id=?2 AND (?3=0 OR w.is_job=1)")?
|
||||||
|
.query_map(params![workspace,subject,jobs_only],|r|Ok(RuntimeWorkerRef::new(r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
for worker in pinned {
|
||||||
|
if !completed_removal_matches(conn, workspace, subject, &worker)? {
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject deletion removal reconciliation receipt is incomplete".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
let missing: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_workers w WHERE w.workspace_id=?1 AND w.subject_id=?2 AND (?3=0 OR w.is_job=1) AND w.originals_absent=0
|
let missing: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_workers w WHERE w.workspace_id=?1 AND w.subject_id=?2 AND (?3=0 OR w.is_job=1) AND w.originals_absent=0
|
||||||
|
AND NOT EXISTS(SELECT 1 FROM subject_deletion_removals d WHERE d.workspace_id=w.workspace_id AND d.subject_id=w.subject_id AND d.runtime_id=w.runtime_id AND d.worker_id=w.worker_id)
|
||||||
AND NOT (
|
AND NOT (
|
||||||
(w.prior_removal_operation_id IS NOT NULL AND EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.operation_id=w.prior_removal_operation_id AND r.workspace_id=w.workspace_id AND r.runtime_id=w.runtime_id AND r.worker_id=w.worker_id AND r.state='succeeded'))
|
(w.prior_removal_operation_id IS NOT NULL AND EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.operation_id=w.prior_removal_operation_id AND r.workspace_id=w.workspace_id AND r.runtime_id=w.runtime_id AND r.worker_id=w.worker_id AND r.state='succeeded'))
|
||||||
OR (w.prior_removal_operation_id IS NULL AND (
|
OR (w.prior_removal_operation_id IS NULL AND (
|
||||||
@@ -1108,6 +1372,8 @@ mod tests {
|
|||||||
"job-owner",
|
"job-owner",
|
||||||
Some("subjektiv-consolidation:subject"),
|
Some("subjektiv-consolidation:subject"),
|
||||||
);
|
);
|
||||||
|
registry(&s, &job_worker);
|
||||||
|
registry(&s, &body);
|
||||||
receipt(&s, &job_worker, "executing", "archive", "retain");
|
receipt(&s, &job_worker, "executing", "archive", "retain");
|
||||||
receipt(&s, &body, "failed", "purge", "purge");
|
receipt(&s, &body, "failed", "purge", "purge");
|
||||||
s.prepare_subject_deletion("w", "subject", "delete", std::slice::from_ref(&body))
|
s.prepare_subject_deletion("w", "subject", "delete", std::slice::from_ref(&body))
|
||||||
@@ -1134,8 +1400,11 @@ mod tests {
|
|||||||
c.execute("UPDATE worker_create_reservations SET state='removed' WHERE workspace_id='w'",[])?;
|
c.execute("UPDATE worker_create_reservations SET state='removed' WHERE workspace_id='w'",[])?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
|
assert!(
|
||||||
s.erase_subject_deleted_jobs("w", "subject", "delete")
|
s.erase_subject_deleted_jobs("w", "subject", "delete")
|
||||||
.unwrap();
|
.is_err(),
|
||||||
|
"replacement canonical Purge cannot bypass the pinned unfinished removal"
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
s.finish_subject_deletion_record("w", "subject", "delete")
|
s.finish_subject_deletion_record("w", "subject", "delete")
|
||||||
.is_err(),
|
.is_err(),
|
||||||
@@ -1143,6 +1412,389 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn in_flight(s: &SqliteWorkspaceStore, w: &RuntimeWorkerRef, state: &str, is_job: bool) {
|
||||||
|
registry(s, w);
|
||||||
|
reservation(
|
||||||
|
s,
|
||||||
|
w,
|
||||||
|
"pending",
|
||||||
|
Some(if is_job {
|
||||||
|
"subjektiv-consolidation:subject"
|
||||||
|
} else {
|
||||||
|
"subjektiv:subject"
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
receipt(s, w, state, "archive", "retain");
|
||||||
|
s.with_conn(|c| {
|
||||||
|
c.execute("UPDATE worker_registry SET profile='builtin:old-profile' WHERE worker_id=?1",[&w.worker_id])?;
|
||||||
|
c.execute("UPDATE worker_removal_operations SET archive_id='old-archive',metadata_disposition='purge',archive_retention_kind='for_seconds',archive_retention_seconds=60 WHERE worker_id=?1",[&w.worker_id])?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn execution_result(
|
||||||
|
request: &WorkerRetentionExecutionRequest,
|
||||||
|
) -> WorkerRetentionExecutionResult {
|
||||||
|
WorkerRetentionExecutionResult {
|
||||||
|
operation_id: request.operation_id.clone(),
|
||||||
|
input_fingerprint: request.input_fingerprint.clone(),
|
||||||
|
worker_id: request.worker_id,
|
||||||
|
session_disposition: request.session_disposition,
|
||||||
|
diagnostics_disposition: request.diagnostics_disposition,
|
||||||
|
archive: request.archive_id.as_ref().map(|archive_id| {
|
||||||
|
worker_runtime::retention::WorkerSessionArchiveManifest {
|
||||||
|
schema_version: 1,
|
||||||
|
archive_id: archive_id.clone(),
|
||||||
|
workspace_id: request.workspace_id.clone(),
|
||||||
|
source_runtime_id: request.source_runtime_id.clone(),
|
||||||
|
source_worker_id: request.worker_id,
|
||||||
|
source_session_id: "old-session".into(),
|
||||||
|
source_active_segment_id: None,
|
||||||
|
segment_ids: vec![],
|
||||||
|
source_created_at: request.source_created_at.clone(),
|
||||||
|
removed_at: request.removed_at.clone(),
|
||||||
|
archived_at_unix_seconds: 0,
|
||||||
|
effective_profile: request.effective_profile.clone(),
|
||||||
|
retention_class: request.retention_class.clone(),
|
||||||
|
content_checksum_sha256: "fixture-checksum".into(),
|
||||||
|
content_bytes: 0,
|
||||||
|
content_file_count: 0,
|
||||||
|
policy_id: request.policy_id.clone(),
|
||||||
|
operation_id: request.operation_id.clone(),
|
||||||
|
input_fingerprint: request.input_fingerprint.clone(),
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
source_removed: true,
|
||||||
|
diagnostics_retained: request.diagnostics_disposition == DiagnosticsDisposition::Retain,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn succeed(s: &SqliteWorkspaceStore, request: &WorkerRetentionExecutionRequest) {
|
||||||
|
// Isolated store proof fixture. HTTP/Runtime tests exercise canonical
|
||||||
|
// commit; here success alone must still leave reconciliation incomplete.
|
||||||
|
s.with_conn(|c| {
|
||||||
|
c.execute("UPDATE worker_removal_operations SET state='succeeded' WHERE operation_id=?1",[&request.operation_id])?;
|
||||||
|
c.execute("DELETE FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",params![request.workspace_id,request.source_runtime_id,request.worker_id.to_string()])?;
|
||||||
|
c.execute("UPDATE worker_create_reservations SET state='removed' WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3",params![request.workspace_id,request.source_runtime_id,request.worker_id.to_string()])?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unfinished_removal_pins_full_identity_and_requires_exact_success_before_release() {
|
||||||
|
let s = setup();
|
||||||
|
let w = worker(71);
|
||||||
|
in_flight(&s, &w, "executing", false);
|
||||||
|
// An executing plan may have an older registry epoch. Its exact Runtime
|
||||||
|
// request is still authoritative; do not turn this into permanent stale.
|
||||||
|
s.with_conn(|c| {
|
||||||
|
c.execute(
|
||||||
|
"UPDATE worker_registry SET updated_at='later' WHERE worker_id=?1",
|
||||||
|
[&w.worker_id],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
s.prepare_subject_deletion("w", "subject", "delete", &[])
|
||||||
|
.unwrap();
|
||||||
|
let saved = s.subject_deletion_removal("w", &w).unwrap().unwrap();
|
||||||
|
let req = &saved.request;
|
||||||
|
assert!(!saved.completed);
|
||||||
|
assert!(!saved.is_job);
|
||||||
|
assert_eq!(req.source_created_at, NOW);
|
||||||
|
assert_eq!(req.removed_at, NOW);
|
||||||
|
assert_eq!(
|
||||||
|
req.effective_profile.as_deref(),
|
||||||
|
Some("builtin:old-profile")
|
||||||
|
);
|
||||||
|
assert_eq!(req.retention_class, None);
|
||||||
|
assert_eq!(req.archive_id.as_deref(), Some("old-archive"));
|
||||||
|
assert_eq!(req.policy_id, "test");
|
||||||
|
assert_eq!(saved_prior(&s, "subject", &w), None);
|
||||||
|
s.with_conn(|c| {
|
||||||
|
assert!(removal_reconciliation_matches(c,"w",&w,&req.operation_id,&req.input_fingerprint)?);
|
||||||
|
assert!(!removal_reconciliation_matches(c,"w",&w,&req.operation_id,"changed")?);
|
||||||
|
assert!(!removal_reconciliation_matches(c,"other",&w,&req.operation_id,&req.input_fingerprint)?);
|
||||||
|
assert!(c.execute("UPDATE subject_deletion_removals SET request_json=request_json WHERE workspace_id='w'",[]).is_err());
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
let result = execution_result(req);
|
||||||
|
assert!(
|
||||||
|
s.confirm_subject_deletion_removal("w", &w, req, &result, None)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
succeed(&s, req);
|
||||||
|
assert!(
|
||||||
|
s.current_worker_singleton_owner("w", "subjektiv:subject")
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
s.finish_subject_deletion_record("w", "subject", "delete")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let fence_request = s
|
||||||
|
.subject_deletion_worker_request("w", "subject", "delete", &w)
|
||||||
|
.unwrap();
|
||||||
|
let fence = runtime_api::SubjectDeleteWorkerResponse {
|
||||||
|
workspace_id: fence_request.workspace_id,
|
||||||
|
worker_id: fence_request.worker_id,
|
||||||
|
create_fingerprint: fence_request.create_fingerprint,
|
||||||
|
operation_id: fence_request.operation_id,
|
||||||
|
originals_absent: true,
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
s.confirm_subject_deletion_absence("w", "subject", "delete", &w, &fence)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
for (field, value) in [
|
||||||
|
("operation_id", serde_json::json!("changed")),
|
||||||
|
("input_fingerprint", serde_json::json!("changed")),
|
||||||
|
("archive_id", serde_json::json!("changed")),
|
||||||
|
("workspace_id", serde_json::json!("other")),
|
||||||
|
("source_runtime_id", serde_json::json!("other-runtime")),
|
||||||
|
("worker_id", serde_json::json!(worker(72).worker_id)),
|
||||||
|
("source_created_at", serde_json::json!("changed")),
|
||||||
|
("removed_at", serde_json::json!("changed")),
|
||||||
|
("effective_profile", serde_json::json!("changed")),
|
||||||
|
("retention_class", serde_json::json!("changed")),
|
||||||
|
("policy_id", serde_json::json!("changed")),
|
||||||
|
("session_disposition", serde_json::json!("purge")),
|
||||||
|
("diagnostics_disposition", serde_json::json!("purge")),
|
||||||
|
] {
|
||||||
|
let mut changed = serde_json::to_value(req).unwrap();
|
||||||
|
changed[field] = value;
|
||||||
|
let changed: WorkerRetentionExecutionRequest = serde_json::from_value(changed).unwrap();
|
||||||
|
assert!(
|
||||||
|
s.confirm_subject_deletion_removal("w", &w, &changed, &result, None)
|
||||||
|
.is_err(),
|
||||||
|
"{field}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!s.subject_deletion_removal("w", &w)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (field, value) in [
|
||||||
|
("operation_id", serde_json::json!("changed")),
|
||||||
|
("input_fingerprint", serde_json::json!("changed")),
|
||||||
|
("worker_id", serde_json::json!(worker(72).worker_id)),
|
||||||
|
("session_disposition", serde_json::json!("purge")),
|
||||||
|
("diagnostics_disposition", serde_json::json!("purge")),
|
||||||
|
("source_removed", serde_json::json!(false)),
|
||||||
|
("diagnostics_retained", serde_json::json!(false)),
|
||||||
|
("archive", serde_json::Value::Null),
|
||||||
|
] {
|
||||||
|
let mut changed = serde_json::to_value(&result).unwrap();
|
||||||
|
changed[field] = value;
|
||||||
|
let changed = serde_json::from_value(changed).unwrap();
|
||||||
|
assert!(
|
||||||
|
s.confirm_subject_deletion_removal("w", &w, req, &changed, None)
|
||||||
|
.is_err(),
|
||||||
|
"{field}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
s.confirm_subject_deletion_removal("w", &w, req, &result, None)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
saved_prior(&s, "subject", &w),
|
||||||
|
None,
|
||||||
|
"not a late prior success"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
s.current_worker_singleton_owner("w", "subjektiv:subject")
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
s.confirm_subject_deletion_absence("w", "subject", "delete", &w, &fence)
|
||||||
|
.unwrap();
|
||||||
|
s.erase_subject_deleted_jobs("w", "subject", "delete")
|
||||||
|
.unwrap();
|
||||||
|
s.finish_subject_deletion_record("w", "subject", "delete")
|
||||||
|
.unwrap();
|
||||||
|
s.with_conn(|c| {
|
||||||
|
assert!(removal_reconciliation_matches(
|
||||||
|
c,
|
||||||
|
"w",
|
||||||
|
&w,
|
||||||
|
&req.operation_id,
|
||||||
|
&req.input_fingerprint
|
||||||
|
)?);
|
||||||
|
c.execute(
|
||||||
|
"UPDATE worker_removal_operations SET policy_id='changed' WHERE operation_id=?1",
|
||||||
|
[&req.operation_id],
|
||||||
|
)?;
|
||||||
|
assert!(!completed_removal_matches(c, "w", "subject", &w)?);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
s.finish_subject_deletion_record("w", "subject", "delete")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn job_reconciliation_requires_additional_purge_and_survives_reopen_and_retry() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("snapshot.db");
|
||||||
|
let s = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
seed(&s);
|
||||||
|
let w = worker(81);
|
||||||
|
in_flight(&s, &w, "failed", true);
|
||||||
|
let other = worker(82);
|
||||||
|
registry(&s, &other);
|
||||||
|
reservation(&s, &other, "other-allocation", Some("subjektiv:another"));
|
||||||
|
receipt(&s, &other, "executing", "archive", "retain");
|
||||||
|
s.prepare_subject_deletion("w", "another", "other-delete", &[])
|
||||||
|
.unwrap();
|
||||||
|
let other_before = s.subject_deletion_removal("w", &other).unwrap();
|
||||||
|
s.prepare_subject_deletion("w", "subject", "delete", &[])
|
||||||
|
.unwrap();
|
||||||
|
let req = s
|
||||||
|
.subject_deletion_removal("w", &w)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.request;
|
||||||
|
drop(s);
|
||||||
|
let s = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
s.prepare_subject_deletion("w", "subject", "delete", &[])
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
s.subject_deletion_removal("w", &w)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.request,
|
||||||
|
req
|
||||||
|
);
|
||||||
|
assert!(s.subject_deletion_removal("other", &w).unwrap().is_none());
|
||||||
|
let result = execution_result(&req);
|
||||||
|
let purge = runtime_api::SubjectJobRemovalPurgeResponse {
|
||||||
|
workspace_id: "w".into(),
|
||||||
|
worker_id: req.worker_id,
|
||||||
|
operation_id: req.operation_id.clone(),
|
||||||
|
input_fingerprint: req.input_fingerprint.clone(),
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
s.confirm_subject_deletion_removal("w", &w, &req, &result, Some(&purge))
|
||||||
|
.is_err(),
|
||||||
|
"failed is not success"
|
||||||
|
);
|
||||||
|
succeed(&s, &req);
|
||||||
|
assert!(
|
||||||
|
s.current_worker_singleton_owner("w", "subjektiv-consolidation:subject")
|
||||||
|
.unwrap()
|
||||||
|
.is_some(),
|
||||||
|
"canonical success cannot release Job occupancy before physical purge"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
s.confirm_subject_deletion_removal("w", &w, &req, &result, None)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
s.erase_subject_deleted_jobs("w", "subject", "delete")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
for (field, value) in [
|
||||||
|
("workspace_id", serde_json::json!("other")),
|
||||||
|
("worker_id", serde_json::json!(worker(82).worker_id)),
|
||||||
|
("operation_id", serde_json::json!("changed")),
|
||||||
|
("input_fingerprint", serde_json::json!("changed")),
|
||||||
|
] {
|
||||||
|
let mut changed = serde_json::to_value(&purge).unwrap();
|
||||||
|
changed[field] = value;
|
||||||
|
let changed = serde_json::from_value(changed).unwrap();
|
||||||
|
assert!(
|
||||||
|
s.confirm_subject_deletion_removal("w", &w, &req, &result, Some(&changed))
|
||||||
|
.is_err(),
|
||||||
|
"{field}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
!s.subject_deletion_removal("w", &w)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
s.confirm_subject_deletion_removal("w", &w, &req, &result, Some(&purge))
|
||||||
|
.unwrap();
|
||||||
|
drop(s);
|
||||||
|
let s = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
s.confirm_subject_deletion_removal("w", &w, &req, &result, Some(&purge))
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
s.subject_deletion_removal("w", &w)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.completed
|
||||||
|
);
|
||||||
|
assert_eq!(saved_prior(&s, "subject", &w), None);
|
||||||
|
assert!(
|
||||||
|
s.current_worker_singleton_owner("w", "subjektiv-consolidation:subject")
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
s.with_conn(|c| {
|
||||||
|
let old:(String,String,String) = c.query_row("SELECT state,session_disposition,metadata_disposition FROM worker_removal_operations WHERE operation_id=?1",[&req.operation_id],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?;
|
||||||
|
assert_eq!(old,("succeeded".into(),"archive".into(),"purge".into()));
|
||||||
|
assert!(c.execute("UPDATE subject_deletion_removals SET completed=0",[]).is_err());
|
||||||
|
assert_eq!(c.query_row("SELECT count(*) FROM pragma_foreign_key_check",[],|r|r.get::<_,i64>(0))?,0);
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
s.subject_deletion_removal("w", &other).unwrap(),
|
||||||
|
other_before
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
s.current_worker_singleton_owner("w", "subjektiv:another")
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
s.erase_subject_deleted_jobs("w", "subject", "delete")
|
||||||
|
.unwrap();
|
||||||
|
s.finish_subject_deletion_record("w", "subject", "delete")
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn planned_pin_uses_current_epoch_and_never_replaces_prior_success() {
|
||||||
|
for (state, current, expected) in [
|
||||||
|
("planned", true, true),
|
||||||
|
("planned", false, false),
|
||||||
|
("failed", false, true),
|
||||||
|
] {
|
||||||
|
let s = setup();
|
||||||
|
let w = worker(91);
|
||||||
|
in_flight(&s, &w, state, false);
|
||||||
|
if !current {
|
||||||
|
s.with_conn(|c| {
|
||||||
|
c.execute("UPDATE worker_registry SET updated_at='later'", [])?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
s.prepare_subject_deletion("w", "subject", "delete", &[])
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
s.subject_deletion_removal("w", &w).unwrap().is_some(),
|
||||||
|
expected,
|
||||||
|
"{state}/{current}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let s = setup();
|
||||||
|
let w = worker(92);
|
||||||
|
in_flight(&s, &w, "executing", true);
|
||||||
|
receipt(&s, &w, "succeeded", "archive", "retain");
|
||||||
|
s.prepare_subject_deletion("w", "subject", "delete", &[])
|
||||||
|
.unwrap();
|
||||||
|
assert!(saved_prior(&s, "subject", &w).is_some());
|
||||||
|
assert!(s.subject_deletion_removal("w", &w).unwrap().is_none());
|
||||||
|
s.erase_subject_deleted_jobs("w", "subject", "delete")
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn saved_prior_receipt_must_remain_the_exact_success_not_a_replacement() {
|
fn saved_prior_receipt_must_remain_the_exact_success_not_a_replacement() {
|
||||||
let s = setup();
|
let s = setup();
|
||||||
|
|||||||
@@ -96,6 +96,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
|||||||
"worker_control_grants",
|
"worker_control_grants",
|
||||||
// Snapshot children precede their fence parent and canonical Worker removal
|
// Snapshot children precede their fence parent and canonical Worker removal
|
||||||
// receipts (subject_deletion_workers.prior_removal_operation_id).
|
// receipts (subject_deletion_workers.prior_removal_operation_id).
|
||||||
|
"subject_deletion_removals",
|
||||||
"subject_deletion_allocations",
|
"subject_deletion_allocations",
|
||||||
"subject_deletion_jobs",
|
"subject_deletion_jobs",
|
||||||
"subject_deletion_workers",
|
"subject_deletion_workers",
|
||||||
@@ -1286,6 +1287,7 @@ mod tests {
|
|||||||
fn workspace_purge_removes_subject_deletion_snapshots_and_preserves_other_workspace() {
|
fn workspace_purge_removes_subject_deletion_snapshots_and_preserves_other_workspace() {
|
||||||
let (store, owner, workspace_id) = setup();
|
let (store, owner, workspace_id) = setup();
|
||||||
let tables = [
|
let tables = [
|
||||||
|
"subject_deletion_removals",
|
||||||
"subject_deletion_allocations",
|
"subject_deletion_allocations",
|
||||||
"subject_deletion_jobs",
|
"subject_deletion_jobs",
|
||||||
"subject_deletion_workers",
|
"subject_deletion_workers",
|
||||||
@@ -1299,9 +1301,11 @@ mod tests {
|
|||||||
.position(|name| *name == table)
|
.position(|name| *name == table)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
};
|
};
|
||||||
for child in &tables[..3] {
|
for child in &tables[..4] {
|
||||||
assert!(position(*child) < position("subject_deletions"));
|
assert!(position(*child) < position("subject_deletions"));
|
||||||
}
|
}
|
||||||
|
assert!(position("subject_deletion_removals") < position("subject_deletion_workers"));
|
||||||
|
assert!(position("subject_deletion_removals") < position("worker_removal_operations"));
|
||||||
assert!(position("subject_deletion_workers") < position("worker_removal_operations"));
|
assert!(position("subject_deletion_workers") < position("worker_removal_operations"));
|
||||||
for workspace in ["workspace-a", "workspace-b"] {
|
for workspace in ["workspace-a", "workspace-b"] {
|
||||||
store.with_conn(|conn| {
|
store.with_conn(|conn| {
|
||||||
@@ -1310,11 +1314,16 @@ mod tests {
|
|||||||
worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition,
|
worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition,
|
||||||
archive_retention_kind,diagnostics_disposition,blockers_json,state,reason,created_at,updated_at)
|
archive_retention_kind,diagnostics_disposition,blockers_json,state,reason,created_at,updated_at)
|
||||||
VALUES(?1,?1,'fingerprint',?1,'runtime','worker','1','policy','digest',
|
VALUES(?1,?1,'fingerprint',?1,'runtime','worker','1','policy','digest',
|
||||||
'purge','purge','forever','purge','[]','succeeded','removed','1','1')", [workspace])?;
|
'purge','purge','forever','purge','[]','executing','removed','1','1')", [workspace])?;
|
||||||
conn.execute("INSERT INTO subject_deletions(workspace_id,subject_id,operation_id,completed,jobs_erased)
|
conn.execute("INSERT INTO subject_deletions(workspace_id,subject_id,operation_id,completed,jobs_erased)
|
||||||
VALUES(?1,'subject','subject-delete',1,1)", [workspace])?;
|
VALUES(?1,'subject','subject-delete',1,1)", [workspace])?;
|
||||||
conn.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id,originals_absent)
|
conn.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id,originals_absent)
|
||||||
VALUES(?1,'subject','runtime','worker',1,?1,1)", [workspace])?;
|
VALUES(?1,'subject','runtime','worker',1,NULL,0)", [workspace])?;
|
||||||
|
conn.execute("INSERT INTO subject_deletion_removals(workspace_id,subject_id,runtime_id,worker_id,operation_id,input_fingerprint,request_json)
|
||||||
|
VALUES(?1,'subject','runtime','worker',?1,'fingerprint',json_object('operation_id',?1,'input_fingerprint','fingerprint'))", [workspace])?;
|
||||||
|
// Settle canonical removal so Workspace deletion's independent
|
||||||
|
// preflight is unblocked; the reconciliation row still needs purge.
|
||||||
|
conn.execute("UPDATE worker_removal_operations SET state='succeeded' WHERE operation_id=?1", [workspace])?;
|
||||||
conn.execute("INSERT INTO subject_deletion_jobs VALUES(?1,'subject','job')", [workspace])?;
|
conn.execute("INSERT INTO subject_deletion_jobs VALUES(?1,'subject','job')", [workspace])?;
|
||||||
conn.execute("INSERT INTO subject_deletion_allocations VALUES(?1,'subject','allocation')", [workspace])?;
|
conn.execute("INSERT INTO subject_deletion_allocations VALUES(?1,'subject','allocation')", [workspace])?;
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -91,7 +91,7 @@
|
|||||||
{#if phase === 'confirming'}
|
{#if phase === 'confirming'}
|
||||||
<form onsubmit={(event) => { event.preventDefault(); void submit(); }}>
|
<form onsubmit={(event) => { event.preventDefault(); void submit(); }}>
|
||||||
<p><strong>This is irreversible.</strong> This Subject, its behavior, Memory, change history, staging, Surface, Journal, permissions, delegations, Writer/Session associations, and Subject Job inputs/results will be deleted. Connected and delegated Workers and Jobs must stop before completion can be confirmed.</p>
|
<p><strong>This is irreversible.</strong> This Subject, its behavior, Memory, change history, staging, Surface, Journal, permissions, delegations, Writer/Session associations, and Subject Job inputs/results will be deleted. Connected and delegated Workers and Jobs must stop before completion can be confirmed.</p>
|
||||||
<p>Original Worker/Session text, existing archives and Workdirs, Tickets, and other separately retained resources are <strong>not deleted</strong>. Minimal deletion retry/audit metadata is retained. Other Subjects and Workspaces are not affected.</p>
|
<p>Original Worker/Session text, existing archives and Workdirs, Tickets, and other separately retained resources are <strong>not deleted</strong>. Minimal deletion retry/audit metadata is retained. Any earlier Worker retention request still follows its original disposition. Other Subjects and Workspaces are not affected.</p>
|
||||||
<label class="delete-confirm"><input type="checkbox" bind:checked={confirmed} /> I have checked the target, deletion scope, retained data, and irreversible effect.</label>
|
<label class="delete-confirm"><input type="checkbox" bind:checked={confirmed} /> I have checked the target, deletion scope, retained data, and irreversible effect.</label>
|
||||||
<div class="delete-actions">
|
<div class="delete-actions">
|
||||||
<button type="submit" class="workspace-danger-button" disabled={!confirmed}>Permanently delete Subject</button>
|
<button type="submit" class="workspace-danger-button" disabled={!confirmed}>Permanently delete Subject</button>
|
||||||
|
|||||||
Reference in New Issue
Block a user