fix: reconcile unfinished retention before Subject deletion
This commit is contained in:
@@ -2104,7 +2104,9 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
|
||||
}
|
||||
if matches!(
|
||||
path,
|
||||
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
|
||||
"/v1/subject-create-recovery"
|
||||
| "/v1/subject-delete-worker"
|
||||
| "/v1/subject-job-removal-purge"
|
||||
) && *method == Method::POST
|
||||
{
|
||||
return Some("workers:delete");
|
||||
|
||||
@@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
}
|
||||
}
|
||||
|
||||
async fn purge_subject_job_removal(
|
||||
&self,
|
||||
request: runtime_api::SubjectJobRemovalPurgeRequest,
|
||||
) -> Result<runtime_api::SubjectJobRemovalPurgeResponse, runtime_api::RuntimeApiError> {
|
||||
let scope = material_scope(&self.state, &request.deletion.workspace_id)?;
|
||||
self.state
|
||||
.runtime
|
||||
.purge_subject_job_removal_scoped(&scope, request)
|
||||
.map_err(RuntimeHttpRestError::runtime)
|
||||
.map_err(api_error)
|
||||
}
|
||||
|
||||
async fn subject_delete_worker(
|
||||
&self,
|
||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
//! verifier are connected. Only controller execution is replaced by AcceptingBackend;
|
||||
//! no model, external server, user credentials or running Runtime is used.
|
||||
use super::*;
|
||||
#[path = "subject_job_removal_purge_tests.rs"]
|
||||
mod subject_job_removal_purge_tests;
|
||||
use crate::retention::{DiagnosticsDisposition, SessionDisposition, runtime_session_retention};
|
||||
use runtime_api::{
|
||||
SessionMaterialCatalogAvailability as Catalog, SessionMaterialReadAvailability as Read,
|
||||
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
//! Real filesystem failures and reopened durable fences for the additional purge.
|
||||
//! Uses the same production stores/router fixture as the copy-erasure suite.
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn subject_job_removal_purge_partial_physical_failure_keeps_fence_and_retry_erases_after_reopen() {
|
||||
let f = Fixture::new();
|
||||
let mut request = f.removal_purge_request(SessionDisposition::Archive);
|
||||
request.removal.diagnostics_disposition = runtime_api::DiagnosticsDisposition::Retain;
|
||||
f.complete_removal(&request);
|
||||
let receipt_path = f.receipt_path();
|
||||
let receipt = std::fs::read(&receipt_path).unwrap();
|
||||
let session_root = f
|
||||
.temp
|
||||
.path()
|
||||
.join("archives/workers/canonical-archive/session");
|
||||
let parked = f.temp.path().join("interrupted-session-fixture");
|
||||
std::fs::rename(&session_root, &parked).unwrap();
|
||||
std::fs::write(&session_root, b"deterministic directory removal failure").unwrap();
|
||||
assert!(
|
||||
f.runtime
|
||||
.purge_subject_job_removal_scoped(&f.scope, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(matches!(
|
||||
runtime_session_retention(f.temp.path()).ensure_live(f.session),
|
||||
Err(session_store::StoreError::SessionErased(_))
|
||||
));
|
||||
assert!(
|
||||
!f.temp
|
||||
.path()
|
||||
.join("archives/diagnostics/remove-source")
|
||||
.exists()
|
||||
);
|
||||
assert_eq!(std::fs::read(&receipt_path).unwrap(), receipt);
|
||||
std::fs::remove_file(&session_root).unwrap();
|
||||
std::fs::rename(parked, &session_root).unwrap();
|
||||
let Fixture {
|
||||
temp,
|
||||
runtime,
|
||||
scope,
|
||||
store,
|
||||
session,
|
||||
..
|
||||
} = f;
|
||||
drop(store);
|
||||
drop(runtime);
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||
crate::FsRuntimeStoreOptions::new(temp.path()).with_runtime_id("runtime-test"),
|
||||
Arc::new(AcceptingBackend),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.bind_runtime_identity("runtime-test").unwrap();
|
||||
runtime
|
||||
.purge_subject_job_removal_scoped(&scope, request)
|
||||
.unwrap();
|
||||
assert!(!session_root.exists());
|
||||
assert!(matches!(
|
||||
runtime_session_retention(temp.path()).lock_live(session),
|
||||
Err(session_store::StoreError::SessionErased(_))
|
||||
));
|
||||
assert_eq!(std::fs::read(receipt_path).unwrap(), receipt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_job_removal_purge_unconfirmed_durable_deletion_fence_preserves_copies_until_reconciled()
|
||||
{
|
||||
let f = Fixture::new();
|
||||
let request = f.removal_purge_request(SessionDisposition::Archive);
|
||||
f.complete_removal(&request);
|
||||
let copies = f.copies();
|
||||
let receipt_path = f.receipt_path();
|
||||
let receipt = std::fs::read(&receipt_path).unwrap();
|
||||
let Fixture {
|
||||
temp,
|
||||
runtime,
|
||||
scope,
|
||||
store,
|
||||
..
|
||||
} = f;
|
||||
drop(store);
|
||||
drop(runtime);
|
||||
// Frozen durable crash-window fixture: permanent admission exclusion was
|
||||
// saved, but no confirmed execution-shutdown response was committed.
|
||||
let snapshot_path = temp.path().join("runtime.json");
|
||||
let mut snapshot: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&snapshot_path).unwrap()).unwrap();
|
||||
snapshot["subject_delete_worker_fences"][request.deletion.worker_id.to_string()]["response"] =
|
||||
serde_json::Value::Null;
|
||||
std::fs::write(&snapshot_path, serde_json::to_vec(&snapshot).unwrap()).unwrap();
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||
crate::FsRuntimeStoreOptions::new(temp.path()).with_runtime_id("runtime-test"),
|
||||
Arc::new(AcceptingBackend),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.bind_runtime_identity("runtime-test").unwrap();
|
||||
assert!(
|
||||
runtime
|
||||
.purge_subject_job_removal_scoped(&scope, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(tree(&temp.path().join("archives")), copies);
|
||||
assert_eq!(std::fs::read(&receipt_path).unwrap(), receipt);
|
||||
runtime
|
||||
.subject_delete_worker_scoped(&scope, request.deletion.clone())
|
||||
.unwrap();
|
||||
runtime
|
||||
.purge_subject_job_removal_scoped(&scope, request)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!temp
|
||||
.path()
|
||||
.join("archives/workers/canonical-archive/session")
|
||||
.exists()
|
||||
);
|
||||
assert_eq!(std::fs::read(receipt_path).unwrap(), receipt);
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
//! Additional removal-copy erasure uses the real Runtime, retention provider,
|
||||
//! Session tombstone, filesystem and signed HTTP router. Only execution is fake
|
||||
//! (the parent fixture's AcceptingBackend); no Backend recovery is simulated here.
|
||||
use super::*;
|
||||
#[path = "subject_job_removal_purge_recovery_tests.rs"]
|
||||
mod recovery;
|
||||
use runtime_api::{SubjectDeleteWorkerRequest, SubjectJobRemovalPurgeRequest};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
fn tree(root: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
|
||||
fn walk(root: &Path, at: &Path, files: &mut BTreeMap<PathBuf, Vec<u8>>) {
|
||||
if !at.exists() {
|
||||
return;
|
||||
}
|
||||
for entry in std::fs::read_dir(at).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
if entry.file_type().unwrap().is_dir() {
|
||||
walk(root, &entry.path(), files);
|
||||
} else {
|
||||
files.insert(
|
||||
entry.path().strip_prefix(root).unwrap().into(),
|
||||
std::fs::read(entry.path()).unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut files = BTreeMap::new();
|
||||
walk(root, root, &mut files);
|
||||
files
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn deletion_request(&self) -> SubjectDeleteWorkerRequest {
|
||||
SubjectDeleteWorkerRequest {
|
||||
workspace_id: self.scope.workspace_id.clone(),
|
||||
worker_id: self.worker.worker_id,
|
||||
create_fingerprint: task_request("identity only").create_fingerprint,
|
||||
operation_id: "subject-deletion".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn removal_purge_request(
|
||||
&self,
|
||||
disposition: SessionDisposition,
|
||||
) -> SubjectJobRemovalPurgeRequest {
|
||||
let removal = self.retention_request(disposition);
|
||||
SubjectJobRemovalPurgeRequest {
|
||||
deletion: self.deletion_request(),
|
||||
removal: serde_json::from_value(serde_json::to_value(removal).unwrap()).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
fn complete_removal(
|
||||
&self,
|
||||
request: &SubjectJobRemovalPurgeRequest,
|
||||
) -> crate::retention::WorkerRetentionExecutionResult {
|
||||
let diagnostics = self.root().parent().unwrap().join("runs/attempt");
|
||||
std::fs::create_dir_all(&diagnostics).unwrap();
|
||||
std::fs::write(diagnostics.join("worker.out.log"), b"Subject Job output").unwrap();
|
||||
self.runtime
|
||||
.subject_delete_worker_scoped(&self.scope, request.deletion.clone())
|
||||
.unwrap();
|
||||
let removal =
|
||||
serde_json::from_value(serde_json::to_value(&request.removal).unwrap()).unwrap();
|
||||
self.runtime.execute_worker_retention(&removal).unwrap()
|
||||
}
|
||||
|
||||
fn receipt_path(&self) -> PathBuf {
|
||||
self.temp
|
||||
.path()
|
||||
.join("retention/operations/remove-source.json")
|
||||
}
|
||||
|
||||
fn copies(&self) -> BTreeMap<PathBuf, Vec<u8>> {
|
||||
tree(&self.temp.path().join("archives"))
|
||||
}
|
||||
|
||||
fn independent_worker(&self) -> WorkerRef {
|
||||
let mut request = task_request("independent Worker");
|
||||
request.config_bundle = None;
|
||||
request.memory_settings.as_mut().unwrap().workspace_id = self.scope.workspace_id.clone();
|
||||
self.runtime
|
||||
.create_worker_scoped(&self.scope, request)
|
||||
.unwrap()
|
||||
.worker_ref
|
||||
}
|
||||
|
||||
// Independent successful snapshot made before the Subject deletion starts.
|
||||
fn prior_archive(&self) -> WorkerRef {
|
||||
let worker = self.independent_worker();
|
||||
let aggregate = self
|
||||
.temp
|
||||
.path()
|
||||
.join("workers")
|
||||
.join(worker.worker_id.to_string());
|
||||
let session = session_store::new_session_id();
|
||||
let segment = session_store::new_segment_id();
|
||||
WorkerSessionStore::new(aggregate.join("session"))
|
||||
.unwrap()
|
||||
.create_segment(
|
||||
session,
|
||||
segment,
|
||||
&[LogEntry::AnnotatedSegmentStart {
|
||||
ts: 1,
|
||||
session_id: session,
|
||||
system_prompt: None,
|
||||
config: Default::default(),
|
||||
history: vec![],
|
||||
forked_from: None,
|
||||
compacted_from: None,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
WorkerAggregateStore::new(&aggregate, format!("worker-runtime-{}", worker.worker_id))
|
||||
.unwrap()
|
||||
.write(
|
||||
&session_store::WorkerMetadata::new(
|
||||
format!("worker-runtime-{}", worker.worker_id),
|
||||
Some(session_store::WorkerActiveSegmentRef::active_segment(
|
||||
session, segment,
|
||||
)),
|
||||
)
|
||||
.with_workspace_id(&self.scope.workspace_id),
|
||||
)
|
||||
.unwrap();
|
||||
self.runtime
|
||||
.stop_worker_scoped(&self.scope, &worker, None)
|
||||
.unwrap();
|
||||
let mut removal = self.retention_request(SessionDisposition::Archive);
|
||||
removal.worker_id = worker.worker_id;
|
||||
removal.operation_id = "prior-removal".into();
|
||||
removal.archive_id = Some("prior-archive".into());
|
||||
removal.input_fingerprint = "prior-fingerprint".into();
|
||||
self.runtime.execute_worker_retention(&removal).unwrap();
|
||||
worker
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_job_removal_purge_erases_copies_preserves_receipt_and_prior_success_after_reopen() {
|
||||
for disposition in [SessionDisposition::Archive, SessionDisposition::Purge] {
|
||||
let f = Fixture::new();
|
||||
f.prior_archive();
|
||||
let prior = tree(&f.temp.path().join("archives/workers/prior-archive"));
|
||||
assert!(!prior.is_empty());
|
||||
let unrelated = f.independent_worker();
|
||||
let unrelated_root = f
|
||||
.temp
|
||||
.path()
|
||||
.join("workers")
|
||||
.join(unrelated.worker_id.to_string());
|
||||
let unrelated_before = tree(&unrelated_root);
|
||||
let mut request = f.removal_purge_request(disposition);
|
||||
request.removal.diagnostics_disposition = runtime_api::DiagnosticsDisposition::Retain;
|
||||
let original_result = f.complete_removal(&request);
|
||||
let receipt = std::fs::read(f.receipt_path()).unwrap();
|
||||
let archive = f.temp.path().join("archives/workers/canonical-archive");
|
||||
let diagnostics = f.temp.path().join("archives/diagnostics/remove-source");
|
||||
assert!(diagnostics.is_dir());
|
||||
let manifest = (disposition == SessionDisposition::Archive)
|
||||
.then(|| std::fs::read(archive.join("manifest.json")).unwrap());
|
||||
let archive_staging = f
|
||||
.temp
|
||||
.path()
|
||||
.join("archives/workers/.canonical-archive.tmp-123-1");
|
||||
if let Some(manifest) = &manifest {
|
||||
std::fs::create_dir_all(archive_staging.join("session/segments")).unwrap();
|
||||
std::fs::write(archive_staging.join("manifest.json"), manifest).unwrap();
|
||||
std::fs::write(
|
||||
archive_staging.join("session/segments/interrupted.jsonl"),
|
||||
b"interrupted Subject Job copy",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
// Diagnostics copying precedes publication of its manifest. Interrupted
|
||||
// staging is still bound to the exact removal operation name.
|
||||
let diagnostics_staging = f
|
||||
.temp
|
||||
.path()
|
||||
.join("archives/diagnostics/.remove-source.tmp-123-2");
|
||||
std::fs::create_dir_all(&diagnostics_staging).unwrap();
|
||||
std::fs::write(
|
||||
diagnostics_staging.join("worker.out.log"),
|
||||
b"interrupted diagnostic copy",
|
||||
)
|
||||
.unwrap();
|
||||
for _ in 0..2 {
|
||||
let response = f
|
||||
.runtime
|
||||
.purge_subject_job_removal_scoped(&f.scope, request.clone())
|
||||
.unwrap();
|
||||
assert_eq!(response.operation_id, request.removal.operation_id);
|
||||
assert_eq!(
|
||||
response.input_fingerprint,
|
||||
request.removal.input_fingerprint
|
||||
);
|
||||
assert_eq!(response.worker_id, request.removal.worker_id);
|
||||
assert_eq!(response.workspace_id, request.removal.workspace_id);
|
||||
assert!(!diagnostics.exists());
|
||||
assert!(!diagnostics_staging.exists());
|
||||
assert!(!archive_staging.join("session").exists());
|
||||
assert!(!archive.join("session").exists());
|
||||
assert_eq!(std::fs::read(f.receipt_path()).unwrap(), receipt);
|
||||
if let Some(manifest) = &manifest {
|
||||
assert_eq!(
|
||||
std::fs::read(archive.join("manifest.json")).unwrap(),
|
||||
*manifest
|
||||
);
|
||||
assert!(matches!(
|
||||
runtime_session_retention(f.temp.path()).ensure_live(f.session),
|
||||
Err(session_store::StoreError::SessionErased(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
tree(&f.temp.path().join("archives/workers/prior-archive")),
|
||||
prior
|
||||
);
|
||||
assert_eq!(tree(&unrelated_root), unrelated_before);
|
||||
let Fixture {
|
||||
temp,
|
||||
runtime,
|
||||
scope,
|
||||
store,
|
||||
session,
|
||||
..
|
||||
} = f;
|
||||
drop(store);
|
||||
drop(runtime);
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||
crate::FsRuntimeStoreOptions::new(temp.path()).with_runtime_id("runtime-test"),
|
||||
Arc::new(AcceptingBackend),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.bind_runtime_identity("runtime-test").unwrap();
|
||||
// Lost response/restart is an exact retry, not a new erasure operation.
|
||||
runtime
|
||||
.purge_subject_job_removal_scoped(&scope, request.clone())
|
||||
.unwrap();
|
||||
let removal =
|
||||
serde_json::from_value(serde_json::to_value(&request.removal).unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
runtime.execute_worker_retention(&removal).unwrap(),
|
||||
original_result
|
||||
);
|
||||
assert!(!archive.join("session").exists());
|
||||
assert!(!diagnostics.exists());
|
||||
assert_eq!(
|
||||
std::fs::read(temp.path().join("retention/operations/remove-source.json")).unwrap(),
|
||||
receipt
|
||||
);
|
||||
assert_eq!(
|
||||
tree(&temp.path().join("archives/workers/prior-archive")),
|
||||
prior
|
||||
);
|
||||
assert_eq!(tree(&unrelated_root), unrelated_before);
|
||||
if disposition == SessionDisposition::Archive {
|
||||
assert!(matches!(
|
||||
runtime_session_retention(temp.path()).lock_live(session),
|
||||
Err(session_store::StoreError::SessionErased(_))
|
||||
));
|
||||
assert!(
|
||||
runtime
|
||||
.session_public_material_reader_scoped(
|
||||
&scope,
|
||||
&Source::Archived {
|
||||
archive_id: "canonical-archive".into()
|
||||
},
|
||||
&session.to_string()
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_job_removal_purge_requires_complete_old_request_and_confirmed_exact_fence() {
|
||||
let f = Fixture::new();
|
||||
let request = f.removal_purge_request(SessionDisposition::Archive);
|
||||
let copies = f.copies();
|
||||
assert!(
|
||||
f.runtime
|
||||
.purge_subject_job_removal_scoped(&f.scope, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(f.copies(), copies);
|
||||
f.complete_removal(&request);
|
||||
let copies = f.copies();
|
||||
let receipt = std::fs::read(f.receipt_path()).unwrap();
|
||||
let original = serde_json::to_value(&request).unwrap();
|
||||
// Every saved request field is bound, not just operation/fingerprint. These
|
||||
// are external serialized requests, including the pre-policy-change values.
|
||||
for (section, field, value) in [
|
||||
(
|
||||
"deletion",
|
||||
"workspace_id",
|
||||
serde_json::json!("foreign-workspace"),
|
||||
),
|
||||
(
|
||||
"deletion",
|
||||
"worker_id",
|
||||
serde_json::json!(WorkerId::now_v7()),
|
||||
),
|
||||
(
|
||||
"deletion",
|
||||
"create_fingerprint",
|
||||
serde_json::json!("foreign-create"),
|
||||
),
|
||||
(
|
||||
"deletion",
|
||||
"operation_id",
|
||||
serde_json::json!("foreign-deletion"),
|
||||
),
|
||||
(
|
||||
"removal",
|
||||
"operation_id",
|
||||
serde_json::json!("unknown-removal"),
|
||||
),
|
||||
(
|
||||
"removal",
|
||||
"input_fingerprint",
|
||||
serde_json::json!("foreign-fingerprint"),
|
||||
),
|
||||
(
|
||||
"removal",
|
||||
"archive_id",
|
||||
serde_json::json!("foreign-archive"),
|
||||
),
|
||||
(
|
||||
"removal",
|
||||
"workspace_id",
|
||||
serde_json::json!("foreign-workspace"),
|
||||
),
|
||||
(
|
||||
"removal",
|
||||
"source_runtime_id",
|
||||
serde_json::json!("foreign-runtime"),
|
||||
),
|
||||
(
|
||||
"removal",
|
||||
"worker_id",
|
||||
serde_json::json!(WorkerId::now_v7()),
|
||||
),
|
||||
("removal", "source_created_at", serde_json::json!("changed")),
|
||||
("removal", "removed_at", serde_json::json!("changed")),
|
||||
("removal", "effective_profile", serde_json::json!("changed")),
|
||||
("removal", "retention_class", serde_json::json!("changed")),
|
||||
("removal", "policy_id", serde_json::json!("new-policy")),
|
||||
("removal", "session_disposition", serde_json::json!("purge")),
|
||||
(
|
||||
"removal",
|
||||
"diagnostics_disposition",
|
||||
serde_json::json!("retain"),
|
||||
),
|
||||
] {
|
||||
let mut changed = original.clone();
|
||||
changed[section][field] = value;
|
||||
let changed = serde_json::from_value(changed).unwrap();
|
||||
assert!(
|
||||
f.runtime
|
||||
.purge_subject_job_removal_scoped(&f.scope, changed)
|
||||
.is_err(),
|
||||
"{section}.{field}"
|
||||
);
|
||||
assert_eq!(f.copies(), copies, "{section}.{field}");
|
||||
assert_eq!(std::fs::read(f.receipt_path()).unwrap(), receipt);
|
||||
}
|
||||
for scope in [
|
||||
RuntimeWorkspaceScope::new("workspace-b", "https://backend.test"),
|
||||
RuntimeWorkspaceScope::new("workspace-a", "https://other-backend.test"),
|
||||
] {
|
||||
assert!(
|
||||
f.runtime
|
||||
.purge_subject_job_removal_scoped(&scope, request.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(f.copies(), copies);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_job_removal_purge_missing_pending_or_foreign_receipt_preserves_copies() {
|
||||
for kind in [
|
||||
"missing",
|
||||
"pending",
|
||||
"foreign-worker",
|
||||
"foreign-disposition",
|
||||
] {
|
||||
let f = Fixture::new();
|
||||
let mut request = f.removal_purge_request(SessionDisposition::Archive);
|
||||
request.removal.diagnostics_disposition = runtime_api::DiagnosticsDisposition::Retain;
|
||||
f.complete_removal(&request);
|
||||
let path = f.receipt_path();
|
||||
let mut receipt: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
|
||||
match kind {
|
||||
"missing" => std::fs::remove_file(&path).unwrap(),
|
||||
_ => {
|
||||
match kind {
|
||||
"pending" => receipt["result"]["source_removed"] = false.into(),
|
||||
"foreign-worker" => {
|
||||
receipt["result"]["worker_id"] = serde_json::json!(WorkerId::now_v7())
|
||||
}
|
||||
_ => receipt["result"]["session_disposition"] = "purge".into(),
|
||||
}
|
||||
std::fs::write(&path, serde_json::to_vec(&receipt).unwrap()).unwrap();
|
||||
}
|
||||
}
|
||||
let copies = f.copies();
|
||||
let retained_receipt = std::fs::read(&path).ok();
|
||||
assert!(
|
||||
f.runtime
|
||||
.purge_subject_job_removal_scoped(&f.scope, request)
|
||||
.is_err(),
|
||||
"{kind}"
|
||||
);
|
||||
assert_eq!(f.copies(), copies, "{kind}");
|
||||
assert_eq!(std::fs::read(path).ok(), retained_receipt);
|
||||
runtime_session_retention(f.temp.path())
|
||||
.ensure_live(f.session)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subject_job_removal_purge_signed_http_requires_delete_and_matching_scope() {
|
||||
let f = Fixture::new();
|
||||
let mut request = f.removal_purge_request(SessionDisposition::Archive);
|
||||
request.removal.diagnostics_disposition = runtime_api::DiagnosticsDisposition::Retain;
|
||||
f.complete_removal(&request);
|
||||
let identity = RuntimeIdentityMaterial::generate("purge-key").unwrap();
|
||||
let trust = workspace_trust(&identity);
|
||||
let app = runtime_http_router_with_workspace_auth(
|
||||
f.runtime.clone(),
|
||||
None,
|
||||
workspace_test_auth(vec![trust.clone()]),
|
||||
);
|
||||
let path = "/v1/subject-job-removal-purge";
|
||||
let copies = f.copies();
|
||||
for operation in ["workers:read", "workers:stop", "workers:create"] {
|
||||
let body = serde_json::to_vec(&request).unwrap();
|
||||
let mut claims = signed_claims(&trust, "POST", path, &body);
|
||||
claims.operation = operation.into();
|
||||
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(f.copies(), copies);
|
||||
}
|
||||
let mut foreign = request.clone();
|
||||
foreign.deletion.workspace_id = "foreign-workspace".into();
|
||||
let body = serde_json::to_vec(&foreign).unwrap();
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||
assert_eq!(f.copies(), copies);
|
||||
for _ in 0..2 {
|
||||
let body = serde_json::to_vec(&request).unwrap();
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
assert_eq!(claims.operation, "workers:delete");
|
||||
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let bytes = axum::body::to_bytes(response.into_body(), 16384)
|
||||
.await
|
||||
.unwrap();
|
||||
let response: runtime_api::SubjectJobRemovalPurgeResponse =
|
||||
serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(response.operation_id, request.removal.operation_id);
|
||||
assert_eq!(
|
||||
response.input_fingerprint,
|
||||
request.removal.input_fingerprint
|
||||
);
|
||||
assert!(
|
||||
!f.temp
|
||||
.path()
|
||||
.join("archives/workers/canonical-archive/session")
|
||||
.exists()
|
||||
);
|
||||
assert!(
|
||||
!f.temp
|
||||
.path()
|
||||
.join("archives/diagnostics/remove-source")
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -359,28 +359,91 @@ fn erase_archived_diagnostics(
|
||||
return Err(invalid());
|
||||
}
|
||||
}
|
||||
erase_operation_diagnostics(
|
||||
runtime_root,
|
||||
&archive.workspace_id,
|
||||
&archive.source_runtime_id,
|
||||
archive.source_worker_id,
|
||||
&archive.operation_id,
|
||||
&archive.input_fingerprint,
|
||||
)
|
||||
}
|
||||
|
||||
fn erase_removal_session_copies(
|
||||
runtime_root: &Path,
|
||||
archive: &WorkerSessionArchiveManifest,
|
||||
) -> Result<(), session_store::StoreError> {
|
||||
let parent = runtime_root.join("archives/workers");
|
||||
let prefix = format!(".{}.tmp-", archive.archive_id);
|
||||
for entry in fs::read_dir(&parent)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else { continue };
|
||||
if name != archive.archive_id && !is_operation_staging(name, &prefix) {
|
||||
continue;
|
||||
}
|
||||
if !entry.file_type()?.is_dir() {
|
||||
return Err(std::io::Error::other("Removal archive source is not a directory").into());
|
||||
}
|
||||
let mut saved: WorkerSessionArchiveManifest =
|
||||
serde_json::from_slice(&fs::read(entry.path().join("manifest.json"))?)?;
|
||||
// Each interrupted attempt attests the same input, with its own time.
|
||||
saved.archived_at_unix_seconds = archive.archived_at_unix_seconds;
|
||||
if saved != *archive {
|
||||
return Err(std::io::Error::other("Removal archive copy identity mismatch").into());
|
||||
}
|
||||
match fs::remove_dir_all(entry.path().join("session")) {
|
||||
Ok(()) => (),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => (),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
sync_directory(&entry.path(), "erase removal Session")
|
||||
.map_err(|error| std::io::Error::other(error.to_string()))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_operation_staging(name: &str, prefix: &str) -> bool {
|
||||
name.strip_prefix(prefix).is_some_and(|suffix| {
|
||||
suffix.split_once('-').is_some_and(|(pid, sequence)| {
|
||||
!pid.is_empty()
|
||||
&& !sequence.is_empty()
|
||||
&& pid.bytes().all(|b| b.is_ascii_digit())
|
||||
&& sequence.bytes().all(|b| b.is_ascii_digit())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn erase_operation_diagnostics(
|
||||
runtime_root: &Path,
|
||||
workspace_id: &str,
|
||||
source_runtime_id: &str,
|
||||
worker_id: WorkerId,
|
||||
operation_id: &str,
|
||||
input_fingerprint: &str,
|
||||
) -> Result<(), session_store::StoreError> {
|
||||
let invalid = || {
|
||||
session_store::StoreError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"Archived diagnostics erasure identity mismatch",
|
||||
))
|
||||
};
|
||||
validate_id("operation_id", operation_id).map_err(|_| invalid())?;
|
||||
let parent = runtime_root.join("archives/diagnostics");
|
||||
let entries = match fs::read_dir(&parent) {
|
||||
Ok(entries) => entries,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let staging_prefix = format!(".{}.tmp-", archive.operation_id);
|
||||
let staging_prefix = format!(".{operation_id}.tmp-");
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else {
|
||||
continue;
|
||||
};
|
||||
let staged = name.strip_prefix(&staging_prefix).is_some_and(|suffix| {
|
||||
suffix.split_once('-').is_some_and(|(pid, sequence)| {
|
||||
!pid.is_empty()
|
||||
&& !sequence.is_empty()
|
||||
&& pid.bytes().all(|b| b.is_ascii_digit())
|
||||
&& sequence.bytes().all(|b| b.is_ascii_digit())
|
||||
})
|
||||
});
|
||||
if name != archive.operation_id && !staged {
|
||||
let staged = is_operation_staging(name, &staging_prefix);
|
||||
if name != operation_id && !staged {
|
||||
continue;
|
||||
}
|
||||
if !entry.file_type()?.is_dir() {
|
||||
@@ -395,11 +458,11 @@ fn erase_archived_diagnostics(
|
||||
};
|
||||
if let Some(manifest) = manifest {
|
||||
if manifest.schema_version != ARCHIVE_SCHEMA_VERSION
|
||||
|| manifest.workspace_id != archive.workspace_id
|
||||
|| manifest.source_runtime_id != archive.source_runtime_id
|
||||
|| manifest.source_worker_id != archive.source_worker_id
|
||||
|| manifest.operation_id != archive.operation_id
|
||||
|| manifest.input_fingerprint != archive.input_fingerprint
|
||||
|| manifest.workspace_id != workspace_id
|
||||
|| manifest.source_runtime_id != source_runtime_id
|
||||
|| manifest.source_worker_id != worker_id
|
||||
|| manifest.operation_id != operation_id
|
||||
|| manifest.input_fingerprint != input_fingerprint
|
||||
{
|
||||
return Err(invalid());
|
||||
}
|
||||
@@ -480,6 +543,109 @@ impl FsWorkerRetentionProvider {
|
||||
Ok(receipt.result.source_removed.then_some(receipt.result))
|
||||
}
|
||||
|
||||
/// Caller holds the Worker operation lock and has verified the durable
|
||||
/// Subject deletion fence. Never execute/recover a missing or pending receipt
|
||||
/// here: the original canonical operation must have completed first.
|
||||
pub(crate) fn purge_subject_job_removal(
|
||||
&self,
|
||||
request: &WorkerRetentionExecutionRequest,
|
||||
) -> Result<(), RuntimeError> {
|
||||
validate_request(request)?;
|
||||
let lock_path = self.runtime_root.join(RETENTION_LOCK);
|
||||
let lock = OpenOptions::new()
|
||||
.create(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(&lock_path)
|
||||
.map_err(|source| RuntimeError::StoreIo {
|
||||
operation: "lock Subject Job removal purge",
|
||||
path: lock_path.clone(),
|
||||
source,
|
||||
})?;
|
||||
lock.lock().map_err(|source| RuntimeError::StoreIo {
|
||||
operation: "lock Subject Job removal purge",
|
||||
path: lock_path,
|
||||
source,
|
||||
})?;
|
||||
let receipt = read_operation_receipt(&self.operation_path(&request.operation_id)?)?
|
||||
.ok_or_else(|| {
|
||||
RuntimeError::InvalidRequest("Canonical removal receipt missing".into())
|
||||
})?;
|
||||
validate_receipt_request(&receipt, request)?;
|
||||
let result = &receipt.result;
|
||||
if !result.source_removed
|
||||
|| result.worker_id != request.worker_id
|
||||
|| result.session_disposition != request.session_disposition
|
||||
|| result.diagnostics_disposition != request.diagnostics_disposition
|
||||
|| result.diagnostics_retained
|
||||
!= (request.diagnostics_disposition == DiagnosticsDisposition::Retain)
|
||||
|| result.archive.is_some()
|
||||
!= (request.session_disposition == SessionDisposition::Archive)
|
||||
|| self.worker_dir(request.worker_id).exists()
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Canonical removal is not an exact successful receipt".into(),
|
||||
));
|
||||
}
|
||||
let archive = if let Some(manifest) = &result.archive {
|
||||
if request.archive_id.as_ref() != Some(&manifest.archive_id)
|
||||
|| manifest.workspace_id != request.workspace_id
|
||||
|| manifest.source_runtime_id != request.source_runtime_id
|
||||
|| manifest.source_worker_id != request.worker_id
|
||||
|| manifest.operation_id != request.operation_id
|
||||
|| manifest.input_fingerprint != request.input_fingerprint
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Removal archive identity mismatch".into(),
|
||||
));
|
||||
}
|
||||
let (saved, _) = self.session_archive_identity(&manifest.archive_id)?;
|
||||
if saved != *manifest {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Removal archive receipt mismatch".into(),
|
||||
));
|
||||
}
|
||||
let session = manifest
|
||||
.source_session_id
|
||||
.parse::<session_store::SessionId>()
|
||||
.map_err(|_| {
|
||||
RuntimeError::InvalidRequest("Invalid removal Session identity".into())
|
||||
})?;
|
||||
Some(session)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let erase = || -> Result<(), session_store::StoreError> {
|
||||
erase_operation_diagnostics(
|
||||
&self.runtime_root,
|
||||
&request.workspace_id,
|
||||
&request.source_runtime_id,
|
||||
request.worker_id,
|
||||
&request.operation_id,
|
||||
&request.input_fingerprint,
|
||||
)?;
|
||||
if let Some(manifest) = &result.archive {
|
||||
erase_removal_session_copies(&self.runtime_root, manifest)?;
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
if let Some(session) = &archive {
|
||||
runtime_session_retention(&self.runtime_root)
|
||||
.delete(
|
||||
*session,
|
||||
session_store::retention::SessionDeletionIntent::ErasePermanently,
|
||||
erase,
|
||||
)
|
||||
.map_err(retention_error)?;
|
||||
} else {
|
||||
// Diagnostics-only old Purge operations have no archive Session ID.
|
||||
// The existing permanent Worker fence plus immutable completed receipt
|
||||
// prevents execution and re-publication; hold the retention lock here.
|
||||
erase().map_err(retention_error)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
|
||||
@@ -10,6 +10,79 @@ pub(crate) struct SubjectDeleteWorkerFence {
|
||||
}
|
||||
|
||||
impl Runtime {
|
||||
/// Erase only copies bound to an exact, completed canonical removal. The
|
||||
/// Subject fence already excludes execution; this does not mint erase authority
|
||||
/// for an arbitrary Worker or reinterpret the historical retention result.
|
||||
pub fn purge_subject_job_removal_scoped(
|
||||
&self,
|
||||
scope: &RuntimeWorkspaceScope,
|
||||
request: runtime_api::SubjectJobRemovalPurgeRequest,
|
||||
) -> Result<runtime_api::SubjectJobRemovalPurgeResponse, RuntimeError> {
|
||||
let deletion = &request.deletion;
|
||||
let removal = &request.removal;
|
||||
if deletion.workspace_id != scope.workspace_id
|
||||
|| removal.workspace_id != deletion.workspace_id
|
||||
|| removal.worker_id != deletion.worker_id
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject Job removal scope mismatch".into(),
|
||||
));
|
||||
}
|
||||
let lease = self.worker_operation_lock(deletion.worker_id)?;
|
||||
let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
state.ensure_workspace_owner(scope, false)?;
|
||||
if state.runtime_identity.as_deref() != Some(removal.source_runtime_id.as_str()) {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject Job removal Runtime identity mismatch".into(),
|
||||
));
|
||||
}
|
||||
state
|
||||
.subject_delete_worker_fences
|
||||
.get(&deletion.worker_id)
|
||||
.filter(|fence| fence.request == *deletion && fence.response.is_some())
|
||||
.ok_or_else(|| {
|
||||
RuntimeError::InvalidRequest(
|
||||
"Exact confirmed Subject deletion fence required".into(),
|
||||
)
|
||||
})?;
|
||||
if state.workers.contains_key(&deletion.worker_id)
|
||||
|| state
|
||||
.unreconciled_worker_writes
|
||||
.contains(&deletion.worker_id)
|
||||
{
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"Subject Job canonical removal must be reconciled first".into(),
|
||||
));
|
||||
}
|
||||
#[cfg(feature = "fs-store")]
|
||||
{
|
||||
let store = state.fs_store().ok_or_else(|| {
|
||||
RuntimeError::InvalidRequest("Subject Job purge requires filesystem storage".into())
|
||||
})?;
|
||||
let removal: WorkerRetentionExecutionRequest = serde_json::from_value(
|
||||
serde_json::to_value(removal)
|
||||
.map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?,
|
||||
)
|
||||
.map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?;
|
||||
// Worker -> Runtime -> retention -> Session is the canonical removal
|
||||
// lock order. The provider validates the receipt under its disk lock.
|
||||
FsWorkerRetentionProvider::new(store.runtime_dir())
|
||||
.purge_subject_job_removal(&removal)?;
|
||||
Ok(runtime_api::SubjectJobRemovalPurgeResponse {
|
||||
workspace_id: removal.workspace_id,
|
||||
worker_id: removal.worker_id,
|
||||
operation_id: removal.operation_id,
|
||||
input_fingerprint: removal.input_fingerprint,
|
||||
})
|
||||
}
|
||||
#[cfg(not(feature = "fs-store"))]
|
||||
Err(RuntimeError::InvalidRequest(
|
||||
"Subject Job purge requires filesystem storage".into(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn subject_delete_worker_scoped(
|
||||
&self,
|
||||
scope: &RuntimeWorkspaceScope,
|
||||
|
||||
Reference in New Issue
Block a user