fix: reconcile erased Archive removals and exercise live compaction recording
Reconcile immutable Runtime removal input under a confirmed permanent erase fence. A confirmed source_removed receipt is audited without republishing erased archives; unknown receipt plus attested remaining aggregate permits an audited stale plan and proof-bound purge replacement. Unknown receipt and inventory remain unresolved. Distinguish domain tombstones from scoped Runtime physical erasure acknowledgements. Erase diagnostic originals and interrupted staging bound to the same archived Session/removal operation, validating Workspace/Worker/fingerprint and saved receipts without rewriting their historical retained flags. Finalize Backend removal only after physical acknowledgement, including previous Retain disposition. Connect public standalone protocol, real Worker request-threshold compaction twice within one turn, cancellation and real pre-AI input rollback, original canonical readers, existing Jobs and sequential Journal tools. Gate generation while compaction completes; verify all durable originals were served, summary/rolled-back input exclusion, exact final cursor, no extra input for successor drain, and one ongoing open entry. External model streams/usage are scripted, not semantic evaluation. Validation: Backend1112lib+11bin (existing ignore1), Runtime429lib+38bin, standalone72lib+27integration (existing model ignore1), runtime-api10, root check/doc/fmt/diff/naming, Server/Runtime builds, isolated worker_environment2 passed. Earlier broad suites remain recorded on the prior source. Fixtures and injected boundaries are not arbitrary crash/remote-transport-loss coverage; worker_environment only verifies startup/connection/cleanup. Acceptance clarification: T-734 assignment answer event22 explicitly defers real-model semantic condition6 to user use, excluding it from this completion scope. It remains unexecuted and unevaluated, not passed. Technical review findings still require a fresh independent verdict on this source.
This commit is contained in:
@@ -812,6 +812,9 @@ pub struct SessionMaterialEraseRequest {
|
|||||||
pub expected_session_id: String,
|
pub expected_session_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Acknowledges the permanent fence and physical erasure of Session originals,
|
||||||
|
/// including archived diagnostics bound to the same removal operations. Historical
|
||||||
|
/// retention receipts remain immutable; their old retained flags are not current availability.
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct SessionMaterialEraseResponse {
|
pub struct SessionMaterialEraseResponse {
|
||||||
pub session_id: String,
|
pub session_id: String,
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ use crate::store::{StaleLeasePolicy, StandaloneWorkerAllocation, StandaloneWorke
|
|||||||
use session_store::{Store, WorkerMetadataStore};
|
use session_store::{Store, WorkerMetadataStore};
|
||||||
use std::sync::atomic::AtomicBool;
|
use std::sync::atomic::AtomicBool;
|
||||||
|
|
||||||
|
mod compaction;
|
||||||
|
|
||||||
struct Fixture {
|
struct Fixture {
|
||||||
_temp: tempfile::TempDir,
|
_temp: tempfile::TempDir,
|
||||||
_allocation: Option<StandaloneWorkerAllocation>,
|
_allocation: Option<StandaloneWorkerAllocation>,
|
||||||
@@ -494,6 +496,7 @@ struct JournalScript {
|
|||||||
gate_first: bool,
|
gate_first: bool,
|
||||||
pause_after_record: bool,
|
pause_after_record: bool,
|
||||||
record_saved: Arc<tokio::sync::Notify>,
|
record_saved: Arc<tokio::sync::Notify>,
|
||||||
|
source_reads: Arc<std::sync::Mutex<std::collections::BTreeMap<String, String>>>,
|
||||||
}
|
}
|
||||||
impl JournalScript {
|
impl JournalScript {
|
||||||
fn response(
|
fn response(
|
||||||
@@ -565,6 +568,16 @@ impl JournalScript {
|
|||||||
let id = format!("source-{index}-{offset}");
|
let id = format!("source-{index}-{offset}");
|
||||||
if let Some(page) = result(&id) {
|
if let Some(page) = result(&id) {
|
||||||
let page = &page["value"];
|
let page = &page["value"];
|
||||||
|
let key = format!("{}:{id}", state["generation_id"].as_str().unwrap());
|
||||||
|
let text = page["text"].as_str().unwrap().to_owned();
|
||||||
|
if let Some(previous) =
|
||||||
|
self.source_reads.lock().unwrap().insert(key, text.clone())
|
||||||
|
{
|
||||||
|
assert_eq!(
|
||||||
|
previous, text,
|
||||||
|
"frozen source read changed within a generation"
|
||||||
|
);
|
||||||
|
}
|
||||||
if let Some(next) = page["next_byte"].as_u64() {
|
if let Some(next) = page["next_byte"].as_u64() {
|
||||||
assert!(next > offset);
|
assert!(next > offset);
|
||||||
offset = next;
|
offset = next;
|
||||||
|
|||||||
@@ -0,0 +1,429 @@
|
|||||||
|
//! Public standalone protocol through real mid-turn compaction and Journal Jobs.
|
||||||
|
//! Only external model streams are scripted. No history, attribution, hook or Job
|
||||||
|
//! result is injected; cancellation and rollback use the controller's public commands.
|
||||||
|
//! Reported model usage and a low configured request threshold force two rewrites.
|
||||||
|
//! The test attests durable public originals, not unfinished display deltas or meaning.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use agen::llm_client::{
|
||||||
|
ClientError, LlmClient, Request,
|
||||||
|
event::{Event, ResponseStatus, StatusEvent, StopReason, UsageEvent},
|
||||||
|
types::Item,
|
||||||
|
};
|
||||||
|
use futures::{Stream, StreamExt};
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
|
type ResponseStream = Pin<Box<dyn Stream<Item = Result<Event, ClientError>> + Send>>;
|
||||||
|
const SUMMARY: &str = "DERIVED_COMPACTION_SENTINEL_NOT_AN_ORIGINAL";
|
||||||
|
const ROLLED_BACK: &str = "ROLLED_BACK_INPUT_MUST_NOT_REACH_JOURNAL";
|
||||||
|
const PARTIAL: &str = "public partial answer before cancellation";
|
||||||
|
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct CompactingDialog {
|
||||||
|
shared: DialogAndJournal,
|
||||||
|
dialog_calls: Arc<AtomicUsize>,
|
||||||
|
compactions: Arc<AtomicUsize>,
|
||||||
|
rollback_entered: Arc<tokio::sync::Notify>,
|
||||||
|
partial_stream_paused: Arc<tokio::sync::Notify>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn events(events: Vec<Event>) -> ResponseStream {
|
||||||
|
Box::pin(futures::stream::iter(events.into_iter().map(Ok)))
|
||||||
|
}
|
||||||
|
fn done() -> ResponseStream {
|
||||||
|
events(vec![
|
||||||
|
Event::text_block_start(0),
|
||||||
|
Event::text_delta(0, "done"),
|
||||||
|
Event::text_block_stop(0, Some(StopReason::EndTurn)),
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl LlmClient for CompactingDialog {
|
||||||
|
async fn stream(&self, request: Request) -> Result<ResponseStream, ClientError> {
|
||||||
|
if request
|
||||||
|
.tools
|
||||||
|
.iter()
|
||||||
|
.any(|tool| tool.name == "write_summary")
|
||||||
|
{
|
||||||
|
if request.items.iter().any(|item| matches!(item, Item::ToolResult { call_id, .. } if call_id == "compact-summary")) {
|
||||||
|
return Ok(done());
|
||||||
|
}
|
||||||
|
let count = self.compactions.fetch_add(1, Ordering::SeqCst);
|
||||||
|
assert!(count < 2, "unexpected additional compaction");
|
||||||
|
return Ok(events(vec![
|
||||||
|
Event::tool_use_start(0, "compact-summary", "write_summary"),
|
||||||
|
Event::tool_input_delta(0, serde_json::json!({"text": SUMMARY}).to_string()),
|
||||||
|
Event::tool_use_stop(0),
|
||||||
|
Event::Status(StatusEvent {
|
||||||
|
status: ResponseStatus::Completed,
|
||||||
|
}),
|
||||||
|
]));
|
||||||
|
}
|
||||||
|
if request.tools.iter().any(|tool| {
|
||||||
|
matches!(
|
||||||
|
tool.name.as_str(),
|
||||||
|
"ReadJournalState" | "SubmitMemorySurface" | "SubmitJobResult"
|
||||||
|
)
|
||||||
|
}) {
|
||||||
|
return self.shared.stream(request).await;
|
||||||
|
}
|
||||||
|
match self.dialog_calls.fetch_add(1, Ordering::SeqCst) {
|
||||||
|
step @ (0 | 1) => {
|
||||||
|
assert!(request.tools.iter().any(|tool| tool.name == "TaskCreate"));
|
||||||
|
Ok(events(vec![
|
||||||
|
Event::tool_use_start(0, format!("public-task-{step}"), "TaskCreate"),
|
||||||
|
Event::tool_input_delta(
|
||||||
|
0,
|
||||||
|
serde_json::json!({
|
||||||
|
"subject": format!("original activity {step}"),
|
||||||
|
"description": format!("original task result before compaction {step}")
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
Event::tool_use_stop(0),
|
||||||
|
Event::Usage(UsageEvent {
|
||||||
|
input_tokens: Some(100_000),
|
||||||
|
output_tokens: Some(10),
|
||||||
|
total_tokens: Some(100_010),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
Event::Status(StatusEvent {
|
||||||
|
status: ResponseStatus::Completed,
|
||||||
|
}),
|
||||||
|
]))
|
||||||
|
}
|
||||||
|
2 => {
|
||||||
|
let paused = self.partial_stream_paused.clone();
|
||||||
|
Ok(Box::pin(
|
||||||
|
events(vec![
|
||||||
|
Event::text_block_start(0),
|
||||||
|
Event::text_delta(0, PARTIAL),
|
||||||
|
Event::text_block_stop(0, None),
|
||||||
|
])
|
||||||
|
.chain(futures::stream::once(async move {
|
||||||
|
paused.notify_one();
|
||||||
|
std::future::pending::<Result<Event, ClientError>>().await
|
||||||
|
})),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
3 => {
|
||||||
|
// No AI event: cancelling this request must roll back the submitted input.
|
||||||
|
self.rollback_entered.notify_one();
|
||||||
|
std::future::pending::<()>().await;
|
||||||
|
unreachable!()
|
||||||
|
}
|
||||||
|
other => panic!("unexpected dialog request {other}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cancel_and_wait(
|
||||||
|
client: &mut client::Client<client::transport::in_process::Socket>,
|
||||||
|
sequence: u64,
|
||||||
|
) {
|
||||||
|
client
|
||||||
|
.send(&protocol::Method::Cancel {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(sequence),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(20), async {
|
||||||
|
loop {
|
||||||
|
match client.next_event().await.unwrap().unwrap() {
|
||||||
|
protocol::Event::RunEnd { result } => {
|
||||||
|
assert_ne!(result, protocol::RunResult::Finished);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
protocol::Event::Error { code, message } => panic!("{code:?}: {message}"),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn public_mid_turn_compactions_preserve_originals_and_drain_after_cancel_and_rollback() {
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(60), async {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let state = temp.path().join("state");
|
||||||
|
let catalog = StandaloneSubjects::open(&state).unwrap();
|
||||||
|
let subject = catalog.create("compaction-journal", None).unwrap();
|
||||||
|
let model = CompactingDialog {
|
||||||
|
shared: DialogAndJournal {
|
||||||
|
journal: JournalScript {
|
||||||
|
gate_first: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut launch = crate::StandaloneLaunchConfig::new(
|
||||||
|
temp.path(),
|
||||||
|
&state,
|
||||||
|
manifest::ProfileSelector::source_named(
|
||||||
|
manifest::ProfileRegistrySource::Builtin,
|
||||||
|
"standalone-subjektiv",
|
||||||
|
),
|
||||||
|
"public-compaction-journal",
|
||||||
|
)
|
||||||
|
.with_subject(subject.id.clone())
|
||||||
|
.resolve()
|
||||||
|
.unwrap();
|
||||||
|
launch.profile.manifest.compaction = Some(manifest::CompactionConfig {
|
||||||
|
threshold: None,
|
||||||
|
request_threshold: Some(50_000),
|
||||||
|
retained_tokens: 0,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let host = crate::StandaloneHost::start_with_model_client(launch, model.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let session = host.record().active_session_id;
|
||||||
|
let workers = StandaloneWorkerStore::open_existing(&state).unwrap();
|
||||||
|
let source_store = session_store::FsStore::new_with_retention(
|
||||||
|
workers.sessions_dir(host.record().worker_id),
|
||||||
|
crate::store::session_retention(&state),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let mut client = host.connect();
|
||||||
|
client
|
||||||
|
.send(&protocol::Method::submit_text(
|
||||||
|
protocol::new_submission_request_id(),
|
||||||
|
"original long turn survives compaction",
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut completed_compactions = 0;
|
||||||
|
loop {
|
||||||
|
match client.next_event().await.unwrap().unwrap() {
|
||||||
|
protocol::Event::CompactionProgress {
|
||||||
|
compaction: Some(progress),
|
||||||
|
} if progress.phase == protocol::CompactionPhase::Committing => {
|
||||||
|
assert_eq!(
|
||||||
|
progress.trigger,
|
||||||
|
protocol::CompactionTrigger::RequestThreshold
|
||||||
|
);
|
||||||
|
completed_compactions += 1;
|
||||||
|
}
|
||||||
|
protocol::Event::TextDelta { text } if text.contains(PARTIAL) => break,
|
||||||
|
protocol::Event::RunEnd { .. } => panic!("dialog ended before cancellation"),
|
||||||
|
protocol::Event::Error { code, message } => panic!("{code:?}: {message}"),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(model.compactions.load(Ordering::SeqCst), 2);
|
||||||
|
assert_eq!(completed_compactions, 2);
|
||||||
|
model.shared.journal.entered.notified().await;
|
||||||
|
let context = JournalHostContext::validated_session(
|
||||||
|
&catalog.store,
|
||||||
|
catalog
|
||||||
|
.store
|
||||||
|
.session_attribution(&session.to_string())
|
||||||
|
.unwrap()
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let reader = JournalHostContext::validated_reader(&catalog.store, &subject.id).unwrap();
|
||||||
|
let journal = catalog
|
||||||
|
.store
|
||||||
|
.list_journals(&reader, None, 1)
|
||||||
|
.unwrap()
|
||||||
|
.items
|
||||||
|
.pop()
|
||||||
|
.unwrap();
|
||||||
|
let first = catalog
|
||||||
|
.store
|
||||||
|
.active_journal_generation(&context, &journal.id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
catalog
|
||||||
|
.store
|
||||||
|
.list_journal_entries(&reader, &journal.id, None, 10)
|
||||||
|
.unwrap()
|
||||||
|
.items
|
||||||
|
.is_empty(),
|
||||||
|
"compaction must finish before generation is released"
|
||||||
|
);
|
||||||
|
model.partial_stream_paused.notified().await;
|
||||||
|
cancel_and_wait(&mut client, 1).await;
|
||||||
|
client
|
||||||
|
.send(&protocol::Method::submit_text(
|
||||||
|
protocol::new_submission_request_id(),
|
||||||
|
ROLLED_BACK,
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
model.rollback_entered.notified().await;
|
||||||
|
let before_rollback: Vec<_> = source_store
|
||||||
|
.list_segments(session)
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.flat_map(|segment| source_store.read_all(session, *segment).unwrap())
|
||||||
|
.collect();
|
||||||
|
assert!(
|
||||||
|
serde_json::to_string(&before_rollback)
|
||||||
|
.unwrap()
|
||||||
|
.contains(ROLLED_BACK),
|
||||||
|
"the submitted input must reach the actual durable rollback boundary"
|
||||||
|
);
|
||||||
|
cancel_and_wait(&mut client, 2).await;
|
||||||
|
assert_eq!(
|
||||||
|
catalog
|
||||||
|
.store
|
||||||
|
.active_journal_generation(&context, &journal.id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap(),
|
||||||
|
first,
|
||||||
|
"compaction/cancellation must not alter the running generation's frozen input"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(matches!(source_store.delete_session(session),
|
||||||
|
Err(session_store::StoreError::SessionPinned(id)) if id == session));
|
||||||
|
let segments = source_store.list_segments(session).unwrap();
|
||||||
|
assert_eq!(segments.len(), 3);
|
||||||
|
let raw: Vec<_> = segments
|
||||||
|
.iter()
|
||||||
|
.flat_map(|segment| source_store.read_all(session, *segment).unwrap())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
raw.iter()
|
||||||
|
.filter(|entry| matches!(
|
||||||
|
entry,
|
||||||
|
session_store::LogEntry::AnnotatedSegmentStart {
|
||||||
|
compacted_from: Some(_),
|
||||||
|
..
|
||||||
|
}
|
||||||
|
))
|
||||||
|
.count(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!serde_json::to_string(&raw).unwrap().contains(ROLLED_BACK),
|
||||||
|
"pre-AI cancellation must roll back the speculative input"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
serde_json::to_string(&raw).unwrap().contains(SUMMARY),
|
||||||
|
"real rewrite did not persist its summary"
|
||||||
|
);
|
||||||
|
let metadata =
|
||||||
|
session_store::FsWorkerStore::new(workers.worker_metadata_dir(host.record().worker_id))
|
||||||
|
.unwrap()
|
||||||
|
.read_by_name(&host.record().storage_key)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let originals =
|
||||||
|
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
|
||||||
|
&workers.sessions_dir(host.record().worker_id),
|
||||||
|
session,
|
||||||
|
metadata.active.unwrap().segment_id.unwrap(),
|
||||||
|
session_store::public_material::PublicMaterialLimits::default(),
|
||||||
|
crate::store::session_retention(&state),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let final_original = originals.descriptors().last().unwrap();
|
||||||
|
let expected_cursor = (final_original.canonical_order + 1, 0);
|
||||||
|
let canonical: Vec<_> = originals
|
||||||
|
.descriptors()
|
||||||
|
.map(|descriptor| {
|
||||||
|
let page = originals
|
||||||
|
.read_content(descriptor, 0, MAX_JOURNAL_READ_BYTES)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(page.next_offset, descriptor.total_bytes);
|
||||||
|
page.content
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
assert!(!canonical.join("\n").contains(ROLLED_BACK));
|
||||||
|
assert!(!canonical.join("\n").contains(SUMMARY));
|
||||||
|
|
||||||
|
// Resume the existing runner, then its one durable successor, with no more input.
|
||||||
|
let jobs = host.jobs();
|
||||||
|
model.shared.journal.release.notify_one();
|
||||||
|
jobs.wait(&first.job.job_id).await.unwrap();
|
||||||
|
model.shared.journal.entered.notified().await;
|
||||||
|
let second = catalog
|
||||||
|
.store
|
||||||
|
.active_journal_generation(&context, &journal.id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_ne!(first.id, second.id);
|
||||||
|
model.shared.journal.release.notify_one();
|
||||||
|
jobs.wait(&second.job.job_id).await.unwrap();
|
||||||
|
assert!(!catalog.store.journal_retention_required(&context).unwrap());
|
||||||
|
let (saved, open) = catalog
|
||||||
|
.store
|
||||||
|
.journal_resume_state(&context, &journal.id)
|
||||||
|
.unwrap();
|
||||||
|
let open = open.unwrap();
|
||||||
|
assert!(!open.ongoing.is_empty());
|
||||||
|
let entries = catalog
|
||||||
|
.store
|
||||||
|
.list_journal_entries(&reader, &journal.id, None, 10)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
entries.items.len(),
|
||||||
|
1,
|
||||||
|
"compaction/turn boundaries must not fix or duplicate the open entry"
|
||||||
|
);
|
||||||
|
assert_eq!(entries.items[0].id, open.id);
|
||||||
|
let supplied = model
|
||||||
|
.shared
|
||||||
|
.journal
|
||||||
|
.source_reads
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.values()
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
for original in [
|
||||||
|
"original long turn survives compaction",
|
||||||
|
"original activity 0",
|
||||||
|
"original activity 1",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
supplied.contains(original),
|
||||||
|
"Journal did not receive {original}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for original in canonical {
|
||||||
|
assert!(
|
||||||
|
supplied.contains(&original),
|
||||||
|
"a durable public original was skipped"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
!supplied.contains(SUMMARY),
|
||||||
|
"derived summary substituted for original input"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!supplied.contains(ROLLED_BACK),
|
||||||
|
"cancelled input was admitted after rollback"
|
||||||
|
);
|
||||||
|
assert_eq!((saved.process_order, saved.process_byte), expected_cursor);
|
||||||
|
let position = catalog
|
||||||
|
.store
|
||||||
|
.read_journal_text(
|
||||||
|
&reader,
|
||||||
|
&journal.id,
|
||||||
|
None,
|
||||||
|
JournalTextField::CurrentPosition,
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
MAX_JOURNAL_READ_BYTES,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(!position.text.is_empty());
|
||||||
|
host.shutdown().await.unwrap();
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded public compaction/Journal integration");
|
||||||
|
}
|
||||||
@@ -659,8 +659,38 @@ fn catalog_rejects_oversize_record_with_explicit_resource_limit() {
|
|||||||
fn archive_scope_and_legacy_active_pointer_are_checked_and_erase_retry_never_restores_originals() {
|
fn archive_scope_and_legacy_active_pointer_are_checked_and_erase_retry_never_restores_originals() {
|
||||||
let f = Fixture::new();
|
let f = Fixture::new();
|
||||||
f.stop();
|
f.stop();
|
||||||
let request = f.retention_request(SessionDisposition::Archive);
|
let mut request = f.retention_request(SessionDisposition::Archive);
|
||||||
f.runtime.execute_worker_retention(&request).unwrap();
|
request.diagnostics_disposition = DiagnosticsDisposition::Retain;
|
||||||
|
let log = f
|
||||||
|
.root()
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.join("runs/run-original/worker.out.log");
|
||||||
|
std::fs::create_dir_all(log.parent().unwrap()).unwrap();
|
||||||
|
std::fs::write(&log, "archived diagnostic original").unwrap();
|
||||||
|
let original_receipt = f.runtime.execute_worker_retention(&request).unwrap();
|
||||||
|
assert!(original_receipt.diagnostics_retained);
|
||||||
|
let diagnostics = f
|
||||||
|
.temp
|
||||||
|
.path()
|
||||||
|
.join("archives/diagnostics")
|
||||||
|
.join(&request.operation_id);
|
||||||
|
assert!(
|
||||||
|
diagnostics
|
||||||
|
.join("runs/run-original/worker.out.log")
|
||||||
|
.is_file()
|
||||||
|
);
|
||||||
|
let staged_diagnostics =
|
||||||
|
diagnostics.with_file_name(format!(".{}.tmp-42-1", request.operation_id));
|
||||||
|
std::fs::create_dir_all(&staged_diagnostics).unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
staged_diagnostics.join("partial-original.log"),
|
||||||
|
"incomplete copy",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let unrelated = diagnostics.with_file_name("another-operation");
|
||||||
|
std::fs::create_dir_all(&unrelated).unwrap();
|
||||||
|
std::fs::write(unrelated.join("untouched.log"), "unrelated diagnostic").unwrap();
|
||||||
let source = Source::Archived {
|
let source = Source::Archived {
|
||||||
archive_id: "canonical-archive".into(),
|
archive_id: "canonical-archive".into(),
|
||||||
};
|
};
|
||||||
@@ -756,12 +786,76 @@ fn archive_scope_and_legacy_active_pointer_are_checked_and_erase_retry_never_res
|
|||||||
.pin_session_public_source_scoped(&f.scope, &source, &f.session.to_string(), pin)
|
.pin_session_public_source_scoped(&f.scope, &source, &f.session.to_string(), pin)
|
||||||
.is_err()
|
.is_err()
|
||||||
);
|
);
|
||||||
// An old removal receipt is evidence only and must not republish erased bytes.
|
// An old receipt is historical evidence, not permission to republish erased bytes.
|
||||||
f.runtime.execute_worker_retention(&request).unwrap();
|
assert_eq!(
|
||||||
|
f.runtime.execute_worker_retention(&request).unwrap(),
|
||||||
|
original_receipt
|
||||||
|
);
|
||||||
|
assert!(!diagnostics.exists());
|
||||||
|
assert!(!staged_diagnostics.exists());
|
||||||
|
assert!(unrelated.join("untouched.log").is_file());
|
||||||
assert!(!root.exists());
|
assert!(!root.exists());
|
||||||
assert!(retention.pins(f.session).unwrap().is_empty());
|
assert!(retention.pins(f.session).unwrap().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn erasure_rejects_mismatched_archived_diagnostics_then_resumes_under_the_same_fence() {
|
||||||
|
let f = Fixture::new();
|
||||||
|
f.stop();
|
||||||
|
let mut request = f.retention_request(SessionDisposition::Archive);
|
||||||
|
request.diagnostics_disposition = DiagnosticsDisposition::Retain;
|
||||||
|
let receipt = f.runtime.execute_worker_retention(&request).unwrap();
|
||||||
|
let source = Source::Archived {
|
||||||
|
archive_id: request.archive_id.clone().unwrap(),
|
||||||
|
};
|
||||||
|
let manifest = f
|
||||||
|
.temp
|
||||||
|
.path()
|
||||||
|
.join("archives/diagnostics")
|
||||||
|
.join(&request.operation_id)
|
||||||
|
.join("manifest.json");
|
||||||
|
let original: serde_json::Value =
|
||||||
|
serde_json::from_slice(&std::fs::read(&manifest).unwrap()).unwrap();
|
||||||
|
for field in ["workspace_id", "input_fingerprint", "source_worker_id"] {
|
||||||
|
let mut changed = original.clone();
|
||||||
|
changed[field] = serde_json::json!(uuid::Uuid::now_v7().to_string());
|
||||||
|
std::fs::write(&manifest, serde_json::to_vec(&changed).unwrap()).unwrap();
|
||||||
|
assert!(
|
||||||
|
f.runtime
|
||||||
|
.delete_session_public_source_scoped(
|
||||||
|
&f.scope,
|
||||||
|
&source,
|
||||||
|
&f.session.to_string(),
|
||||||
|
SessionDeletionIntent::ErasePermanently
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(manifest.exists());
|
||||||
|
assert!(matches!(
|
||||||
|
runtime_session_retention(f.temp.path()).ensure_live(f.session),
|
||||||
|
Err(session_store::StoreError::SessionErased(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
std::fs::write(&manifest, serde_json::to_vec(&original).unwrap()).unwrap();
|
||||||
|
f.runtime
|
||||||
|
.delete_session_public_source_scoped(
|
||||||
|
&f.scope,
|
||||||
|
&source,
|
||||||
|
&f.session.to_string(),
|
||||||
|
SessionDeletionIntent::ErasePermanently,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(!manifest.parent().unwrap().exists());
|
||||||
|
assert_eq!(
|
||||||
|
f.runtime.execute_worker_retention(&request).unwrap(),
|
||||||
|
receipt
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
receipt.diagnostics_retained,
|
||||||
|
"historical receipt must not be rewritten after erasure"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn material_request_bounds_return_explicit_unavailable_without_previews() {
|
fn material_request_bounds_return_explicit_unavailable_without_previews() {
|
||||||
let f = Fixture::new();
|
let f = Fixture::new();
|
||||||
|
|||||||
@@ -297,14 +297,15 @@ pub(crate) fn remove_archived_session_copies(
|
|||||||
}
|
}
|
||||||
let session_root = entry.path().join("session");
|
let session_root = entry.path().join("session");
|
||||||
let owner = match fs::read(entry.path().join("manifest.json")) {
|
let owner = match fs::read(entry.path().join("manifest.json")) {
|
||||||
Ok(bytes) => Some(
|
Ok(bytes) => Some(serde_json::from_slice::<WorkerSessionArchiveManifest>(
|
||||||
serde_json::from_slice::<WorkerSessionArchiveManifest>(&bytes)?.source_session_id,
|
&bytes,
|
||||||
),
|
)?),
|
||||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
|
||||||
Err(error) => return Err(error.into()),
|
Err(error) => return Err(error.into()),
|
||||||
};
|
};
|
||||||
if let Some(owner) = owner {
|
if let Some(owner) = owner {
|
||||||
if owner == session.to_string() {
|
if owner.source_session_id == session.to_string() {
|
||||||
|
erase_archived_diagnostics(runtime_root, &owner)?;
|
||||||
match fs::remove_dir_all(&session_root) {
|
match fs::remove_dir_all(&session_root) {
|
||||||
Ok(()) => (),
|
Ok(()) => (),
|
||||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => (),
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => (),
|
||||||
@@ -326,6 +327,89 @@ pub(crate) fn remove_archived_session_copies(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Erasure includes diagnostic originals of the same archived removal operation.
|
||||||
|
/// The immutable removal receipt is historical evidence and is never rewritten.
|
||||||
|
/// Staging names are bound to that exact operation; unrelated archives are untouched.
|
||||||
|
fn erase_archived_diagnostics(
|
||||||
|
runtime_root: &Path,
|
||||||
|
archive: &WorkerSessionArchiveManifest,
|
||||||
|
) -> Result<(), session_store::StoreError> {
|
||||||
|
let invalid = || {
|
||||||
|
session_store::StoreError::Io(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidData,
|
||||||
|
"Archived diagnostics erasure identity mismatch",
|
||||||
|
))
|
||||||
|
};
|
||||||
|
if archive.schema_version != ARCHIVE_SCHEMA_VERSION {
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
validate_id("operation_id", &archive.operation_id).map_err(|_| invalid())?;
|
||||||
|
let receipt_path = runtime_root
|
||||||
|
.join("retention/operations")
|
||||||
|
.join(format!("{}.json", archive.operation_id));
|
||||||
|
if let Some(receipt) = read_operation_receipt(&receipt_path).map_err(|_| invalid())? {
|
||||||
|
if receipt.request.workspace_id != archive.workspace_id
|
||||||
|
|| receipt.request.source_runtime_id != archive.source_runtime_id
|
||||||
|
|| receipt.request.worker_id != archive.source_worker_id
|
||||||
|
|| receipt.request.operation_id != archive.operation_id
|
||||||
|
|| receipt.request.input_fingerprint != archive.input_fingerprint
|
||||||
|
|| receipt.request.archive_id.as_ref() != Some(&archive.archive_id)
|
||||||
|
|| receipt.result.archive.as_ref() != Some(archive)
|
||||||
|
{
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let parent = runtime_root.join("archives/diagnostics");
|
||||||
|
let entries = match fs::read_dir(&parent) {
|
||||||
|
Ok(entries) => entries,
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||||
|
Err(error) => return Err(error.into()),
|
||||||
|
};
|
||||||
|
let staging_prefix = format!(".{}.tmp-", archive.operation_id);
|
||||||
|
for entry in entries {
|
||||||
|
let entry = entry?;
|
||||||
|
let name = entry.file_name();
|
||||||
|
let Some(name) = name.to_str() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let staged = name.strip_prefix(&staging_prefix).is_some_and(|suffix| {
|
||||||
|
suffix.split_once('-').is_some_and(|(pid, sequence)| {
|
||||||
|
!pid.is_empty()
|
||||||
|
&& !sequence.is_empty()
|
||||||
|
&& pid.bytes().all(|b| b.is_ascii_digit())
|
||||||
|
&& sequence.bytes().all(|b| b.is_ascii_digit())
|
||||||
|
})
|
||||||
|
});
|
||||||
|
if name != archive.operation_id && !staged {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !entry.file_type()?.is_dir() {
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
let manifest = match fs::read(entry.path().join("manifest.json")) {
|
||||||
|
Ok(bytes) => Some(serde_json::from_slice::<DiagnosticsArchiveManifest>(
|
||||||
|
&bytes,
|
||||||
|
)?),
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound && staged => None,
|
||||||
|
Err(error) => return Err(error.into()),
|
||||||
|
};
|
||||||
|
if let Some(manifest) = manifest {
|
||||||
|
if manifest.schema_version != ARCHIVE_SCHEMA_VERSION
|
||||||
|
|| manifest.workspace_id != archive.workspace_id
|
||||||
|
|| manifest.source_runtime_id != archive.source_runtime_id
|
||||||
|
|| manifest.source_worker_id != archive.source_worker_id
|
||||||
|
|| manifest.operation_id != archive.operation_id
|
||||||
|
|| manifest.input_fingerprint != archive.input_fingerprint
|
||||||
|
{
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fs::remove_dir_all(entry.path())?;
|
||||||
|
}
|
||||||
|
sync_directory(&parent, "erase archived Worker diagnostics").map_err(|_| invalid())?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn retention_error(error: session_store::StoreError) -> RuntimeError {
|
pub(crate) fn retention_error(error: session_store::StoreError) -> RuntimeError {
|
||||||
RuntimeError::InvalidRequest(format!("Session retention rejected operation: {error}"))
|
RuntimeError::InvalidRequest(format!("Session retention rejected operation: {error}"))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -110,6 +110,50 @@ impl ErasedJournalSession {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Physical Runtime erasure acknowledgement, not merely a domain tombstone.
|
||||||
|
/// Only the scoped erase request and a matching Session response construct it.
|
||||||
|
/// The Runtime acknowledgement includes associated archived diagnostic originals;
|
||||||
|
/// historical retention receipts retain their original values.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct RuntimeJournalErasureAck {
|
||||||
|
domain: ErasedJournalSession,
|
||||||
|
source: runtime_api::SessionPublicSource,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ErasedJournalSession {
|
||||||
|
pub(crate) fn erase_runtime_source(
|
||||||
|
self,
|
||||||
|
runtime: &crate::hosts::RuntimeRegistry,
|
||||||
|
source: runtime_api::SessionPublicSource,
|
||||||
|
) -> crate::Result<RuntimeJournalErasureAck> {
|
||||||
|
let response = runtime
|
||||||
|
.session_material_erase(
|
||||||
|
&self.worker.runtime_id,
|
||||||
|
runtime_api::SessionMaterialEraseRequest {
|
||||||
|
workspace_id: self.workspace_id.clone(),
|
||||||
|
source: source.clone(),
|
||||||
|
expected_session_id: self.session_id.clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|e| e.into_error())?;
|
||||||
|
if response.session_id != self.session_id {
|
||||||
|
return Err(StoreError::RepositoryConflict(
|
||||||
|
"Session erase receipt differs from the requested identity".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(RuntimeJournalErasureAck {
|
||||||
|
domain: self,
|
||||||
|
source,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RuntimeJournalErasureAck {
|
||||||
|
pub(crate) fn domain_proof(&self) -> &ErasedJournalSession {
|
||||||
|
&self.domain
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||||
pub enum WorkerRemovalBlocker {
|
pub enum WorkerRemovalBlocker {
|
||||||
@@ -490,12 +534,125 @@ impl SqliteWorkspaceStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An unknown acceptance is not failure. Supersede only after physical erasure
|
||||||
|
/// fenced Archive effects and Runtime attested the same remaining aggregate.
|
||||||
|
pub(crate) fn supersede_erased_worker_removal(
|
||||||
|
&self,
|
||||||
|
prepared: &PreparedWorkerRemoval,
|
||||||
|
ack: &RuntimeJournalErasureAck,
|
||||||
|
inventory: &WorkerRetentionInventory,
|
||||||
|
) -> Result<(), WorkerRetentionError> {
|
||||||
|
let proof = ack.domain_proof();
|
||||||
|
if proof.workspace_id != prepared.plan.workspace_id
|
||||||
|
|| proof.worker != prepared.plan.worker
|
||||||
|
|| prepared.plan.session_disposition != SessionDisposition::Archive
|
||||||
|
|| inventory.workspace_id != proof.workspace_id
|
||||||
|
|| inventory.runtime_id != proof.worker.runtime_id
|
||||||
|
|| inventory.worker_id.to_string() != proof.worker.worker_id
|
||||||
|
|| inventory
|
||||||
|
.session_id
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|id| id != &proof.session_id)
|
||||||
|
{
|
||||||
|
return Err(WorkerRetentionError::Invalid(
|
||||||
|
"Erasure reconciliation source mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
self.with_conn_mut(|conn| {
|
||||||
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
let plan = load_plan_op(&tx, &prepared.plan.operation_id)?
|
||||||
|
.ok_or_else(|| StoreError::InvalidInput("operation missing".into()))?;
|
||||||
|
if plan != prepared.plan
|
||||||
|
|| !matches!(
|
||||||
|
plan.state,
|
||||||
|
WorkerRemovalPlanState::Planned
|
||||||
|
| WorkerRemovalPlanState::Executing
|
||||||
|
| WorkerRemovalPlanState::Failed
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Err(StoreError::InvalidInput("active operation mismatch".into()));
|
||||||
|
}
|
||||||
|
insert_erasure_reconciliation_audit(&tx, &plan, ack, None)?;
|
||||||
|
mark_stale(&tx, &plan, "journal_session_erased_archive_fenced")?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn audit_erased_worker_removal_receipt(
|
||||||
|
&self,
|
||||||
|
prepared: &PreparedWorkerRemoval,
|
||||||
|
ack: &RuntimeJournalErasureAck,
|
||||||
|
result: &WorkerRetentionExecutionResult,
|
||||||
|
) -> Result<(), WorkerRetentionError> {
|
||||||
|
let proof = ack.domain_proof();
|
||||||
|
if proof.workspace_id != prepared.plan.workspace_id
|
||||||
|
|| proof.worker != prepared.plan.worker
|
||||||
|
|| result.operation_id != prepared.plan.operation_id
|
||||||
|
|| result.input_fingerprint != prepared.plan.input_fingerprint
|
||||||
|
|| result.worker_id.to_string() != proof.worker.worker_id
|
||||||
|
|| !result.source_removed
|
||||||
|
|| result.session_disposition != prepared.plan.session_disposition
|
||||||
|
|| result.diagnostics_disposition != prepared.plan.diagnostics_disposition
|
||||||
|
|| result.archive.as_ref().is_none_or(|a| {
|
||||||
|
a.source_session_id != proof.session_id
|
||||||
|
|| Some(&a.archive_id) != prepared.plan.archive_id.as_ref()
|
||||||
|
|| a.workspace_id != proof.workspace_id
|
||||||
|
|| a.source_runtime_id != proof.worker.runtime_id
|
||||||
|
|| a.source_worker_id.to_string() != proof.worker.worker_id
|
||||||
|
|| a.operation_id != result.operation_id
|
||||||
|
|| a.input_fingerprint != result.input_fingerprint
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(WorkerRetentionError::Invalid(
|
||||||
|
"Erasure receipt mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
self.with_conn_mut(|conn| {
|
||||||
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
insert_erasure_reconciliation_audit(&tx, &prepared.plan, ack, Some(result))?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn commit_worker_removal(
|
pub fn commit_worker_removal(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
operation_id: &str,
|
operation_id: &str,
|
||||||
fp: &str,
|
fp: &str,
|
||||||
result: &WorkerRetentionExecutionResult,
|
result: &WorkerRetentionExecutionResult,
|
||||||
|
) -> Result<WorkerRemovalCommit, WorkerRetentionError> {
|
||||||
|
self.commit_worker_removal_inner(workspace_id, operation_id, fp, result, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Receipt-confirmed removal after erasure must not republish the erased Archive.
|
||||||
|
/// The original operation, fingerprint and Runtime result remain unchanged.
|
||||||
|
pub(crate) fn commit_erased_worker_removal(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
prepared: &PreparedWorkerRemoval,
|
||||||
|
result: &WorkerRetentionExecutionResult,
|
||||||
|
ack: &RuntimeJournalErasureAck,
|
||||||
|
) -> Result<WorkerRemovalCommit, WorkerRetentionError> {
|
||||||
|
self.commit_worker_removal_inner(
|
||||||
|
workspace_id,
|
||||||
|
&prepared.plan.operation_id,
|
||||||
|
&prepared.plan.input_fingerprint,
|
||||||
|
result,
|
||||||
|
Some(ack),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn commit_worker_removal_inner(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
operation_id: &str,
|
||||||
|
fp: &str,
|
||||||
|
result: &WorkerRetentionExecutionResult,
|
||||||
|
erasure: Option<&RuntimeJournalErasureAck>,
|
||||||
) -> Result<WorkerRemovalCommit, WorkerRetentionError> {
|
) -> Result<WorkerRemovalCommit, WorkerRetentionError> {
|
||||||
self.with_conn_mut(|conn|{
|
self.with_conn_mut(|conn|{
|
||||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
@@ -519,6 +676,13 @@ impl SqliteWorkspaceStore {
|
|||||||
return Err(StoreError::InvalidInput("Runtime retention result does not match removal plan".into()));
|
return Err(StoreError::InvalidInput("Runtime retention result does not match removal plan".into()));
|
||||||
}
|
}
|
||||||
if !result.source_removed{return Err(StoreError::InvalidInput("Runtime source was not removed".into()));}
|
if !result.source_removed{return Err(StoreError::InvalidInput("Runtime source was not removed".into()));}
|
||||||
|
if let Some(ack) = erasure {
|
||||||
|
let proof = ack.domain_proof();
|
||||||
|
if proof.workspace_id != workspace_id || proof.worker != plan.worker
|
||||||
|
|| result.archive.as_ref().is_none_or(|a| a.source_session_id != proof.session_id) {
|
||||||
|
return Err(StoreError::InvalidInput("Erased removal receipt requires matching source and physical Runtime erasure acknowledgement".into()));
|
||||||
|
}
|
||||||
|
}
|
||||||
let worker=load_worker(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput("Worker missing before commit".into()))?;
|
let worker=load_worker(&tx,workspace_id,&plan.worker)?.ok_or_else(||StoreError::InvalidInput("Worker missing before commit".into()))?;
|
||||||
if worker.updated_at!=plan.worker_updated_at{return Err(StoreError::InvalidInput(format!("stale:{}:Worker updated_at changed",plan.plan_id)));}
|
if worker.updated_at!=plan.worker_updated_at{return Err(StoreError::InvalidInput(format!("stale:{}:Worker updated_at changed",plan.plan_id)));}
|
||||||
if worker.retention_state=="pinned" { return Err(StoreError::InvalidInput(format!("stale:{}:hold added",plan.plan_id))); }
|
if worker.retention_state=="pinned" { return Err(StoreError::InvalidInput(format!("stale:{}:hold added",plan.plan_id))); }
|
||||||
@@ -527,6 +691,7 @@ impl SqliteWorkspaceStore {
|
|||||||
let now=Utc::now().to_rfc3339();
|
let now=Utc::now().to_rfc3339();
|
||||||
if let Some(a)=&result.archive{
|
if let Some(a)=&result.archive{
|
||||||
if plan.archive_id.as_deref()!=Some(&a.archive_id)||a.workspace_id!=workspace_id||a.source_runtime_id!=plan.worker.runtime_id||a.source_worker_id.to_string()!=plan.worker.worker_id||a.policy_id!=plan.policy_id||a.operation_id!=operation_id||a.input_fingerprint!=fp{return Err(StoreError::InvalidInput("archive manifest mismatch".into()));}
|
if plan.archive_id.as_deref()!=Some(&a.archive_id)||a.workspace_id!=workspace_id||a.source_runtime_id!=plan.worker.runtime_id||a.source_worker_id.to_string()!=plan.worker.worker_id||a.policy_id!=plan.policy_id||a.operation_id!=operation_id||a.input_fingerprint!=fp{return Err(StoreError::InvalidInput("archive manifest mismatch".into()));}
|
||||||
|
if erasure.is_none() {
|
||||||
let expires_at=match plan.archive_retention { ArchiveRetention::Forever=>None, ArchiveRetention::ForSeconds{seconds}=>{let seconds=i64::try_from(seconds).map_err(|_|StoreError::InvalidInput("archive retention deadline overflow".into()))?;Some((Utc::now()+chrono::Duration::seconds(seconds)).to_rfc3339())} };
|
let expires_at=match plan.archive_retention { ArchiveRetention::Forever=>None, ArchiveRetention::ForSeconds{seconds}=>{let seconds=i64::try_from(seconds).map_err(|_|StoreError::InvalidInput("archive retention deadline overflow".into()))?;Some((Utc::now()+chrono::Duration::seconds(seconds)).to_rfc3339())} };
|
||||||
tx.execute("INSERT OR IGNORE INTO worker_session_archives(archive_id,workspace_id,runtime_id,worker_id,session_id,checksum_sha256,content_bytes,policy_id,policy_digest,operation_id,committed_at,expires_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",params![a.archive_id,workspace_id,plan.worker.runtime_id,plan.worker.worker_id,a.source_session_id,a.content_checksum_sha256,a.content_bytes,plan.policy_id,plan.policy_digest,operation_id,now,expires_at])?;
|
tx.execute("INSERT OR IGNORE INTO worker_session_archives(archive_id,workspace_id,runtime_id,worker_id,session_id,checksum_sha256,content_bytes,policy_id,policy_digest,operation_id,committed_at,expires_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",params![a.archive_id,workspace_id,plan.worker.runtime_id,plan.worker.worker_id,a.source_session_id,a.content_checksum_sha256,a.content_bytes,plan.policy_id,plan.policy_digest,operation_id,now,expires_at])?;
|
||||||
let observe_grants = {
|
let observe_grants = {
|
||||||
@@ -574,20 +739,21 @@ impl SqliteWorkspaceStore {
|
|||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}else if plan.session_disposition==SessionDisposition::Archive{return Err(StoreError::InvalidInput("archive manifest missing".into()));}
|
}else if plan.session_disposition==SessionDisposition::Archive{return Err(StoreError::InvalidInput("archive manifest missing".into()));}
|
||||||
match plan.diagnostics_disposition {
|
match plan.diagnostics_disposition {
|
||||||
DiagnosticsDisposition::Purge if result.diagnostics_retained => return Err(StoreError::InvalidInput("Runtime retained diagnostics for purge disposition".into())),
|
DiagnosticsDisposition::Purge if result.diagnostics_retained => return Err(StoreError::InvalidInput("Runtime retained diagnostics for purge disposition".into())),
|
||||||
DiagnosticsDisposition::Retain if !result.diagnostics_retained => return Err(StoreError::InvalidInput("Runtime did not retain diagnostics".into())),
|
DiagnosticsDisposition::Retain if !result.diagnostics_retained => return Err(StoreError::InvalidInput("Runtime did not retain diagnostics".into())),
|
||||||
DiagnosticsDisposition::Retain => {
|
DiagnosticsDisposition::Retain if erasure.is_none() => {
|
||||||
let seconds=plan.diagnostics_retention_seconds.ok_or_else(||StoreError::InvalidInput("diagnostics retention deadline missing".into()))?;
|
let seconds=plan.diagnostics_retention_seconds.ok_or_else(||StoreError::InvalidInput("diagnostics retention deadline missing".into()))?;
|
||||||
let seconds=i64::try_from(seconds).map_err(|_|StoreError::InvalidInput("diagnostics retention deadline overflow".into()))?;
|
let seconds=i64::try_from(seconds).map_err(|_|StoreError::InvalidInput("diagnostics retention deadline overflow".into()))?;
|
||||||
let expires_at=(Utc::now()+chrono::Duration::seconds(seconds)).to_rfc3339();
|
let expires_at=(Utc::now()+chrono::Duration::seconds(seconds)).to_rfc3339();
|
||||||
tx.execute("INSERT OR IGNORE INTO worker_diagnostics_archives(operation_id,workspace_id,runtime_id,worker_id,policy_id,policy_digest,committed_at,expires_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8)",params![operation_id,workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.policy_id,plan.policy_digest,now,expires_at])?;
|
tx.execute("INSERT OR IGNORE INTO worker_diagnostics_archives(operation_id,workspace_id,runtime_id,worker_id,policy_id,policy_digest,committed_at,expires_at) VALUES(?1,?2,?3,?4,?5,?6,?7,?8)",params![operation_id,workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.policy_id,plan.policy_digest,now,expires_at])?;
|
||||||
}
|
}
|
||||||
DiagnosticsDisposition::Purge => {}
|
DiagnosticsDisposition::Purge | DiagnosticsDisposition::Retain => {}
|
||||||
}
|
}
|
||||||
if plan.metadata_disposition==MetadataDisposition::Tombstone{
|
if plan.metadata_disposition==MetadataDisposition::Tombstone{
|
||||||
tx.execute("INSERT OR IGNORE INTO worker_tombstones(workspace_id,runtime_id,worker_id,display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_digest,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,worker.display_name,worker.profile,worker.created_at,now,plan.archive_id,plan.policy_id,plan.policy_digest,operation_id])?;
|
tx.execute("INSERT OR IGNORE INTO worker_tombstones(workspace_id,runtime_id,worker_id,display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_digest,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,worker.display_name,worker.profile,worker.created_at,now,if erasure.is_some() { None } else { plan.archive_id.clone() },plan.policy_id,plan.policy_digest,operation_id])?;
|
||||||
}
|
}
|
||||||
tx.execute("UPDATE worker_workdir_links SET unlinked_at=?4 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND unlinked_at IS NULL",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,now])?;
|
tx.execute("UPDATE worker_workdir_links SET unlinked_at=?4 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND unlinked_at IS NULL",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,now])?;
|
||||||
let deleted=tx.execute("DELETE FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND updated_at=?4",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.worker_updated_at])?;
|
let deleted=tx.execute("DELETE FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND updated_at=?4",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.worker_updated_at])?;
|
||||||
@@ -595,6 +761,9 @@ impl SqliteWorkspaceStore {
|
|||||||
if deleted!=1{return Err(StoreError::InvalidInput(format!("stale:{}:removal fence changed",plan.plan_id)));}
|
if deleted!=1{return Err(StoreError::InvalidInput(format!("stale:{}:removal fence changed",plan.plan_id)));}
|
||||||
let catalog=commit_worker_catalog_removal(&tx,workspace_id,&plan.worker)?;
|
let catalog=commit_worker_catalog_removal(&tx,workspace_id,&plan.worker)?;
|
||||||
tx.execute("UPDATE worker_removal_operations SET state='succeeded',failure_category=NULL,updated_at=?1 WHERE operation_id=?2",params![now,operation_id])?;
|
tx.execute("UPDATE worker_removal_operations SET state='succeeded',failure_category=NULL,updated_at=?1 WHERE operation_id=?2",params![now,operation_id])?;
|
||||||
|
if let Some(ack) = erasure {
|
||||||
|
insert_erasure_reconciliation_audit(&tx, &plan, ack, Some(result))?;
|
||||||
|
}
|
||||||
tx.execute("INSERT OR IGNORE INTO worker_retention_audit_events(event_id,operation_id,workspace_id,event_kind,detail,created_at) VALUES(?1,?2,?3,'worker_removed',?4,?5)",params![stable("wre",operation_id),operation_id,workspace_id,format!("runtime_id={} worker_id={} session={} metadata={} diagnostics={}",plan.worker.runtime_id,plan.worker.worker_id,sess(plan.session_disposition),meta(plan.metadata_disposition),diag(plan.diagnostics_disposition)),now])?;
|
tx.execute("INSERT OR IGNORE INTO worker_retention_audit_events(event_id,operation_id,workspace_id,event_kind,detail,created_at) VALUES(?1,?2,?3,'worker_removed',?4,?5)",params![stable("wre",operation_id),operation_id,workspace_id,format!("runtime_id={} worker_id={} session={} metadata={} diagnostics={}",plan.worker.runtime_id,plan.worker.worker_id,sess(plan.session_disposition),meta(plan.metadata_disposition),diag(plan.diagnostics_disposition)),now])?;
|
||||||
tx.commit()?;plan.state=WorkerRemovalPlanState::Succeeded;plan.updated_at=now;Ok(WorkerRemovalCommit { plan, catalog })
|
tx.commit()?;plan.state=WorkerRemovalPlanState::Succeeded;plan.updated_at=now;Ok(WorkerRemovalCommit { plan, catalog })
|
||||||
}).map_err(map_error)
|
}).map_err(map_error)
|
||||||
@@ -969,6 +1138,32 @@ fn load_plan_q(c: &Connection, key: &str, id: &str) -> crate::Result<Option<Work
|
|||||||
.optional()
|
.optional()
|
||||||
.map_err(StoreError::from)
|
.map_err(StoreError::from)
|
||||||
}
|
}
|
||||||
|
fn insert_erasure_reconciliation_audit(
|
||||||
|
tx: &rusqlite::Transaction<'_>,
|
||||||
|
plan: &WorkerRemovalPlan,
|
||||||
|
ack: &RuntimeJournalErasureAck,
|
||||||
|
result: Option<&WorkerRetentionExecutionResult>,
|
||||||
|
) -> crate::Result<()> {
|
||||||
|
let proof = ack.domain_proof();
|
||||||
|
let kind = if result.is_some() {
|
||||||
|
"erased_removal_receipt"
|
||||||
|
} else {
|
||||||
|
"erased_archive_superseded"
|
||||||
|
};
|
||||||
|
let detail = serde_json::json!({
|
||||||
|
"input_fingerprint": plan.input_fingerprint,
|
||||||
|
"session_id": proof.session_id,
|
||||||
|
"archive_id": plan.archive_id,
|
||||||
|
"source_removed": result.map(|r| r.source_removed),
|
||||||
|
"old_diagnostics_retained": result.map(|r| r.diagnostics_retained),
|
||||||
|
"runtime_erasure_ack": { "session_id": proof.session_id, "source": ack.source },
|
||||||
|
"old_acceptance": if result.is_some() { "receipt_confirmed" } else { "unknown_archive_fenced" },
|
||||||
|
});
|
||||||
|
tx.execute("INSERT OR IGNORE INTO worker_retention_audit_events(event_id,operation_id,workspace_id,event_kind,detail,created_at) VALUES(?1,?2,?3,?4,?5,?6)",
|
||||||
|
params![stable("wre", &format!("{}:{kind}", plan.operation_id)), plan.operation_id, plan.workspace_id, kind, detail.to_string(), Utc::now().to_rfc3339()])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn mark_stale(
|
fn mark_stale(
|
||||||
tx: &rusqlite::Transaction<'_>,
|
tx: &rusqlite::Transaction<'_>,
|
||||||
plan: &WorkerRemovalPlan,
|
plan: &WorkerRemovalPlan,
|
||||||
|
|||||||
@@ -1889,8 +1889,98 @@ impl WorkerRemovalService {
|
|||||||
&self,
|
&self,
|
||||||
runtime: &RuntimeRegistry,
|
runtime: &RuntimeRegistry,
|
||||||
target: &RuntimeWorkerRef,
|
target: &RuntimeWorkerRef,
|
||||||
prepared: crate::retention::PreparedWorkerRemoval,
|
mut prepared: crate::retention::PreparedWorkerRemoval,
|
||||||
|
parent_workspace_deletion_operation_id: Option<&str>,
|
||||||
) -> std::result::Result<worker::WorkspaceResponse, String> {
|
) -> std::result::Result<worker::WorkspaceResponse, String> {
|
||||||
|
if prepared.plan.session_disposition
|
||||||
|
== worker_runtime::retention::SessionDisposition::Archive
|
||||||
|
{
|
||||||
|
let erasure_ack = match subjektiv_journal::erased_archive_removal_proof(
|
||||||
|
self, runtime, target, &prepared,
|
||||||
|
) {
|
||||||
|
Ok(ack) => ack,
|
||||||
|
Err(error) => {
|
||||||
|
return Ok(worker_remove_error_response(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"erased_removal_reconciliation_unconfirmed",
|
||||||
|
&error.to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Some(ack) = erasure_ack {
|
||||||
|
// Replay EXACT old input under the acknowledged permanent fence.
|
||||||
|
// Runtime can return an existing receipt, but cannot re-Archive.
|
||||||
|
match runtime.execute_worker_retention(target, prepared.runtime_request.clone()) {
|
||||||
|
Ok(result) => {
|
||||||
|
if let Err(error) = self
|
||||||
|
.store
|
||||||
|
.audit_erased_worker_removal_receipt(&prepared, &ack, &result)
|
||||||
|
{
|
||||||
|
return Ok(worker_retention_error_response(error));
|
||||||
|
}
|
||||||
|
return match self.worker_projection.publish_ordered(|| {
|
||||||
|
self.store
|
||||||
|
.commit_erased_worker_removal(
|
||||||
|
&self.workspace_id,
|
||||||
|
&prepared,
|
||||||
|
&result,
|
||||||
|
&ack,
|
||||||
|
)
|
||||||
|
.map(|commit| ((), commit.catalog))
|
||||||
|
}) {
|
||||||
|
Ok(()) => Ok(worker_remove_success_response(target)),
|
||||||
|
Err(error) => Ok(worker_retention_error_response(error)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
// Errors are unknown acceptance, not failure. A missing
|
||||||
|
// aggregate without a receipt remains unresolved.
|
||||||
|
let inventory = match runtime.worker_retention_inventory(target) {
|
||||||
|
Ok(inventory) => inventory,
|
||||||
|
Err(_) => {
|
||||||
|
return Ok(worker_remove_error_response(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"erased_removal_outcome_unknown",
|
||||||
|
"Old Runtime outcome cannot be reconciled; removal remains recoverable",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Err(error) = self
|
||||||
|
.store
|
||||||
|
.supersede_erased_worker_removal(&prepared, &ack, &inventory)
|
||||||
|
{
|
||||||
|
return Ok(worker_retention_error_response(error));
|
||||||
|
}
|
||||||
|
let request = crate::retention::WorkerRemovalPlanRequest {
|
||||||
|
workspace_id: self.workspace_id.clone(),
|
||||||
|
worker: target.clone(),
|
||||||
|
reason: prepared.plan.reason.clone(),
|
||||||
|
journal_erasure: Some(ack.domain_proof().clone()),
|
||||||
|
};
|
||||||
|
let plan = match self.store.plan_worker_removal(&request, &inventory) {
|
||||||
|
Ok(plan) => plan,
|
||||||
|
Err(error) => return Ok(worker_retention_error_response(error)),
|
||||||
|
};
|
||||||
|
prepared = match self.store.prepare_worker_removal_execution(
|
||||||
|
&self.workspace_id,
|
||||||
|
&plan.plan_id,
|
||||||
|
&plan.input_fingerprint,
|
||||||
|
) {
|
||||||
|
Ok(prepared) => prepared,
|
||||||
|
Err(error) => return Ok(worker_retention_error_response(error)),
|
||||||
|
};
|
||||||
|
if let Some(parent) = parent_workspace_deletion_operation_id {
|
||||||
|
self.store
|
||||||
|
.append_workspace_deletion_child_operation(
|
||||||
|
parent,
|
||||||
|
&prepared.plan.operation_id,
|
||||||
|
)
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if let Err(error) = subjektiv_journal::validate_removal_retention(self, target, &prepared) {
|
if let Err(error) = subjektiv_journal::validate_removal_retention(self, target, &prepared) {
|
||||||
return Ok(worker_remove_error_response(
|
return Ok(worker_remove_error_response(
|
||||||
StatusCode::CONFLICT,
|
StatusCode::CONFLICT,
|
||||||
@@ -2168,7 +2258,12 @@ impl WorkerRemovalService {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
return self
|
return self
|
||||||
.resume_worker_retention(runtime, &target, prepared)
|
.resume_worker_retention(
|
||||||
|
runtime,
|
||||||
|
&target,
|
||||||
|
prepared,
|
||||||
|
parent_workspace_deletion_operation_id,
|
||||||
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -878,6 +878,51 @@ pub(super) fn prepare_worker_removal(
|
|||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A successful erase response proves the Runtime's permanent fence. The reserved
|
||||||
|
/// archive covers transfer-before-Backend-commit crashes too. Never admit or generate.
|
||||||
|
pub(super) fn erased_archive_removal_proof(
|
||||||
|
service: &WorkerRemovalService,
|
||||||
|
runtime: &RuntimeRegistry,
|
||||||
|
target: &RuntimeWorkerRef,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
) -> Result<Option<crate::retention::RuntimeJournalErasureAck>> {
|
||||||
|
let Some(attr) = removal_attribution(service, target)? else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let store = crate::subjektiv::SubjektivStore::open(
|
||||||
|
&service.journal_storage,
|
||||||
|
&service.journal_registration,
|
||||||
|
)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?;
|
||||||
|
let Some(proof) =
|
||||||
|
crate::retention::ErasedJournalSession::validated(&service.workspace_id, &store, &attr)?
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let api = service.journal_dispatch_api.upgrade().ok_or_else(|| {
|
||||||
|
Error::Store("Workspace API unavailable for erased removal recovery".into())
|
||||||
|
})?;
|
||||||
|
reconcile_session_jobs(&api, &attr, true)?;
|
||||||
|
let source = runtime_api::SessionPublicSource::Retained {
|
||||||
|
worker_id: prepared.runtime_request.worker_id,
|
||||||
|
};
|
||||||
|
let ack = match proof.clone().erase_runtime_source(runtime, source) {
|
||||||
|
Ok(ack) => ack,
|
||||||
|
Err(original) => {
|
||||||
|
let Some(archive_id) = prepared.runtime_request.archive_id.clone() else {
|
||||||
|
return Err(original);
|
||||||
|
};
|
||||||
|
proof
|
||||||
|
.erase_runtime_source(
|
||||||
|
runtime,
|
||||||
|
runtime_api::SessionPublicSource::Archived { archive_id },
|
||||||
|
)
|
||||||
|
.map_err(|_| original)?
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Ok(Some(ack))
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn validate_removal_retention(
|
pub(super) fn validate_removal_retention(
|
||||||
service: &WorkerRemovalService,
|
service: &WorkerRemovalService,
|
||||||
target: &RuntimeWorkerRef,
|
target: &RuntimeWorkerRef,
|
||||||
|
|||||||
@@ -2141,3 +2141,503 @@ async fn serialized_removal_input_cannot_supply_journal_erasure_authority() {
|
|||||||
assert!(!f.store.journal_session_erased(&f.host).unwrap());
|
assert!(!f.store.journal_session_erased(&f.host).unwrap());
|
||||||
assert!(session_root(f.dir.path(), &f.target).exists());
|
assert!(session_root(f.dir.path(), &f.target).exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl SubjectRemovalFixture {
|
||||||
|
async fn reopen(self) -> Self {
|
||||||
|
let Self {
|
||||||
|
dir,
|
||||||
|
api,
|
||||||
|
dispatcher,
|
||||||
|
controller,
|
||||||
|
target,
|
||||||
|
attr,
|
||||||
|
host,
|
||||||
|
store,
|
||||||
|
} = self;
|
||||||
|
let config = api.config.clone();
|
||||||
|
drop(api);
|
||||||
|
drop(dispatcher);
|
||||||
|
drop(store);
|
||||||
|
let control = Arc::new(test_control_store(&config));
|
||||||
|
let dispatcher = Arc::new(
|
||||||
|
crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new(
|
||||||
|
config.clone(),
|
||||||
|
control.clone(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let api = WorkspaceApi::new_with_execution_backend_and_broker(
|
||||||
|
config,
|
||||||
|
control,
|
||||||
|
Arc::new(DeterministicExecutionBackend::default()),
|
||||||
|
BackendResourceBroker::default(),
|
||||||
|
HostExecutors::default(),
|
||||||
|
Some(dispatcher.clone()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let store = open_subjektiv_store(&api).unwrap();
|
||||||
|
Self {
|
||||||
|
dir,
|
||||||
|
api,
|
||||||
|
dispatcher,
|
||||||
|
controller,
|
||||||
|
target,
|
||||||
|
attr,
|
||||||
|
host,
|
||||||
|
store,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_erased_removal(&self) {
|
||||||
|
assert!(
|
||||||
|
self.api
|
||||||
|
.store
|
||||||
|
.get_worker_registry(TEST_WORKSPACE_ID, &self.target)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
self.api
|
||||||
|
.config_store
|
||||||
|
.worker_session_archive(TEST_WORKSPACE_ID, &self.target, &self.attr.session_id)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
self.api
|
||||||
|
.store
|
||||||
|
.list_backend_journal_jobs(TEST_WORKSPACE_ID, &self.attr.session_id, None, 100)
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert!(!session_root(self.dir.path(), &self.target).exists());
|
||||||
|
assert!(matches!(
|
||||||
|
source_retention(self.dir.path()).ensure_live(self.attr.session_id.parse().unwrap()),
|
||||||
|
Err(session_store::StoreError::SessionErased(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn erased_archive_plan_resend_and_reopen_replan_without_republishing_sources() {
|
||||||
|
for stage in ["planned", "executing", "failed"] {
|
||||||
|
for restart in [false, true] {
|
||||||
|
let mut f = SubjectRemovalFixture::new().await;
|
||||||
|
f.disable();
|
||||||
|
let policy = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.worker_retention_policy(TEST_WORKSPACE_ID)
|
||||||
|
.unwrap();
|
||||||
|
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||||
|
let archive_parent = f
|
||||||
|
.dir
|
||||||
|
.path()
|
||||||
|
.join(".test-embedded-runtime-store/archives/workers");
|
||||||
|
if stage == "planned" {
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE UPDATE OF state ON worker_removal_operations WHEN NEW.state='executing' BEGIN SELECT RAISE(ABORT,'injected execution interruption'); END;").unwrap();
|
||||||
|
} else {
|
||||||
|
// Fail Archive publication before any receipt. Suppress recording
|
||||||
|
// failure too to model the executing-at-Host-stop durable boundary.
|
||||||
|
std::fs::create_dir_all(archive_parent.parent().unwrap()).unwrap();
|
||||||
|
std::fs::write(&archive_parent, "publication blocked").unwrap();
|
||||||
|
if stage == "executing" {
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE UPDATE OF state ON worker_removal_operations WHEN NEW.state='failed' BEGIN SELECT RAISE(ABORT,'injected failure-save interruption'); END;").unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||||
|
conn.execute_batch("DROP TRIGGER IF EXISTS interrupt_removal")
|
||||||
|
.unwrap();
|
||||||
|
if stage != "planned" {
|
||||||
|
std::fs::remove_file(&archive_parent).unwrap();
|
||||||
|
}
|
||||||
|
let old = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(serde_json::to_value(old.plan.state).unwrap(), stage);
|
||||||
|
assert_eq!(
|
||||||
|
old.plan.session_disposition,
|
||||||
|
worker_runtime::retention::SessionDisposition::Archive
|
||||||
|
);
|
||||||
|
f.api
|
||||||
|
.erase_subjektiv_journal_session(
|
||||||
|
&f.attr.session_id,
|
||||||
|
"erase-interrupted-archive-plan",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
if restart {
|
||||||
|
f = f.reopen().await;
|
||||||
|
}
|
||||||
|
let response = f.remove();
|
||||||
|
assert_eq!(
|
||||||
|
response.status,
|
||||||
|
StatusCode::OK.as_u16(),
|
||||||
|
"{stage}/{restart}: {}",
|
||||||
|
response.body
|
||||||
|
);
|
||||||
|
let saved = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
saved.plan.session_disposition,
|
||||||
|
worker_runtime::retention::SessionDisposition::Purge
|
||||||
|
);
|
||||||
|
assert_eq!(saved.plan.reason, old.plan.reason);
|
||||||
|
assert_ne!(saved.plan.operation_id, old.plan.operation_id);
|
||||||
|
let (state, fingerprint, detail): (String, String, String) = conn.query_row(
|
||||||
|
"SELECT o.state,o.input_fingerprint,a.detail FROM worker_removal_operations o JOIN worker_retention_audit_events a USING(operation_id) WHERE o.operation_id=?1 AND a.event_kind='erased_archive_superseded'",
|
||||||
|
[&old.plan.operation_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||||
|
assert_eq!(state, "stale");
|
||||||
|
assert_eq!(fingerprint, old.plan.input_fingerprint);
|
||||||
|
let detail: Value = serde_json::from_str(&detail).unwrap();
|
||||||
|
assert_eq!(detail["old_acceptance"], "unknown_archive_fenced");
|
||||||
|
assert!(detail["source_removed"].is_null());
|
||||||
|
f.assert_erased_removal();
|
||||||
|
assert_eq!(
|
||||||
|
f.api
|
||||||
|
.config_store
|
||||||
|
.worker_retention_policy(TEST_WORKSPACE_ID)
|
||||||
|
.unwrap(),
|
||||||
|
policy
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn erased_completed_archive_receipt_is_audited_without_republishing_on_resend_or_reopen() {
|
||||||
|
for restart in [false, true] {
|
||||||
|
let mut f = SubjectRemovalFixture::new().await;
|
||||||
|
f.disable();
|
||||||
|
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||||
|
// Runtime has committed source_removed, but Backend commit/response is lost.
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected metadata commit interruption'); END;").unwrap();
|
||||||
|
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||||
|
conn.execute_batch("DROP TRIGGER interrupt_removal")
|
||||||
|
.unwrap();
|
||||||
|
let old = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
old.plan.state,
|
||||||
|
crate::retention::WorkerRemovalPlanState::Failed
|
||||||
|
);
|
||||||
|
let receipt = f
|
||||||
|
.api
|
||||||
|
.runtime
|
||||||
|
.execute_worker_retention(&f.target, old.runtime_request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert!(receipt.source_removed);
|
||||||
|
let archive_id = old.plan.archive_id.as_ref().unwrap();
|
||||||
|
let archive_source = f
|
||||||
|
.dir
|
||||||
|
.path()
|
||||||
|
.join(".test-embedded-runtime-store/archives/workers")
|
||||||
|
.join(archive_id)
|
||||||
|
.join("session");
|
||||||
|
assert!(archive_source.exists());
|
||||||
|
// The domain tombstone commits even though retained-source resolution fails.
|
||||||
|
// Recovery must locate only this plan's exact reserved archive, not publish it.
|
||||||
|
assert!(
|
||||||
|
f.api
|
||||||
|
.erase_subjektiv_journal_session(&f.attr.session_id, "erase-transferred-original")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(f.store.journal_session_erased(&f.host).unwrap());
|
||||||
|
if restart {
|
||||||
|
f = f.reopen().await;
|
||||||
|
}
|
||||||
|
let response = f.remove();
|
||||||
|
assert_eq!(
|
||||||
|
response.status,
|
||||||
|
StatusCode::OK.as_u16(),
|
||||||
|
"{restart}: {}",
|
||||||
|
response.body
|
||||||
|
);
|
||||||
|
let saved = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(saved.plan.operation_id, old.plan.operation_id);
|
||||||
|
assert_eq!(saved.plan.input_fingerprint, old.plan.input_fingerprint);
|
||||||
|
assert_eq!(saved.plan.session_disposition, old.plan.session_disposition);
|
||||||
|
let detail: String = conn.query_row("SELECT detail FROM worker_retention_audit_events WHERE operation_id=?1 AND event_kind='erased_removal_receipt'", [&old.plan.operation_id], |r| r.get(0)).unwrap();
|
||||||
|
let detail: Value = serde_json::from_str(&detail).unwrap();
|
||||||
|
assert_eq!(detail["source_removed"], true);
|
||||||
|
assert_eq!(detail["old_acceptance"], "receipt_confirmed");
|
||||||
|
assert!(!archive_source.exists());
|
||||||
|
f.assert_erased_removal();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn erased_archive_without_receipt_or_inventory_remains_unknown_without_replanning() {
|
||||||
|
let f = SubjectRemovalFixture::new().await;
|
||||||
|
f.disable();
|
||||||
|
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE UPDATE OF state ON worker_removal_operations WHEN NEW.state='executing' BEGIN SELECT RAISE(ABORT,'injected interruption'); END;").unwrap();
|
||||||
|
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||||
|
conn.execute_batch("DROP TRIGGER interrupt_removal")
|
||||||
|
.unwrap();
|
||||||
|
let old = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
f.api
|
||||||
|
.erase_subjektiv_journal_session(&f.attr.session_id, "erase-with-unknown-outcome")
|
||||||
|
.unwrap();
|
||||||
|
// Corrupt/missing aggregate after an unacknowledged operation is NOT evidence
|
||||||
|
// that source_removed completed. Runtime has no matching receipt here.
|
||||||
|
std::fs::remove_file(
|
||||||
|
session_root(f.dir.path(), &f.target)
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.join("worker.json"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let response = f.remove();
|
||||||
|
assert_eq!(
|
||||||
|
response.status,
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE.as_u16(),
|
||||||
|
"{}",
|
||||||
|
response.body
|
||||||
|
);
|
||||||
|
let body: Value = serde_json::from_str(&response.body).unwrap();
|
||||||
|
assert_eq!(body["code"], "erased_removal_outcome_unknown");
|
||||||
|
let saved = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(saved.plan.operation_id, old.plan.operation_id);
|
||||||
|
assert_eq!(
|
||||||
|
saved.plan.state,
|
||||||
|
crate::retention::WorkerRemovalPlanState::Executing
|
||||||
|
);
|
||||||
|
let count: i64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT COUNT(*) FROM worker_removal_operations WHERE worker_id=?1",
|
||||||
|
[&f.target.worker_id],
|
||||||
|
|r| r.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn erased_archive_retained_diagnostics_receipt_finalizes_only_after_physical_ack_on_resend_or_reopen()
|
||||||
|
{
|
||||||
|
for restart in [false, true] {
|
||||||
|
for interrupt_erase in [false, true] {
|
||||||
|
let mut f = SubjectRemovalFixture::new().await;
|
||||||
|
f.disable();
|
||||||
|
let policy = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.worker_retention_policy(TEST_WORKSPACE_ID)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let policy = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.update_worker_retention_policy(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&policy.policy_digest,
|
||||||
|
&crate::retention::WorkerRetentionPolicyUpdate {
|
||||||
|
policy_id: "fixture-retained-diagnostics".into(),
|
||||||
|
session_disposition: policy.session_disposition,
|
||||||
|
metadata_disposition: policy.metadata_disposition,
|
||||||
|
archive_retention: policy.archive_retention,
|
||||||
|
diagnostics_disposition:
|
||||||
|
worker_runtime::retention::DiagnosticsDisposition::Retain,
|
||||||
|
diagnostics_retention_seconds: Some(3600),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let worker_root = session_root(f.dir.path(), &f.target)
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.to_path_buf();
|
||||||
|
std::fs::create_dir_all(worker_root.join("runs/fixture-run")).unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
worker_root.join("runs/fixture-run/worker.out.log"),
|
||||||
|
"diagnostic original",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let conn = rusqlite::Connection::open(&f.api.config.database_path).unwrap();
|
||||||
|
// Real Runtime receipt exists; Backend commit/response is interrupted.
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_removal BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected metadata commit interruption'); END;").unwrap();
|
||||||
|
assert_ne!(f.remove().status, StatusCode::OK.as_u16());
|
||||||
|
conn.execute_batch("DROP TRIGGER interrupt_removal")
|
||||||
|
.unwrap();
|
||||||
|
let old = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let runtime_root = f.dir.path().join(".test-embedded-runtime-store");
|
||||||
|
let receipt_path = runtime_root
|
||||||
|
.join("retention/operations")
|
||||||
|
.join(format!("{}.json", old.plan.operation_id));
|
||||||
|
let receipt_bytes = std::fs::read(&receipt_path).unwrap();
|
||||||
|
let receipt = f
|
||||||
|
.api
|
||||||
|
.runtime
|
||||||
|
.execute_worker_retention(&f.target, old.runtime_request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert!(receipt.source_removed);
|
||||||
|
assert!(receipt.diagnostics_retained);
|
||||||
|
let diagnostics_root = runtime_root
|
||||||
|
.join("archives/diagnostics")
|
||||||
|
.join(&old.plan.operation_id);
|
||||||
|
let diagnostics_manifest = diagnostics_root.join("manifest.json");
|
||||||
|
let diagnostics_bytes = std::fs::read(&diagnostics_manifest).unwrap();
|
||||||
|
let archive_source = runtime_root
|
||||||
|
.join("archives/workers")
|
||||||
|
.join(old.plan.archive_id.as_ref().unwrap())
|
||||||
|
.join("session");
|
||||||
|
assert!(diagnostics_root.exists());
|
||||||
|
assert!(archive_source.exists());
|
||||||
|
// Domain erasure succeeds, but no physical acknowledgement can be
|
||||||
|
// claimed from the old retained locator after aggregate removal.
|
||||||
|
assert!(
|
||||||
|
f.api
|
||||||
|
.erase_subjektiv_journal_session(
|
||||||
|
&f.attr.session_id,
|
||||||
|
"erase-retained-diagnostics"
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
if interrupt_erase {
|
||||||
|
// Runtime must reject mismatched diagnostics identity rather than
|
||||||
|
// acknowledge removal of an unrelated operation. No fake Host ack.
|
||||||
|
let mut damaged: Value = serde_json::from_slice(&diagnostics_bytes).unwrap();
|
||||||
|
damaged["input_fingerprint"] = json!("unrelated-operation");
|
||||||
|
std::fs::write(&diagnostics_manifest, serde_json::to_vec(&damaged).unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let response = f.remove();
|
||||||
|
assert_eq!(
|
||||||
|
response.status,
|
||||||
|
StatusCode::CONFLICT.as_u16(),
|
||||||
|
"{}",
|
||||||
|
response.body
|
||||||
|
);
|
||||||
|
let body: Value = serde_json::from_str(&response.body).unwrap();
|
||||||
|
assert_eq!(body["code"], "erased_removal_reconciliation_unconfirmed");
|
||||||
|
assert!(
|
||||||
|
f.api
|
||||||
|
.store
|
||||||
|
.get_worker_registry(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
let count: i64 = conn.query_row("SELECT COUNT(*) FROM worker_retention_audit_events WHERE operation_id=?1 AND event_kind='erased_removal_receipt'", [&old.plan.operation_id], |r| r.get(0)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
count, 0,
|
||||||
|
"failed acknowledgement cannot authorize finalization/audit"
|
||||||
|
);
|
||||||
|
assert_eq!(std::fs::read(&receipt_path).unwrap(), receipt_bytes);
|
||||||
|
let waiting = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(waiting.plan.operation_id, old.plan.operation_id);
|
||||||
|
assert_eq!(waiting.plan.input_fingerprint, old.plan.input_fingerprint);
|
||||||
|
assert_eq!(
|
||||||
|
waiting.plan.state,
|
||||||
|
crate::retention::WorkerRemovalPlanState::Executing
|
||||||
|
);
|
||||||
|
let plans: i64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT COUNT(*) FROM worker_removal_operations WHERE worker_id=?1",
|
||||||
|
[&f.target.worker_id],
|
||||||
|
|r| r.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
plans, 1,
|
||||||
|
"missing physical ack cannot authorize a replacement plan"
|
||||||
|
);
|
||||||
|
std::fs::write(&diagnostics_manifest, &diagnostics_bytes).unwrap();
|
||||||
|
}
|
||||||
|
if restart {
|
||||||
|
f = f.reopen().await;
|
||||||
|
}
|
||||||
|
let response = f.remove();
|
||||||
|
assert_eq!(
|
||||||
|
response.status,
|
||||||
|
StatusCode::OK.as_u16(),
|
||||||
|
"restart={restart}, interrupted={interrupt_erase}: {}",
|
||||||
|
response.body
|
||||||
|
);
|
||||||
|
assert!(!diagnostics_root.exists());
|
||||||
|
assert!(!archive_source.exists());
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&receipt_path).unwrap(),
|
||||||
|
receipt_bytes,
|
||||||
|
"historical Runtime receipt must not be rewritten"
|
||||||
|
);
|
||||||
|
let saved = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &f.target)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(saved.plan.operation_id, old.plan.operation_id);
|
||||||
|
assert_eq!(saved.plan.input_fingerprint, old.plan.input_fingerprint);
|
||||||
|
assert_eq!(
|
||||||
|
saved.plan.diagnostics_disposition,
|
||||||
|
old.plan.diagnostics_disposition
|
||||||
|
);
|
||||||
|
let detail: String = conn.query_row("SELECT detail FROM worker_retention_audit_events WHERE operation_id=?1 AND event_kind='erased_removal_receipt'", [&old.plan.operation_id], |r| r.get(0)).unwrap();
|
||||||
|
let detail: Value = serde_json::from_str(&detail).unwrap();
|
||||||
|
assert_eq!(detail["source_removed"], true);
|
||||||
|
assert_eq!(detail["old_diagnostics_retained"], true);
|
||||||
|
assert_eq!(
|
||||||
|
detail["runtime_erasure_ack"]["session_id"],
|
||||||
|
f.attr.session_id
|
||||||
|
);
|
||||||
|
let diagnostics_catalog: i64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT COUNT(*) FROM worker_diagnostics_archives WHERE operation_id=?1",
|
||||||
|
[&old.plan.operation_id],
|
||||||
|
|r| r.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(diagnostics_catalog, 0);
|
||||||
|
let tombstone_archive: Option<String> = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT archive_id FROM worker_tombstones WHERE operation_id=?1",
|
||||||
|
[&old.plan.operation_id],
|
||||||
|
|r| r.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(tombstone_archive.is_none());
|
||||||
|
f.assert_erased_removal();
|
||||||
|
assert_eq!(
|
||||||
|
f.api
|
||||||
|
.config_store
|
||||||
|
.worker_retention_policy(TEST_WORKSPACE_ID)
|
||||||
|
.unwrap(),
|
||||||
|
Some(policy)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1451,6 +1451,40 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
|||||||
"Worker retention authority is unavailable".to_string(),
|
"Worker retention authority is unavailable".to_string(),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
fn supersede_erased_worker_removal(
|
||||||
|
&self,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||||
|
inventory: &worker_runtime::retention::WorkerRetentionInventory,
|
||||||
|
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
|
||||||
|
let _ = (prepared, proof, inventory);
|
||||||
|
Err(crate::retention::WorkerRetentionError::Invalid(
|
||||||
|
"Erased removal reconciliation authority is unavailable".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
fn audit_erased_worker_removal_receipt(
|
||||||
|
&self,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||||
|
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||||
|
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
|
||||||
|
let _ = (prepared, proof, result);
|
||||||
|
Err(crate::retention::WorkerRetentionError::Invalid(
|
||||||
|
"Erased removal reconciliation authority is unavailable".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
fn commit_erased_worker_removal(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||||
|
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||||
|
) -> std::result::Result<WorkerRemovalCommit, crate::retention::WorkerRetentionError> {
|
||||||
|
let _ = (workspace_id, prepared, result, proof);
|
||||||
|
Err(crate::retention::WorkerRetentionError::Invalid(
|
||||||
|
"Erased removal reconciliation authority is unavailable".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>>;
|
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>>;
|
||||||
fn upsert_repository(&self, record: &RepositoryRecord) -> Result<()>;
|
fn upsert_repository(&self, record: &RepositoryRecord) -> Result<()>;
|
||||||
fn insert_repository(&self, record: &RepositoryRecord) -> Result<RepositoryInsertOutcome>;
|
fn insert_repository(&self, record: &RepositoryRecord) -> Result<RepositoryInsertOutcome>;
|
||||||
@@ -5438,6 +5472,37 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn supersede_erased_worker_removal(
|
||||||
|
&self,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||||
|
inventory: &worker_runtime::retention::WorkerRetentionInventory,
|
||||||
|
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
|
||||||
|
SqliteWorkspaceStore::supersede_erased_worker_removal(self, prepared, proof, inventory)
|
||||||
|
}
|
||||||
|
fn audit_erased_worker_removal_receipt(
|
||||||
|
&self,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||||
|
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||||
|
) -> std::result::Result<(), crate::retention::WorkerRetentionError> {
|
||||||
|
SqliteWorkspaceStore::audit_erased_worker_removal_receipt(self, prepared, proof, result)
|
||||||
|
}
|
||||||
|
fn commit_erased_worker_removal(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||||
|
proof: &crate::retention::RuntimeJournalErasureAck,
|
||||||
|
) -> std::result::Result<WorkerRemovalCommit, crate::retention::WorkerRetentionError> {
|
||||||
|
SqliteWorkspaceStore::commit_erased_worker_removal(
|
||||||
|
self,
|
||||||
|
workspace_id,
|
||||||
|
prepared,
|
||||||
|
result,
|
||||||
|
proof,
|
||||||
|
)
|
||||||
|
}
|
||||||
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>> {
|
fn list_workspaces(&self) -> Result<Vec<WorkspaceRecord>> {
|
||||||
self.with_conn(|conn| {
|
self.with_conn(|conn| {
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(
|
||||||
|
|||||||
Reference in New Issue
Block a user