feat: replace Subject retirement with durable explicit deletion

This commit is contained in:
2026-10-28 11:18:43 +09:00
parent 618880d38d
commit a06f6bfb3d
71 changed files with 7448 additions and 315 deletions
@@ -478,7 +478,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
(101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into()),
(103, "Passkey metadata and atomic addition results".into()),
(104, "Converge Passkey results and Job input retention".into())
(104, "Converge Passkey results and Job input retention".into()),
(105, "Durable Subject deletion snapshots and Job admission fences".into())
]
);
for (table, rows) in retained.iter().zip(original.iter()) {
+102 -1
View File
@@ -1292,6 +1292,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
})
}
fn subject_delete_worker(
&self,
_request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
Err("Subject deletion fencing is unavailable on this Runtime".into())
}
fn recover_subject_create(
&self,
_request: runtime_api::SubjectCreateRecoveryRequest,
@@ -2269,6 +2276,21 @@ impl RuntimeRegistry {
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
}
pub fn subject_delete_worker(
&self,
runtime_id: &str,
request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.subject_delete_worker(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "subject_delete_worker_unproven".into(),
message,
})
}
pub fn recover_subject_create(
&self,
runtime_id: &str,
@@ -3367,6 +3389,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
})
}
fn subject_delete_worker(
&self,
request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
if request.workspace_id != self.workspace_id {
return Err("Subject deletion Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
self.runtime
.subject_delete_worker_scoped(&scope, request)
.map_err(|e| e.to_string())
}
fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
@@ -4255,7 +4290,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
}
if path.contains("/retention/")
|| (path.starts_with("/v1/workers/") && method == "DELETE")
|| (path == "/v1/subject-create-recovery" && method == "POST")
|| (matches!(
path,
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
) && method == "POST")
{
return "workers:delete";
}
@@ -5656,6 +5694,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
})
}
fn subject_delete_worker(
&self,
request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.subject_delete_worker(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
@@ -8507,6 +8557,57 @@ mod tests {
);
}
#[test]
fn remote_subject_delete_uses_generated_exact_identity_transport() {
assert_eq!(
workspace_runtime_operation("POST", "/v1/subject-delete-worker"),
"workers:delete"
);
let request = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: "workspace-test".into(),
worker_id: EmbeddedWorkerId::now_v7(),
create_fingerprint: "sha256:reserved-create".into(),
operation_id: "delete-worker-1".into(),
};
let response = runtime_api::SubjectDeleteWorkerResponse {
originals_absent: true,
workspace_id: request.workspace_id.clone(),
worker_id: request.worker_id,
create_fingerprint: request.create_fingerprint.clone(),
operation_id: request.operation_id.clone(),
};
// Only remote transport is substituted; actual Controller shutdown and
// signed admission are covered by worker-runtime's independent suites.
let (base_url, server) = serve_mock_http(vec![mock_response(
"POST",
"/v1/subject-delete-worker",
true,
200,
serde_json::to_string(&response).unwrap(),
)]);
let registry = RuntimeRegistry::new(Vec::new());
registry.register(
RemoteWorkerRuntime::new(
RemoteRuntimeConfig::new(
"remote:delete",
"Deletion transport fixture",
base_url,
Some("secret-token".into()),
),
"workspace-test".into(),
"http://127.0.0.1:8787".into(),
)
.unwrap(),
);
assert_eq!(
registry
.subject_delete_worker("remote:delete", request)
.unwrap(),
response
);
server.join().unwrap();
}
#[test]
fn remote_subject_create_recovery_uses_generated_management_transport() {
assert_eq!(
@@ -367,14 +367,54 @@ async fn implementation_result_rejects_forged_evidence_and_rolls_back_without_no
.unwrap();
}
#[tokio::test]
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
let (dir, store) = tracking_store_fixture().await;
tracking_store_assign(&store, "delegation", "assign");
let worker = routing_worker_assign(&store, "implementation").worker;
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
store.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER notification_mr_event;
// Preserve seeded assignment evidence while rebuilding the actual pre-business
// capture composition. Changing only a receipt leaves schema105 admission fences
// and erasure guards in an otherwise legacy fixture.
fn reconstruct_implementation_schema96(conn: &Connection) -> Result<()> {
assert_eq!(current_schema_version(conn)?, 105);
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("retained96.db");
prepare_retained_schema(&path, 96);
let retained = Connection::open(path)?;
for trigger in [
"notification_sources_no_delete",
"notification_decisions_no_delete",
"notification_requests_no_delete",
"notification_acknowledgements_no_delete",
"notification_failures_no_delete",
"notification_job_targets_no_delete",
"notification_job_legacy_no_delete",
"notification_source_preparation_failures_no_delete",
] {
let definition: String = retained.query_row(
"SELECT sql FROM sqlite_schema WHERE type='trigger' AND name=?1",
[trigger],
|row| row.get(0),
)?;
conn.execute_batch(&format!("DROP TRIGGER {trigger}; {definition};"))?;
}
for trigger in [
"subject_deleted_job_insert",
"subject_deleted_job_result",
"subject_deleted_attempt_insert",
"subject_deleted_attempt_result",
"subject_deleted_worker_reservation",
"subject_deleted_notification_source",
"subject_deleted_notification_payload",
"subject_deleted_notification_send",
] {
conn.execute_batch(&format!("DROP TRIGGER {trigger};"))?;
}
// Children before parents; dropping the tables also drops snapshot guards.
conn.execute_batch(
"DROP TABLE subject_deletion_allocations;
DROP TABLE subject_deletion_jobs;
DROP TABLE subject_deletion_workers;
DROP TABLE subject_deletions;
UPDATE __yoi_schema_migrations SET version=104 WHERE version=105;",
)?;
conn.execute_batch(
"DROP TRIGGER notification_mr_event;
DROP TRIGGER notification_mr_source;
DROP TRIGGER notification_implementation_result;
DROP TRIGGER IF EXISTS implementation_delegation_capture;
@@ -396,7 +436,43 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;",
)?;
assert_eq!(current_schema_version(conn)?, 96);
assert_eq!(conn.query_row(
"SELECT count(*) FROM sqlite_schema WHERE name LIKE 'subject_delet%' OR sql LIKE '%subject_deletion%'",
[], |row| row.get::<_, i64>(0),
)?, 0);
for table in [
"notification_sources",
"notification_decisions",
"notification_requests",
"notification_acknowledgements",
"notification_failures",
"notification_job_targets",
"notification_job_legacy_receipts",
"notification_source_preparation_failures",
"backend_jobs",
"backend_job_attempts",
"worker_create_reservations",
] {
assert_eq!(
state_contract_schema_signature(conn, table),
state_contract_schema_signature(&retained, table),
"schema96 {table}"
);
}
Ok(())
}
#[tokio::test]
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
let (dir, store) = tracking_store_fixture().await;
tracking_store_assign(&store, "delegation", "assign");
let worker = routing_worker_assign(&store, "implementation").worker;
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
store.with_conn(|conn| {
reconstruct_implementation_schema96(conn)?;
migrate_business_notifications_v96_to_v97(conn)?;
assert_eq!(current_schema_version(conn)?, 97);
Ok(())
@@ -522,24 +598,7 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
store.with_conn(|conn| {
conn.execute("INSERT INTO ticket_assignment_operation_fences VALUES('w','operation-known-implementation','delegation','r','worker','workspace-orchestrator','tracking-binding',?1)",[serde_json::to_string(&tracking_store_source()).unwrap()])?;
// Reconstruct the prior composition before its new business capture existed.
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TRIGGER passkey_metadata_insert;
DROP TABLE passkey_addition_receipts;
DROP TABLE passkey_metadata;
DROP TRIGGER backend_job_input_retirement;
DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations;
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
reconstruct_implementation_schema96(conn)?;
conn.execute_batch("INSERT INTO merge_requests VALUES('w','historical-mr','main','open','refs/heads/work','refs/heads/main','3','3');
INSERT INTO merge_request_ticket_relations VALUES('w','historical-mr','tracking','implements','3');
INSERT INTO merge_request_thread_events VALUES('w','historical-mr','approved-event',1,'review','{\"subject_ref\":\"source-a\",\"decision\":\"approve\"}',NULL,'3');
@@ -691,24 +750,7 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
];
store.with_conn(|conn| {
if historical {
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TRIGGER passkey_metadata_insert;
DROP TABLE passkey_addition_receipts;
DROP TABLE passkey_metadata;
DROP TRIGGER backend_job_input_retirement;
DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations;
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
reconstruct_implementation_schema96(conn)?;
}
conn.execute_batch("INSERT INTO merge_requests VALUES('w','actor-mr','main','open','work','main','3','3');
INSERT INTO merge_request_ticket_relations VALUES('w','actor-mr','tracking','implements','3');")?;
@@ -1757,3 +1757,5 @@ WHEN (OLD.runtime_id != NEW.runtime_id OR OLD.worker_id != NEW.worker_id OR OLD.
WHERE workspace_id = OLD.workspace_id AND runtime_id = OLD.runtime_id
AND worker_id = OLD.worker_id)
BEGIN SELECT RAISE(ABORT, 'Worker has unfinished Ticket work'); END;
-- Subject deletion schema is composed by store.rs.
@@ -444,6 +444,28 @@ fn job_result_facts(
pub(super) fn deliver_job_result(api: &WorkspaceApi, acceptance: &BackendJobResultAcceptance) {
use crate::store::JobNotificationFacts;
// A recovered/captured result is not delivery authority. Serialize even the
// Runtime send with deletion, so no prepared bytes escape after its fence.
let subject = subjektiv_deletion::job_subject(&acceptance.job.request);
let gate = match subject
.map(|subject| subjektiv_journal::lifecycle_lock(api, subject))
.transpose()
{
Ok(gate) => gate,
Err(error) => {
tracing::warn!(%error, "Subject Job notification gate unavailable");
return;
}
};
let _guard = match gate.as_ref().map(|gate| gate.lock()).transpose() {
Ok(guard) => guard,
Err(_) => return,
};
if let Some(subject) = subject
&& subjektiv_deletion::require_available(api, subject).is_err()
{
return;
}
let Some(_permit) = api.notification_delivery_lifecycle.begin() else {
return;
};
@@ -849,7 +849,16 @@ async fn notification_routing_fresh_and_upgraded_schema_and_guards_match() {
let snapshot = |conn: &Connection| -> Vec<(String, String)> {
conn.prepare("SELECT name,sql FROM sqlite_schema WHERE name LIKE 'notification_%' OR name LIKE 'ticket_assignment_content%' OR name IN ('ticket_assignment_title_input','ticket_assignment_initial_post_input') ORDER BY name").unwrap().query_map([],|r|Ok((r.get(0)?,r.get(1)?))).unwrap().collect::<rusqlite::Result<_>>().unwrap()
};
let upgraded = snapshot(&conn);
assert_eq!(current_schema_version(&conn).unwrap(), 98);
drop(conn);
// Compare the complete current composition, not upgraded98 against fresh105.
let upgraded_store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let upgraded = upgraded_store
.with_conn(|conn| {
assert_eq!(current_schema_version(conn)?, 105);
Ok(snapshot(conn))
})
.unwrap();
let (fresh_directory, fresh) = tracking_store_fixture().await;
fresh
.with_conn(|conn| {
+167 -2
View File
@@ -333,6 +333,15 @@ impl SqliteWorkspaceStore {
return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()}));
}
};
if crate::store::subject_deletion_worker_kind(&tx, &req.workspace_id, &req.worker)?.is_some() {
let prior = tx.prepare("SELECT plan_id FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('executing','failed')")?
.query_map(params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|r.get::<_,String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
for id in prior {
let prior=load_plan(&tx,&id)?.ok_or_else(||StoreError::Store("removal plan disappeared".into()))?;
require_current_subject_deletion_plan(&tx,&prior)?;
}
}
let mut blockers=Vec::new();
if worker.retention_state=="pinned" { blockers.push(WorkerRemovalBlocker::Hold); }
if let Some((assignment_id,ticket_id))=tx.query_row("SELECT a.assignment_id,a.ticket_id FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3",params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()? {
@@ -363,6 +372,7 @@ impl SqliteWorkspaceStore {
let mut plan=load_plan(&tx,plan_id)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:plan missing")))?;
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
if plan.input_fingerprint!=fp{return Err(StoreError::InvalidInput(format!("fingerprint:{}",plan.operation_id)));}
require_current_subject_deletion_plan(&tx, &plan)?;
if plan.state==WorkerRemovalPlanState::Succeeded{tx.commit()?;return Ok(plan);}
if plan.state==WorkerRemovalPlanState::Blocked {
return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));
@@ -420,6 +430,8 @@ impl SqliteWorkspaceStore {
WorkerRetentionError::Invalid("Worker id must be a canonical UUIDv7".to_string())
})?;
let removed_at = plan.created_at.clone();
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
.map_err(map_error)?;
let prior_failure_category = plan.failure_category.clone();
Ok(PreparedWorkerRemoval {
runtime_request: WorkerRetentionExecutionRequest {
@@ -459,8 +471,8 @@ impl SqliteWorkspaceStore {
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
)
)
ORDER BY CASE state WHEN 'succeeded' THEN 0 ELSE 1 END,
created_at DESC LIMIT 1",
ORDER BY CASE WHEN EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=worker_removal_operations.workspace_id AND d.runtime_id=worker_removal_operations.runtime_id AND d.worker_id=worker_removal_operations.worker_id AND d.prior_removal_operation_id=worker_removal_operations.operation_id) THEN 0 WHEN state='succeeded' THEN 1 ELSE 2 END,
created_at DESC,operation_id DESC LIMIT 1",
params![workspace_id, worker.runtime_id, worker.worker_id],
|row| row.get::<_, String>(0),
)
@@ -474,6 +486,8 @@ impl SqliteWorkspaceStore {
let Some(plan) = plan else {
return Ok(None);
};
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
.map_err(map_error)?;
let prior_failure_category = plan.failure_category.clone();
let worker_id = plan
.worker
@@ -659,6 +673,7 @@ impl SqliteWorkspaceStore {
let mut plan=load_plan_op(&tx,operation_id)?.ok_or_else(||StoreError::InvalidInput("operation missing".into()))?;
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
if plan.input_fingerprint!=fp||result.input_fingerprint!=fp||result.operation_id!=operation_id{return Err(StoreError::InvalidInput(format!("fingerprint:{operation_id}")));}
require_current_subject_deletion_plan(&tx, &plan)?;
if plan.state==WorkerRemovalPlanState::Succeeded{
// Repair projection fences left by the pre-atomic implementation and
// return a replayable change for a broadcast missed after commit.
@@ -1024,6 +1039,49 @@ fn load_removal_policy(
let Some(mut policy) = load_policy(c, w)? else {
return Ok(None);
};
// Snapshot authority outranks normal policy and Journal cleanup policy. Body
// and delegated Sessions must never inherit the ephemeral Job exception.
if let Some(is_job) = crate::store::subject_deletion_worker_kind(c, w, worker)? {
let update = WorkerRetentionPolicyUpdate {
policy_id: if is_job {
"subject-deletion-job-v1"
} else {
"subject-deletion-conversation-v1"
}
.into(),
session_disposition: if is_job {
SessionDisposition::Purge
} else {
SessionDisposition::Archive
},
metadata_disposition: MetadataDisposition::Tombstone,
archive_retention: ArchiveRetention::Forever,
// Runtime calls diagnostic archiving Retain, not Archive. Preserve
// the Workspace's duration, or thirty days when it normally purges.
diagnostics_disposition: if is_job {
DiagnosticsDisposition::Purge
} else {
DiagnosticsDisposition::Retain
},
diagnostics_retention_seconds: if is_job {
None
} else {
Some(
policy
.diagnostics_retention_seconds
.unwrap_or(30 * 24 * 60 * 60),
)
},
};
policy.policy_id = update.policy_id.clone();
policy.policy_digest = worker_retention_policy_digest(&update);
policy.session_disposition = update.session_disposition;
policy.metadata_disposition = update.metadata_disposition;
policy.archive_retention = update.archive_retention;
policy.diagnostics_disposition = update.diagnostics_disposition;
policy.diagnostics_retention_seconds = update.diagnostics_retention_seconds;
return Ok(Some(policy));
}
if erased_journal || crate::store::terminal_journal_job_worker(c, w, worker)? {
let update = WorkerRetentionPolicyUpdate {
policy_id: if erased_journal {
@@ -1047,6 +1105,39 @@ fn load_removal_policy(
}
Ok(Some(policy))
}
fn require_current_subject_deletion_plan(
c: &Connection,
plan: &WorkerRemovalPlan,
) -> crate::Result<()> {
if plan.state == WorkerRemovalPlanState::Succeeded
&& crate::store::subject_deletion_prior_removal_matches(
c,
&plan.workspace_id,
&plan.worker,
&plan.operation_id,
)?
{
// An exact successful receipt pinned before Subject deletion owns its
// independent original/archive lifecycle. Never retarget its policy.
return Ok(());
}
if crate::store::subject_deletion_worker_kind(c, &plan.workspace_id, &plan.worker)?.is_some() {
let current = load_removal_policy(c, &plan.workspace_id, &plan.worker, false)?
.ok_or_else(|| StoreError::InvalidInput("Subject deletion policy missing".into()))?;
if plan.session_disposition != current.session_disposition
|| plan.metadata_disposition != current.metadata_disposition
|| plan.archive_retention != current.archive_retention
|| plan.diagnostics_disposition != current.diagnostics_disposition
{
return Err(stale_error(
plan,
"Subject deletion retention differs from saved plan; reconcile canonical Runtime removal before retry",
));
}
}
Ok(())
}
fn load_policy(c: &Connection, w: &str) -> crate::Result<Option<WorkerRetentionPolicy>> {
c.query_row(
"SELECT p.policy_id,p.policy_digest,r.session_disposition,r.metadata_disposition,
@@ -2736,4 +2827,78 @@ mod tests {
);
assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Failed);
}
#[test]
fn subject_deletion_snapshot_overrides_policy_without_changing_normal_removal() {
let store = setup();
let ordinary = store.plan_worker_removal(&req(), &inv()).unwrap();
store
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
.unwrap();
let body = store.plan_worker_removal(&req(), &inv()).unwrap();
assert_eq!(body.session_disposition, SessionDisposition::Archive);
assert_eq!(body.archive_retention, ArchiveRetention::Forever);
assert_eq!(body.metadata_disposition, MetadataDisposition::Tombstone);
assert_eq!(body.diagnostics_disposition, DiagnosticsDisposition::Retain);
assert!(body.diagnostics_retention_seconds.is_some());
assert_ne!(body.policy_id, ordinary.policy_id);
// A saved plan that would discard diagnostics cannot silently stand in
// for the Subject conversation archive plan.
assert!(matches!(
store.begin_worker_removal("w", &ordinary.plan_id, &ordinary.input_fingerprint),
Err(WorkerRetentionError::StalePlan { .. })
));
let ordinary_store = setup();
assert_eq!(
ordinary_store
.plan_worker_removal(&req(), &inv())
.unwrap()
.policy_id,
ordinary.policy_id
);
}
#[test]
fn subject_deletion_job_purge_rejects_recovery_of_old_executing_archive() {
let store = setup();
let old = store.plan_worker_removal(&req(), &inv()).unwrap();
store
.prepare_worker_removal_execution("w", &old.plan_id, &old.input_fingerprint)
.unwrap();
store
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
.unwrap();
store
.with_conn(|c| {
c.execute(
"UPDATE subject_deletion_workers SET is_job=1 WHERE workspace_id='w'",
[],
)?;
Ok(())
})
.unwrap();
assert!(matches!(
store.recover_worker_removal_execution("w", &req().worker),
Err(WorkerRetentionError::StalePlan { .. })
));
assert!(matches!(
store.plan_worker_removal(&req(), &inv()),
Err(WorkerRetentionError::StalePlan { .. })
));
let fresh = store
.with_conn(|c| load_removal_policy(c, "w", &req().worker, false))
.unwrap()
.unwrap();
assert_eq!(fresh.session_disposition, SessionDisposition::Purge);
assert_eq!(fresh.diagnostics_disposition, DiagnosticsDisposition::Purge);
assert_eq!(fresh.metadata_disposition, MetadataDisposition::Tombstone);
assert_eq!(
store
.with_conn(|c| load_plan(c, &old.plan_id))
.unwrap()
.unwrap()
.state,
WorkerRemovalPlanState::Executing
);
}
}
@@ -91,10 +91,16 @@ mod schema_convergence {
assert_eq!(plan.current_schema_version, 103);
assert_eq!(
plan.migrations,
vec![WorkspaceSchemaMigrationStep {
version: 104,
name: "Converge Passkey results and Job input retention".into(),
}]
vec![
WorkspaceSchemaMigrationStep {
version: 104,
name: "Converge Passkey results and Job input retention".into(),
},
WorkspaceSchemaMigrationStep {
version: 105,
name: "Durable Subject deletion snapshots and Job admission fences".into(),
},
]
);
assert_eq!(
schema_rows(&conn),
@@ -109,6 +115,7 @@ mod schema_convergence {
assert_eq!(feature_rows(conn, job_variant), rows);
let mut expected = history.clone();
expected.push((104, "Converge Passkey results and Job input retention".into()));
expected.push((105, "Durable Subject deletion snapshots and Job admission fences".into()));
assert_eq!(workspace_schema_migration_history(conn)?, expected);
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
@@ -138,7 +145,7 @@ mod schema_convergence {
assert_eq!(feature_rows(conn, job_variant), rows);
assert_eq!(
workspace_schema_migration_history(conn)?.len(),
history.len() + 1
history.len() + 2
);
Ok(())
})
@@ -176,7 +183,7 @@ mod schema_convergence {
upgraded
.with_conn(|conn| {
assert_eq!(feature_rows(conn, job_variant), rows);
assert_eq!(current_schema_version(conn)?, 104);
assert_eq!(current_schema_version(conn)?, 105);
Ok(())
})
.unwrap();
+132 -48
View File
@@ -28,6 +28,7 @@ use subjektiv::api::*;
mod backend_job_profile;
mod external_operation_audit;
mod subject_authorization;
mod subjektiv_deletion;
mod subjektiv_job;
mod subjektiv_journal;
mod subjektiv_journal_browser;
@@ -4577,6 +4578,12 @@ impl WorkspaceApi {
.run(move || recover_workdir_removals(&workdir_recovery_api))
.await?;
recover_runtime_removals(&api).await;
let deletion_api = api.clone();
let deletion_runtime = tokio::runtime::Handle::current();
api.host_executors
.host()?
.run(move || deletion_runtime.block_on(subjektiv_deletion::recover(&deletion_api)))
.await?;
api.host_executors
.host()?
.run(move || {
@@ -4604,9 +4611,23 @@ impl WorkspaceApi {
) -> Result<BackendJobReservation> {
self.resolve_backend_job_profile(request)?;
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
let reservation =
let reservation = {
let gate = subjektiv_deletion::job_subject(request)
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
.transpose()?;
let _guard = gate
.as_ref()
.map(|gate| {
gate.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
})
.transpose()?;
if let Some(subject) = subjektiv_deletion::job_subject(request) {
subjektiv_deletion::require_available(self, subject)?;
}
self.store
.reserve_backend_job(&self.config.workspace_id, request, &now)?;
.reserve_backend_job(&self.config.workspace_id, request, &now)?
};
let result = self.dispatch_backend_job_reservation(reservation);
if result.is_err() {
self.drain_reserved_backend_jobs();
@@ -4621,9 +4642,27 @@ impl WorkspaceApi {
/// immutable request.
pub fn retry_backend_job(&self, job_id: &str) -> Result<BackendJobReservation> {
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
let reservation =
let reservation = {
let current = self
.store
.get_backend_job(self.workspace_id(), job_id)?
.ok_or_else(|| Error::InvalidInput("unknown Backend Job".into()))?;
let gate = subjektiv_deletion::job_subject(&current.request)
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
.transpose()?;
let _guard = gate
.as_ref()
.map(|gate| {
gate.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
})
.transpose()?;
if let Some(subject) = subjektiv_deletion::job_subject(&current.request) {
subjektiv_deletion::require_available(self, subject)?;
}
self.store
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?;
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?
};
let result = self.dispatch_backend_job_reservation(reservation);
if result.is_err() {
self.drain_reserved_backend_jobs();
@@ -4637,13 +4676,8 @@ impl WorkspaceApi {
) -> Result<BackendJobReservation> {
// Reread even a terminal/stale reservation before deciding whether it can
// dispatch or replay: an explicit erasure may have retired its saved input.
let journal_gate = reservation
.job
.request
.grants
.subjektiv_journal
.as_ref()
.map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id))
let journal_gate = subjektiv_deletion::job_subject(&reservation.job.request)
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
.transpose()?;
let _journal_guard = journal_gate
.as_ref()
@@ -4652,6 +4686,9 @@ impl WorkspaceApi {
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
})
.transpose()?;
if let Some(subject) = subjektiv_deletion::job_subject(&reservation.job.request) {
subjektiv_deletion::require_available(self, subject)?;
}
let current_job = self
.store
.get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)?
@@ -5571,6 +5608,27 @@ impl WorkspaceApi {
source_worker: &RuntimeWorkerRef,
submission: &BackendJobResultSubmission,
) -> Result<BackendJobResultAcceptance> {
let result_job = self
.store
.get_backend_job(self.workspace_id(), &submission.job_id)?;
let subject_gate = result_job
.as_ref()
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
.transpose()?;
let subject_guard = subject_gate
.as_ref()
.map(|gate| {
gate.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
})
.transpose()?;
if let Some(subject) = result_job
.as_ref()
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
{
subjektiv_deletion::require_available(self, subject)?;
}
let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
let mut verified_submission = submission.clone();
if let Some(job) = self
@@ -5696,6 +5754,7 @@ impl WorkspaceApi {
&verified_submission,
&accepted_at,
)?;
drop(subject_guard);
if let Some(job) = self
.store
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
@@ -5899,15 +5958,21 @@ impl WorkspaceApi {
.map(|assignment| assignment.operation_id.clone())
})
.unwrap_or_else(|| format!("manual:{}", WorkerId::now_v7()));
let subject = singleton_key
let body_subject = singleton_key
.as_deref()
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
if let Some(subject) = subject
if let Some(subject) = body_subject
&& let Err(error) = subjektiv_journal::recover_failed_subject_create(self, subject)
{
let diagnostics = cleanup_spawn_created_workdirs(self, &spawned_workdir_ids);
return Err(ApiError::with_diagnostics(error, diagnostics));
}
let subject = body_subject.or_else(|| {
request
.resolved_execution_grant
.as_ref()
.map(|grant| grant.subject_id.as_str())
});
let subject_gate = subject
.map(|s| subjektiv_journal::lifecycle_lock(self, s))
.transpose()?;
@@ -5919,6 +5984,9 @@ impl WorkspaceApi {
})
.transpose()?;
if let Some(subject) = subject {
subjektiv_deletion::require_available(self, subject)?;
}
if let Some(subject) = body_subject {
subjektiv_journal::check_start_allowed(self, subject)?;
}
let reservation = match self.config_store.reserve_worker_create(
@@ -5943,11 +6011,11 @@ impl WorkspaceApi {
return Err(ApiError::with_diagnostics(error, diagnostics));
}
};
drop(subject_guard);
let worker_id = reservation.worker_id;
if let Some(grant) = request.resolved_execution_grant.as_ref() {
subject_authorization::bind_reserved_grant(self, grant, &worker_id.to_string())?;
}
drop(subject_guard);
// A completed exact-source allocation is historical acceptance, not a new
// launch. Source/grant authorization and the full fingerprint were checked
// above. Do not reserve its already-linked attachments again, redispatch
@@ -7458,7 +7526,7 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou
.merge(server_api::server_api_axum::subjektiv_memory_changes(
service.clone(),
))
.merge(server_api::server_api_axum::subjektiv_subject_retire(
.merge(server_api::server_api_axum::subjektiv_subject_delete(
service.clone(),
))
.merge(server_api::server_api_axum::subjektiv_stage_candidate(
@@ -9724,22 +9792,25 @@ impl server_api::ServerApi for ServerApiContractService {
.map_err(ApiError::into_repository_api_error)
}
async fn subjektiv_subject_retire(
async fn subjektiv_subject_delete(
&self,
context: server_api::ServerRequestContext,
workspace_id: String,
subject_id: String,
) -> std::result::Result<server_api::SubjektivSubjectResponse, server_api::RepositoryApiError>
{
scoped_retire_subjektiv_subject(
State(self.workspace_api()?.clone()),
AxumPath(ScopedSubjektivSubjectPath {
workspace_id,
subject_id,
}),
)
.await
.map(|Json(response)| response)
.map_err(ApiError::into_repository_api_error)
request: server_api::SubjektivSubjectDeleteRequest,
) -> std::result::Result<
server_api::SubjektivSubjectDeleteResponse,
server_api::RepositoryApiError,
> {
let api = self.workspace_api()?.clone();
let actor = subject_authorization::administration_actor(&context)
.map_err(ApiError::into_repository_api_error)?;
require_workspace_owner(&api, &workspace_id, actor, "Subject deletion")
.await
.map_err(ApiError::into_repository_api_error)?;
subjektiv_deletion::delete(api, workspace_id, subject_id, request)
.await
.map_err(ApiError::into_repository_api_error)
}
async fn subjektiv_stage_candidate(
@@ -22674,7 +22745,9 @@ fn subjektiv_subject_response(
memory_fingerprint,
state: match subject.state {
crate::subjektiv::SubjectState::Active => server_api::SubjektivSubjectState::Active,
crate::subjektiv::SubjectState::Retired => server_api::SubjektivSubjectState::Retired,
crate::subjektiv::SubjectState::LegacyRetired => {
server_api::SubjektivSubjectState::LegacyRetired
}
},
created_at: subject.created_at,
updated_at: subject.updated_at,
@@ -22962,24 +23035,6 @@ async fn scoped_list_subjektiv_memory_changes(
.await
}
async fn scoped_retire_subjektiv_subject(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedSubjektivSubjectPath>,
) -> ApiResult<Json<server_api::SubjektivSubjectResponse>> {
api.host_executors
.host()?
.run(move || {
validate_workspace_scope(&api, &path.workspace_id)?;
let subject = open_subjektiv_store(&api)?
.retire_subject(&path.subject_id)
.map_err(subjektiv_store_error)?;
// Retirement deliberately does not stop or remove the current Worker. The
// Worker lease and subject lifecycle are independent authorities.
Ok(Json(subjektiv_subject_response(&api, subject)?))
})
.await
}
async fn scoped_stage_subjektiv_candidate(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedWorkspacePath>,
@@ -24564,6 +24619,7 @@ fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error {
Error::WorkspacePermissionDenied(error.to_string())
}
crate::subjektiv::SubjektivError::SubjectPermissionsConflict(_)
| crate::subjektiv::SubjektivError::SubjectDeletionConflict(_)
| crate::subjektiv::SubjektivError::PermissionOperationConflict(_)
| crate::subjektiv::SubjektivError::WorkerExecutionGrantConflict(_) => {
Error::RepositoryConflict(error.to_string())
@@ -31543,6 +31599,7 @@ async fn workspace_response(
manage_secrets: is_owner,
manage_runtimes: is_owner,
delete_workspace: is_owner,
delete_subjects: is_owner,
},
extension_points: WorkspaceExtensionPoints {
store: "sqlite".to_string(),
@@ -37777,6 +37834,7 @@ mod tests {
mod restore_workdir_tests;
mod restored_parent_control_tests;
mod subject_authorization_tests;
mod subject_deletion_tests;
mod subject_spawn_tests;
mod subjektiv_jobs_tests;
mod subjektiv_journal_tests;
@@ -46241,7 +46299,13 @@ mod tests {
let retired = store
.create_subject(crate::subjektiv::SubjectRole::new("retired").unwrap())
.unwrap();
store.retire_subject(&retired.id).unwrap();
// Frozen lifecycle fixture: migration preserves old inactive Subjects; there is
// intentionally no public operation that creates this historical state anymore.
api.feature_storage.open(&api.subjektiv_registration).unwrap()
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
tx.execute("UPDATE subjects SET state='retired', record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&retired.id])?;
Ok(())
}).unwrap();
let request = |subject_id: Option<&str>, profile: &str, singleton_key: Option<&str>| {
CreateWorkspaceWorkerRequest {
@@ -48843,6 +48907,7 @@ mod tests {
>,
materializer: worker_runtime::working_directory::RuntimeGitMaterializer,
spawn_failure: std::sync::Mutex<Option<String>>,
subject_delete_failure: std::sync::Mutex<Option<String>>,
recovery_failure: std::sync::Mutex<Option<String>>,
recovery_requests: std::sync::Mutex<Vec<runtime_api::SubjectCreateRecoveryRequest>>,
recovery_hook: std::sync::Mutex<Option<Box<dyn FnOnce() + Send>>>,
@@ -48887,6 +48952,7 @@ mod tests {
std::env::temp_dir().join(unique),
),
spawn_failure: std::sync::Mutex::new(None),
subject_delete_failure: std::sync::Mutex::new(None),
recovery_failure: std::sync::Mutex::new(None),
recovery_requests: std::sync::Mutex::new(Vec::new()),
recovery_hook: std::sync::Mutex::new(None),
@@ -49242,6 +49308,24 @@ mod tests {
.map(|execution| execution.snapshot())
}
fn preflight_subject_delete_worker(
&self,
_request: &runtime_api::SubjectDeleteWorkerRequest,
) -> std::result::Result<(), String> {
Ok(())
}
fn subject_delete_worker(
&self,
request: &runtime_api::SubjectDeleteWorkerRequest,
) -> std::result::Result<(), String> {
if let Some(message) = self.subject_delete_failure.lock().unwrap().clone() {
return Err(message);
}
self.stop_worker(&worker_runtime::identity::WorkerRef::new(request.worker_id));
Ok(())
}
fn stop_worker(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
@@ -149,6 +149,13 @@ pub(super) fn worker_permissions(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
) -> ApiResult<Option<Vec<ScopedPermission>>> {
if subjektiv_deletion::deleting_worker(api, worker)?
|| open_subjektiv_store(api)?
.worker_subject_deleted(&worker.runtime_id, &worker.worker_id)
.map_err(subjektiv_store_error)?
{
return Err(denied("Worker belonged to an explicitly deleted Subject"));
}
if let Some(grant) = live_child_grant(api, worker)? {
return Ok(Some(grant.permissions));
}
@@ -226,6 +233,8 @@ pub(super) fn prepare_delegation(
singleton_key: Option<&str>,
subject_connection: bool,
) -> ApiResult<Option<WorkerExecutionGrant>> {
// A deleted grant must never become an ordinary non-Subject delegation source.
worker_permissions(api, source)?;
let child = live_child_grant(api, source)?;
let body = if child.is_none() {
current_body_subject(api, source)?
@@ -0,0 +1,288 @@
//! Explicit Subject destruction is independent of ordinary Worker retention. The domain
//! fence commits before external effects, remains after failures, and rejects cached grants.
//! Worker originals are archived; Job execution copies are purged. Existing archives, Workdirs,
//! Tickets and other Subjects are not erased. Archive contents can include previously injected
//! Memory: their independent retention/access policy still applies. This is logical content
//! deletion, not a promise to erase backups, external providers or already exported material.
//!
//! Child removal/create-fence receipts, not acceptance, Idle or missing projections, prove
//! quiescence. A retry carries the same scope/Subject/operation; no new child identity is issued.
//! The domain receipt is authoritative completion. Backend inventories retain only identifiers
//! needed for cleanup and fail-closed authorization, never a copy of Subject content.
use super::*;
use rusqlite::params;
pub(super) fn job_subject(request: &BackendJobRequest) -> Option<&str> {
request
.grants
.subjektiv_journal
.as_ref()
.map(|g| g.subject_id.as_str())
.or_else(|| {
request
.grants
.subjektiv_consolidation
.as_ref()
.map(|g| g.subject_id.as_str())
})
}
pub(super) fn require_available(api: &WorkspaceApi, subject: &str) -> Result<()> {
let store = open_subjektiv_store(api).map_err(|error| error.error)?;
if store
.subject(subject)
.map_err(|e| Error::Store(e.to_string()))?
.is_none()
{
return Err(Error::SubjektivSubjectNotFound(subject.into()));
}
Ok(())
}
pub(super) fn pending(api: &WorkspaceApi, subject: &str) -> Result<bool> {
api.config_store.with_conn(|connection| {
Ok(connection.query_row(
"SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND subject_id=?2)",
params![api.workspace_id(), subject], |row| row.get(0),
)?)
})
}
/// This is internal cleanup evidence, never a user-supplied removal flag. It exists only
/// after the independently durable domain fence and exact-target inventory commit.
pub(super) fn deleting_worker(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Result<bool> {
api.config_store.with_conn(|connection| {
Ok(connection.query_row(
"SELECT EXISTS(SELECT 1 FROM subject_deletion_workers WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3)",
params![api.workspace_id(), worker.runtime_id, worker.worker_id], |row| row.get(0),
)?)
})
}
pub(super) async fn delete(
api: WorkspaceApi,
workspace: String,
subject: String,
request: server_api::SubjektivSubjectDeleteRequest,
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
let handle = tokio::runtime::Handle::current();
api.host_executors
.host()?
.run(move || {
validate_workspace_scope(&api, &workspace)?;
handle.block_on(resume(&api, &subject, &request.operation_id))
})
.await
}
pub(super) async fn recover(api: &WorkspaceApi) -> Result<()> {
let pending = open_subjektiv_store(api)
.map_err(|error| error.error)?
.pending_subject_deletions()
.map_err(|e| Error::Store(e.to_string()))?;
for deletion in pending {
if let Err(error) = resume(api, &deletion.subject_id, &deletion.operation_id).await {
tracing::warn!(%error.error, subject_id=%deletion.subject_id,
"Subject deletion restart recovery remains incomplete; same request can resume");
}
}
Ok(())
}
async fn resume(
api: &WorkspaceApi,
subject: &str,
operation: &str,
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
let store = open_subjektiv_store(api)?;
let gate = subjektiv_journal::lifecycle_lock(api, subject)?;
let receipt = {
let _guard = gate
.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
let receipt = store
.begin_subject_deletion(subject, operation)
.map_err(subjektiv_store_error)?;
if !receipt.completed {
let delegated = store
.deletion_subject_workers(subject)
.map_err(subjektiv_store_error)?
.into_iter()
.map(|(runtime, worker)| RuntimeWorkerRef::new(runtime, worker))
.collect::<Vec<_>>();
api.config_store.prepare_subject_deletion(
api.workspace_id(),
subject,
operation,
&delegated,
)?;
}
receipt
};
let response = |completed| server_api::SubjektivSubjectDeleteResponse {
subject_id: subject.into(),
operation_id: operation.into(),
completed,
};
if receipt.completed {
return Ok(response(true));
}
// Never hold the Host callback gate across a Runtime call. A create/restore can be
// waiting for the callback that now has to reject this permanently fenced Subject.
let workers =
api.config_store
.list_subject_deletion_workers(api.workspace_id(), subject, operation)?;
for (worker, _is_job) in workers {
if let Err(error) = remove_worker(api, subject, operation, &worker).await {
tracing::warn!(%error.error, subject_id=subject, operation_id=operation, "Subject deletion remains incomplete; exact retry required");
return Ok(response(false));
}
}
{
let _guard = gate
.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
api.config_store
.erase_subject_deleted_jobs(api.workspace_id(), subject, operation)?;
// Finish Backend cleanup before the authoritative domain receipt. If domain
// purge fails, its separately committed fence and pending operation survive.
// No retry can report domain completion while occupancy/Job cleanup is unproven.
api.config_store
.finish_subject_deletion_record(api.workspace_id(), subject, operation)?;
store
.finish_subject_deletion(subject, operation)
.map_err(subjektiv_store_error)?;
}
Ok(response(true))
}
async fn remove_worker(
api: &WorkspaceApi,
subject: &str,
operation: &str,
worker: &RuntimeWorkerRef,
) -> ApiResult<()> {
let request = api.config_store.subject_deletion_worker_request(
api.workspace_id(),
subject,
operation,
worker,
)?;
let fence = api
.runtime
.subject_delete_worker(&worker.runtime_id, request.clone())
.map_err(RuntimeRegistryError::into_error)?;
if fence.workspace_id != request.workspace_id
|| fence.worker_id != request.worker_id
|| fence.create_fingerprint != request.create_fingerprint
|| fence.operation_id != request.operation_id
{
return Err(Error::RepositoryConflict(
"Subject Worker deletion fence receipt mismatch".into(),
)
.into());
}
// Replay the canonical child even when physical removal already happened:
// its retained receipt can finish the Backend catalog midpoint. The Runtime
// fence above also prevents an older create from arriving after that removal.
if api
.config_store
.recover_worker_removal_execution(api.workspace_id(), worker)
.map_err(|error| Error::Store(error.to_string()))?
.is_some()
{
return execute_removal(api, operation, worker).await;
}
match api.runtime.worker(worker) {
Ok(detail) => {
if !detail.state.eq_ignore_ascii_case("stopped") {
let stopped = api
.runtime
.stop_worker(
worker,
WorkerLifecycleRequest {
reason: Some(format!("Explicit Subject deletion {operation}")),
ticket_assignment: None,
},
)
.map_err(RuntimeRegistryError::into_error)?;
if stopped.state != InternalWorkerOperationState::Accepted {
return Err(Error::RepositoryConflict(
"Subject Worker stop was not accepted".into(),
)
.into());
}
}
let actual = api
.runtime
.worker(worker)
.map_err(RuntimeRegistryError::into_error)?;
if !actual.state.eq_ignore_ascii_case("stopped") {
return Err(
Error::RepositoryConflict("Subject Worker stop is unconfirmed".into()).into(),
);
}
if api
.store
.get_worker_registry(api.workspace_id(), worker)?
.is_none()
{
// The Runtime fence authenticated this exact reserved fingerprint. A create
// can have reached Runtime before its Backend publication failed. Publish
// only its stopped record so the normal archive/Purge owner can remove it.
record_worker_summary(
api,
&actual,
&actual.display_name,
actual.profile.clone(),
WorkerRegistryDisplayNamePolicy::PreserveExisting,
)?;
}
}
Err(RuntimeRegistryError::UnknownWorker { .. }) => {
if api
.store
.get_worker_registry(api.workspace_id(), worker)?
.is_some()
{
return Err(Error::RepositoryConflict(
"Runtime object missing before canonical original archival".into(),
)
.into());
}
api.config_store.confirm_subject_deletion_absence(
api.workspace_id(),
subject,
operation,
worker,
&fence,
)?;
return Ok(());
}
Err(error) => return Err(error.into_error().into()),
}
execute_removal(api, operation, worker).await
}
async fn execute_removal(
api: &WorkspaceApi,
operation: &str,
worker: &RuntimeWorkerRef,
) -> ApiResult<()> {
let response = WorkerRemovalService::new(api)
.execute_target_removal_on_host(
api.runtime.as_ref(),
worker,
&format!("Explicit Subject deletion {operation}"),
None,
)
.await
.map_err(Error::Store)?;
if response.status != StatusCode::OK.as_u16() {
return Err(Error::RepositoryConflict(
"Subject Worker archival/Purge remains unconfirmed".into(),
)
.into());
}
Ok(())
}
@@ -446,6 +446,7 @@ pub(super) fn bind_host_session(
/// Runs under the same Subject gate as final revoke/singleton release. A failed create or
/// legacy removal may have released its Backend reservation but cannot hide a durable writer.
pub(super) fn check_start_allowed(api: &WorkspaceApi, subject: &str) -> Result<()> {
subjektiv_deletion::require_available(api, subject)?;
let store = store(api)?;
let key = crate::subjektiv::subject_worker_singleton_key(subject)
.map_err(|e| Error::InvalidInput(e.to_string()))?;
@@ -1747,6 +1748,13 @@ pub(super) fn prepare_worker_removal(
runtime: &RuntimeRegistry,
target: &RuntimeWorkerRef,
) -> Result<Option<crate::retention::ErasedJournalSession>> {
if let Some(api) = service.journal_dispatch_api.upgrade()
&& subjektiv_deletion::deleting_worker(&api, target)?
{
// Full Subject deletion has a permanent domain fence. It deliberately drops
// Journal continuity, but retains source originals through its archive policy.
return Ok(None);
}
let Some(attr) = removal_attribution(service, target)? else {
if service
.store
@@ -1809,6 +1817,13 @@ pub(super) fn erased_archive_removal_proof(
target: &RuntimeWorkerRef,
prepared: &crate::retention::PreparedWorkerRemoval,
) -> Result<Option<crate::retention::RuntimeJournalErasureAck>> {
if let Some(api) = service.journal_dispatch_api.upgrade()
&& subjektiv_deletion::deleting_worker(&api, target)?
{
// Resume the exact canonical retention request after full Subject fencing.
// Subject deletion retains originals; it is not per-Session source erasure.
return Ok(None);
}
let Some(attr) = removal_attribution(service, target)? else {
return Ok(None);
};
@@ -1852,6 +1867,13 @@ pub(super) fn validate_removal_retention(
target: &RuntimeWorkerRef,
prepared: &crate::retention::PreparedWorkerRemoval,
) -> Result<()> {
if let Some(api) = service.journal_dispatch_api.upgrade()
&& subjektiv_deletion::deleting_worker(&api, target)?
{
// The plan-local retention policy and canonical Runtime receipt remain required.
// Neither unfinished Journals nor a stopped old Controller are deletion blockers.
return Ok(());
}
let Some(attr) = removal_attribution(service, target)? else {
if service
.store
@@ -2066,6 +2088,9 @@ pub(super) fn session_erased(api: &WorkspaceApi, request: &BackendJobRequest) ->
let Some(grant) = &request.grants.subjektiv_journal else {
return Ok(false);
};
if subjektiv_deletion::pending(api, &grant.subject_id)? {
return Ok(true);
}
let store = store(api)?;
store
.journal_session_erased(&host(api, &store, grant)?)
@@ -0,0 +1,729 @@
//! Real router/authentication, feature store and Backend DB boundaries. Runtime
//! execution/unknown and restart fences are covered in worker-runtime and the
//! canonical removal suites, rather than claiming HTTP acceptance proves shutdown.
use super::*;
async fn delete_request(
api: &WorkspaceApi,
subject: &str,
key: &str,
token: Option<&str>,
workspace: &str,
) -> (StatusCode, Value) {
let mut builder = Request::builder()
.method("DELETE")
.uri(format!("/api/w/{workspace}/subjektiv/subjects/{subject}"))
.header(CONTENT_TYPE, "application/json");
if let Some(token) = token {
builder = builder.header("authorization", format!("Bearer {token}"));
}
let response = build_router(api.clone())
.oneshot(
builder
.body(Body::from(json!({"operation_id":key}).to_string()))
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let bytes = to_bytes(response.into_body(), 2 * 1024 * 1024)
.await
.unwrap();
let body = serde_json::from_slice(&bytes)
.unwrap_or_else(|_| json!({"text":String::from_utf8_lossy(&bytes)}));
(status, body)
}
#[tokio::test]
async fn deletion_owner_scope_exact_retry_and_subject_isolation() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("delete me").unwrap(),
"private behavior".into(),
)
.unwrap();
let other = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("keep me").unwrap(),
"unchanged".into(),
)
.unwrap();
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-owner").await;
let stranger = seed_test_api_token(api.store.as_ref(), "subject-delete-stranger");
for (token, workspace, expected) in [
(None, TEST_WORKSPACE_ID, StatusCode::UNAUTHORIZED),
(
Some(stranger.as_str()),
TEST_WORKSPACE_ID,
StatusCode::FORBIDDEN,
),
(
Some(owner.as_str()),
"another-workspace",
StatusCode::NOT_FOUND,
),
] {
let (status, body) =
delete_request(&api, &subject.id, "same-operation", token, workspace).await;
assert_eq!(status, expected, "{body}");
assert!(store.subject(&subject.id).unwrap().is_some());
}
let (status, body) = delete_request(
&api,
&subject.id,
"same-operation",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
assert!(store.subject(&subject.id).unwrap().is_none());
assert_eq!(
store.subject(&other.id).unwrap().unwrap().behavior_md,
"unchanged"
);
assert_eq!(store.list_subjects(100).unwrap().items.len(), 1);
let (status, replay) = delete_request(
&api,
&subject.id,
"same-operation",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{replay}");
assert_eq!(replay, body);
let (status, _) = delete_request(
&api,
&subject.id,
"different-operation",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::CONFLICT);
let (status, _) = delete_request(
&api,
&other.id,
"same-operation",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::CONFLICT);
assert!(store.subject(&other.id).unwrap().is_some());
let fresh = store
.create_subject(crate::subjektiv::SubjectRole::new("fresh").unwrap())
.unwrap();
assert_ne!(fresh.id, subject.id);
assert!(store.subject_permissions(&fresh.id).unwrap().is_none());
assert_eq!(
store
.pending_staging_candidates(&fresh.id, 100)
.unwrap()
.len(),
0
);
}
#[tokio::test]
async fn legacy_inactive_subject_is_preserved_until_explicit_delete_and_retire_route_is_gone() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("legacy fixture").unwrap(),
"retained original".into(),
)
.unwrap();
api.feature_storage.open(&api.subjektiv_registration).unwrap()
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
tx.execute("UPDATE subjects SET state='retired',record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&subject.id])?;
Ok(())
}).unwrap();
assert_eq!(
store.subject(&subject.id).unwrap().unwrap().state,
crate::subjektiv::SubjectState::LegacyRetired
);
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-legacy").await;
let request = Request::builder()
.method("POST")
.uri(format!(
"/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/retire",
subject.id
))
.header("authorization", format!("Bearer {owner}"))
.body(Body::empty())
.unwrap();
let old = build_router(api.clone()).oneshot(request).await.unwrap();
assert_eq!(old.status(), StatusCode::METHOD_NOT_ALLOWED);
assert_eq!(
store.subject(&subject.id).unwrap().unwrap().behavior_md,
"retained original"
);
let (status, body) = delete_request(
&api,
&subject.id,
"explicit-legacy-delete",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn unconfirmed_worker_shutdown_is_not_completion_and_exact_retry_converges() {
let dir = tempfile::tempdir().unwrap();
init_clean_git_workspace(dir.path());
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("delete connected body").unwrap(),
"private behavior".into(),
)
.unwrap();
let Json(created) = create_workspace_worker(
State(api.clone()),
HeaderMap::new(),
Json(CreateWorkspaceWorkerRequest {
runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(),
display_name: "Subject deletion boundary".into(),
singleton_key: None,
profile: Some("builtin:companion".into()),
ticket_assignment: None,
initial_submit: vec![],
workdir_attachments: vec![],
feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest {
subjektiv: Some(server_api::SubjektivWorkerConnectionRequest {
subject_id: subject.id.clone(),
}),
},
control_operation_id: None,
execution_grant: None,
}),
)
.await
.unwrap();
let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id);
seed_worker_session_for_cleanup(dir.path(), &worker);
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unknown").await;
*execution.subject_delete_failure.lock().unwrap() = Some("shutdown completion unknown".into());
let (status, body) = delete_request(
&api,
&subject.id,
"unknown-delete",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], false);
assert!(store.subject(&subject.id).unwrap().is_none());
assert!(
!store
.subject_deletion(&subject.id)
.unwrap()
.unwrap()
.completed
);
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
assert!(
api.store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
.unwrap()
.is_some()
);
*execution.subject_delete_failure.lock().unwrap() = None;
api.config_store.with_conn(|conn| {
conn.execute_batch("CREATE TRIGGER injected_worker_catalog_failure BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected catalog failure'); END;")?;
Ok(())
}).unwrap();
let (status, midpoint) = delete_request(
&api,
&subject.id,
"unknown-delete",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{midpoint}");
assert_eq!(midpoint["completed"], false);
assert!(matches!(
api.runtime.worker(&worker),
Err(RuntimeRegistryError::UnknownWorker { .. })
));
assert!(
api.store
.get_worker_registry(TEST_WORKSPACE_ID, &worker)
.unwrap()
.is_some()
);
api.config_store
.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER injected_worker_catalog_failure;")?;
Ok(())
})
.unwrap();
let (status, body) = delete_request(
&api,
&subject.id,
"unknown-delete",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true, "{body}");
assert!(
api.store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
.unwrap()
.is_none()
);
let removal = api
.config_store
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
.unwrap()
.unwrap();
assert_eq!(
removal.plan.state,
crate::retention::WorkerRemovalPlanState::Succeeded
);
assert_eq!(
removal.runtime_request.session_disposition,
worker_runtime::retention::SessionDisposition::Archive
);
assert!(
store
.subject_deletion(&subject.id)
.unwrap()
.unwrap()
.completed
);
}
#[tokio::test]
async fn captured_subject_job_result_cannot_deliver_after_domain_fence() {
let dir = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject(crate::subjektiv::SubjectRole::new("notification fence").unwrap())
.unwrap();
let target = spawn_ticket_check_source(&api, "subject-notification-target");
let worker = spawn_ticket_check_source(&api, "subject-notification-result");
let request = BackendJobRequest {
job_id: "captured-subject-notification".into(),
purpose: "subject_notification_contract".into(),
input_ref: "subject:notification".into(),
input: json!({"private":"input"}),
instruction: "Read input".into(),
profile: "builtin:backend-job".into(),
grants: crate::backend_job::BackendJobGrants {
subjektiv_consolidation: Some(crate::backend_job::SubjektivConsolidationGrant {
subject_id: subject.id.clone(),
candidate_ids: vec!["candidate".into()],
}),
..Default::default()
},
serialization_key: None,
source_worker: Some(target.clone()),
notification_target: Some(target),
limits: Default::default(),
};
// Enter at the committed result boundary: execution/result validation is
// outside this notification test, but routing and Runtime transport are real.
let now = Utc::now().to_rfc3339();
let reserved = api
.store
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &now)
.unwrap();
let attempt = &reserved.attempt.attempt_id;
api.store
.claim_backend_job_attempt_dispatch(TEST_WORKSPACE_ID, &request.job_id, attempt, &now)
.unwrap();
api.store
.bind_backend_job_attempt_worker(
TEST_WORKSPACE_ID,
&request.job_id,
attempt,
&worker,
Some("run"),
&now,
)
.unwrap();
let captured = api
.store
.accept_backend_job_result(
TEST_WORKSPACE_ID,
&worker,
&BackendJobResultSubmission {
job_id: request.job_id.clone(),
attempt_id: attempt.clone(),
input_digest: request.input_digest().unwrap(),
result: json!({"private":"result"}),
},
&now,
)
.unwrap();
let delivery = crate::backend_job::delivery_id(&request.job_id, attempt);
assert!(
api.store
.job_notification_source(TEST_WORKSPACE_ID, &request.job_id, attempt)
.is_ok()
);
execution.take_inputs();
store
.begin_subject_deletion(&subject.id, "notification-delete")
.unwrap();
notification_delivery::deliver_job_result(&api, &captured);
notification_delivery::recover_job_notifications(&api);
assert!(
execution.take_inputs().is_empty(),
"captured or recovered result must not escape the domain fence"
);
assert!(
api.store
.job_notification_request(TEST_WORKSPACE_ID, &delivery)
.unwrap()
.is_none()
);
}
#[tokio::test]
async fn domain_purge_failure_keeps_fence_and_backend_reopen_resumes_exact_operation() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("retry purge").unwrap(),
"do not resurrect".into(),
)
.unwrap();
let other = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("untouched").unwrap(),
"keep".into(),
)
.unwrap();
let database = api
.feature_storage
.open(&api.subjektiv_registration)
.unwrap();
database.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
tx.execute_batch("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected domain purge failure'); END;")?;
Ok(())
}).unwrap();
let owner =
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-purge-failure").await;
let (status, body) = delete_request(
&api,
&subject.id,
"purge-retry",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert!(status.is_server_error(), "{status}: {body}");
assert!(
!store
.subject_deletion(&subject.id)
.unwrap()
.unwrap()
.completed
);
assert!(store.subject(&subject.id).unwrap().is_none());
assert_eq!(
store.subject(&other.id).unwrap().unwrap().behavior_md,
"keep"
);
database
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
tx.execute_batch("DROP TRIGGER injected_subject_purge_failure;")?;
Ok(())
})
.unwrap();
let mut reopened = api.clone();
let config_store =
Arc::new(SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap());
reopened.config_store = config_store.clone();
reopened.store = config_store;
let recovery_api = reopened.clone();
let handle = tokio::runtime::Handle::current();
reopened
.host_executors
.host()
.unwrap()
.run(move || handle.block_on(subjektiv_deletion::recover(&recovery_api)))
.await
.unwrap();
let recovered = open_subjektiv_store(&reopened)
.unwrap()
.subject_deletion(&subject.id)
.unwrap()
.unwrap();
assert_eq!(recovered.operation_id, "purge-retry");
assert!(recovered.completed);
let (status, body) = delete_request(
&reopened,
&subject.id,
"purge-retry",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
assert_eq!(
store.subject(&other.id).unwrap().unwrap().behavior_md,
"keep"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn pending_consolidation_is_purged_and_captured_dispatch_and_late_result_are_rejected() {
let dir = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject(crate::subjektiv::SubjectRole::new("delete running Job").unwrap())
.unwrap();
let candidate: memory::extract::StagingRecord = serde_json::from_value(json!({
"schema_version": 2, "id": "delete-job-candidate", "extract_run_id": "extract",
"source": {"segment_id":"segment", "range":[1,2]}, "kind":"constraint",
"claim":"private candidate marker", "why_useful":"retained only until explicit erasure",
"evidence": [], "source_refs": []
}))
.unwrap();
store
.stage_candidate(crate::subjektiv::SubjectStagingRecord::attach(
&subject.id,
candidate,
))
.unwrap();
let started = start_subjektiv_staging_consolidation(
api.clone(),
&subject.id,
MemoryConsolidateStagingOperation { force: true },
)
.await
.unwrap();
assert_eq!(started.status, "started");
let job = api
.store
.find_active_backend_job_for_resource(
TEST_WORKSPACE_ID,
&crate::backend_job::subjektiv_serialization_key(&subject.id),
)
.unwrap()
.unwrap();
let attempt = api
.store
.get_backend_job_attempt(
TEST_WORKSPACE_ID,
&job.request.job_id,
&crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt),
)
.unwrap()
.unwrap();
let worker = attempt.worker.clone().unwrap();
seed_worker_session_for_cleanup(dir.path(), &worker);
let captured = BackendJobReservation {
job: job.clone(),
attempt: attempt.clone(),
replayed: true,
resource_reused: false,
};
let owner =
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-consolidation").await;
*execution.subject_delete_failure.lock().unwrap() = Some("completion unknown".into());
let (status, body) = delete_request(
&api,
&subject.id,
"delete-job",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], false);
assert!(
api.dispatch_backend_job_reservation(captured.clone())
.is_err()
);
assert!(
api.accept_backend_job_result(
&worker,
&BackendJobResultSubmission {
job_id: job.request.job_id.clone(),
attempt_id: attempt.attempt_id.clone(),
input_digest: job.input_digest.clone(),
result: json!({"late":true}),
}
)
.is_err()
);
*execution.subject_delete_failure.lock().unwrap() = None;
let (status, body) = delete_request(
&api,
&subject.id,
"delete-job",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
assert!(
api.store
.get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id)
.unwrap()
.is_none()
);
assert!(
api.store
.get_backend_job_attempt(TEST_WORKSPACE_ID, &job.request.job_id, &attempt.attempt_id)
.unwrap()
.is_none()
);
assert!(api.dispatch_backend_job_reservation(captured).is_err());
assert!(api.dispatch_backend_job(&job.request).is_err());
let removal = api
.config_store
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
.unwrap()
.unwrap();
assert_eq!(
removal.plan.state,
crate::retention::WorkerRemovalPlanState::Succeeded
);
assert_eq!(
removal.runtime_request.session_disposition,
worker_runtime::retention::SessionDisposition::Purge
);
assert_eq!(
removal.runtime_request.diagnostics_disposition,
worker_runtime::retention::DiagnosticsDisposition::Purge
);
assert!(
!dir.path()
.join(".test-embedded-runtime-store/workers")
.join(&worker.worker_id)
.exists()
);
}
#[tokio::test]
async fn unmaterialized_body_and_job_reservations_finish_only_with_runtime_storage_absence_proof() {
let dir = tempfile::tempdir().unwrap();
let (api, _) = test_api_with_recording_backend(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let owner =
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unmaterialized").await;
for is_job in [false, true] {
let subject = store
.create_subject(crate::subjektiv::SubjectRole::new("reserved before deletion").unwrap())
.unwrap();
let singleton = (!is_job).then(|| format!("subjektiv:{}", subject.id));
let allocation = if is_job {
let request = BackendJobRequest {
job_id: "unmaterialized-subject-job".into(),
purpose: "subject_reservation_contract".into(),
input_ref: "subject:reserved".into(),
input: json!({"private":"reserved input"}),
instruction: "read".into(),
profile: "builtin:backend-job".into(),
grants: crate::backend_job::BackendJobGrants {
subjektiv_consolidation: Some(
crate::backend_job::SubjektivConsolidationGrant {
subject_id: subject.id.clone(),
candidate_ids: vec!["candidate".into()],
},
),
..Default::default()
},
serialization_key: None,
source_worker: None,
notification_target: None,
limits: Default::default(),
};
let reserved = api
.store
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &Utc::now().to_rfc3339())
.unwrap();
crate::backend_job::allocation_key(&reserved.attempt.attempt_id)
} else {
"unmaterialized-subject-body".into()
};
let settings = api
.config_store
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
.unwrap();
let reserved = api
.config_store
.reserve_worker_create(
TEST_WORKSPACE_ID,
EMBEDDED_WORKER_RUNTIME_ID,
&allocation,
"unmaterialized-fingerprint",
singleton.as_deref(),
&settings,
)
.unwrap();
let worker =
RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, reserved.worker_id.to_string());
assert!(matches!(
api.runtime.worker(&worker),
Err(RuntimeRegistryError::UnknownWorker { .. })
));
let (status, body) = delete_request(
&api,
&subject.id,
&format!("delete-reserved-{is_job}"),
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
let proved = api.config_store.with_conn(|conn| Ok(conn.query_row("SELECT originals_absent FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4", rusqlite::params![TEST_WORKSPACE_ID,subject.id,worker.runtime_id,worker.worker_id], |row|row.get::<_,bool>(0))?)).unwrap();
assert!(proved);
assert!(
api.config_store
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
.unwrap()
.is_none(),
"absence is not a fabricated Purge/archive receipt"
);
assert!(
api.config_store
.reserve_worker_create(
TEST_WORKSPACE_ID,
EMBEDDED_WORKER_RUNTIME_ID,
&allocation,
"unmaterialized-fingerprint",
singleton.as_deref(),
&settings
)
.is_err()
);
if let Some(singleton) = singleton {
assert!(
api.store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton)
.unwrap()
.is_none()
);
}
}
}
@@ -1143,6 +1143,8 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
&Utc::now().to_rfc3339(),
)
.unwrap();
// Discard only the original delivery, before recovery can automatically retry.
f.execution.take_inputs();
f.api.recover_backend_job_worker_cleanups().unwrap();
wait_for_backend_job_worker_cleanup(
&f.api,
@@ -1151,23 +1153,64 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
BackendJobWorkerCleanupState::Completed,
)
.await;
// Cleanup commits before domain reconciliation; wait for its authoritative outcome.
tokio::time::timeout(std::time::Duration::from_secs(10), async {
// Cleanup commits before domain reconciliation. Job Failed is only transient:
// reconcile_unknown fences this attempt, then automatically reserves/queues its
// retry. Observe the immutable attempt outcome and the delivered new generation,
// not the Job's transient state; a manual retry would race the automatic one.
let reconciliation = tokio::time::timeout(std::time::Duration::from_secs(10), async {
loop {
let job = f
.api
.store
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
.unwrap()
.unwrap();
if job.state == BackendJobState::Failed {
break;
let original = f.api.store.get_backend_job_attempt(
TEST_WORKSPACE_ID,
&f.request.job_id,
&dispatched.attempt.attempt_id,
).unwrap().unwrap();
if original.state == BackendJobAttemptState::Failed {
assert_eq!(original.failure_category.as_deref(), Some("journal_attempt_fenced"));
assert_eq!(original.worker_cleanup_state, Some(BackendJobWorkerCleanupState::Completed));
assert!(original.worker_cleanup_completed_at.is_some());
if let Some(generation) = f.store.active_journal_generation(&f.host, &f.id).unwrap()
&& generation.job.attempt_id != original.attempt_id
&& let Some(retry) = f.api.store.get_backend_job_attempt(
TEST_WORKSPACE_ID,
&f.request.job_id,
&generation.job.attempt_id,
).unwrap()
&& retry.state == BackendJobAttemptState::Dispatched
&& let Some(resumed) = &retry.worker
// Worker binding precedes input delivery/attestation. ReadState
// succeeds only once the new generation's delivery is attested.
&& subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok()
{
break (original, retry, generation);
}
}
tokio::task::yield_now().await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.unwrap();
.await;
assert!(
reconciliation.is_ok(),
"post-cleanup reconciliation timed out: job={:?}, original_attempt={:?}, active_generation={:?}",
f.api
.store
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
.unwrap(),
f.api
.store
.get_backend_job_attempt(
TEST_WORKSPACE_ID,
&f.request.job_id,
&dispatched.attempt.attempt_id
)
.unwrap(),
f.store.active_journal_generation(&f.host, &f.id).unwrap(),
);
let (fenced, retry, generation) = reconciliation.unwrap();
assert_eq!(fenced.worker, dispatched.attempt.worker);
assert_eq!(retry.attempt, dispatched.attempt.attempt + 1);
assert_eq!(generation.job.job_id, f.request.job_id);
assert_eq!(generation.job.attempt_id, retry.attempt_id);
assert_eq!(retry.input_digest, f.request.input_digest().unwrap());
f.assert_purged(worker);
assert!(f.store.journal_retention_required(&f.host).unwrap());
assert_eq!(
@@ -1187,25 +1230,25 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
1,
"source pin survives failed/partial Job cleanup"
);
f.execution.take_inputs();
let retry = f
let job = f
.api
.store
.reserve_backend_job_retry(
TEST_WORKSPACE_ID,
&f.request.job_id,
&Utc::now().to_rfc3339(),
)
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
.unwrap()
.unwrap();
assert_eq!(retry.job.request.input, f.request.input);
assert_eq!(retry.job.input_digest, f.request.input_digest().unwrap());
let retry = f.api.dispatch_backend_job_reservation(retry).unwrap();
let resumed = retry.attempt.worker.as_ref().unwrap();
assert_eq!(job.state, BackendJobState::Pending);
assert_eq!(job.current_attempt, retry.attempt);
assert_eq!(
job.load_executable_request().unwrap().input,
f.request.input
);
assert_eq!(job.input_digest, f.request.input_digest().unwrap());
let resumed = retry.worker.as_ref().unwrap();
let delivered = f.execution.take_inputs();
assert_eq!(delivered.len(), 1);
assert_eq!(
delivered[0].1,
f.request.worker_input(&retry.attempt.attempt_id).unwrap()
f.request.worker_input(&retry.attempt_id).unwrap()
);
assert_eq!(
f.store
+55 -8
View File
@@ -92,6 +92,13 @@ use crate::backend_job::{
use crate::workspace_deletion::WorkspaceDeletionStore;
use crate::{Error, Result};
#[path = "subject_deletion_store.rs"]
mod subject_deletion_store;
pub(crate) use subject_deletion_store::{
prior_removal_matches as subject_deletion_prior_removal_matches,
worker_kind as subject_deletion_worker_kind,
};
#[path = "activation.rs"]
mod activation;
pub use activation::ActivationCodeIssue;
@@ -119,7 +126,7 @@ pub use ticket::{
};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 104;
const LATEST_SCHEMA_VERSION: i64 = 105;
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
@@ -447,6 +454,11 @@ const MIGRATIONS: &[Migration] = &[
name: "Converge Passkey results and Job input retention",
apply: migrate_schema_103_variants_to_v104,
},
Migration {
version: 105,
name: "Durable Subject deletion snapshots and Job admission fences",
apply: subject_deletion_store::migrate,
},
];
// Both schema-103 identities shipped. Keep the alternate implementation and accept
@@ -3000,6 +3012,7 @@ impl SqliteWorkspaceStore {
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
subject_deletion_store::require_worker_admission(&tx, workspace_id, allocation_key, singleton_key)?;
let fixed_worker_id = prepare(&tx)?;
let existing = tx
.query_row(
@@ -7455,6 +7468,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let deadline = backend_job_deadline(now, request.limits.timeout_seconds)?;
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
subject_deletion_store::require_job_admission(&tx, workspace_id, request)?;
if let Some(job) = read_backend_job(&tx, workspace_id, &request.job_id)? {
if job.intent_fingerprint != fingerprint {
return Err(Error::InvalidInput(format!(
@@ -7532,7 +7546,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
return Ok(job);
}
if reason == JobInputRetirementReason::Completed {
job.load_executable_request()?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))?
.ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?;
if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed
@@ -7562,6 +7577,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let mut job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let previous_id = attempt_id(job_id, job.current_attempt);
let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)?
@@ -7701,6 +7717,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
@@ -7764,6 +7781,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
@@ -7827,6 +7845,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| {
@@ -7880,6 +7899,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{}`", submission.job_id)))?;
let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
if submission.input_digest != job.input_digest
|| attempt.input_digest != submission.input_digest
@@ -7908,6 +7928,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
tx.commit()?;
return Ok(BackendJobResultAcceptance { job, attempt, replayed: true });
}
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
return Err(Error::InvalidInput(
@@ -7972,6 +7993,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?;
// Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence.
if &attempt != expected || job.current_attempt != attempt.attempt
@@ -8277,6 +8299,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<(BackendJobDeliveryRecord, bool)> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let deleted:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_jobs WHERE workspace_id=?1 AND job_id=?2)",params![workspace_id,job_id],|r|r.get(0))?;
if deleted {return Err(Error::RepositoryConflict("subject_deleted: Job delivery fenced".into()));}
if let Some(existing) = read_backend_job_delivery(&tx, workspace_id, delivery_id)? {
if existing.job_id != job_id || existing.target != *target {
return Err(Error::InvalidInput(
@@ -15545,6 +15569,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
notification_routing::jobs::install(conn)?;
notification_routing::install_business_events(conn)?;
notification_routing::upgrade_result_reports(conn)?;
subject_deletion_store::install_notification_erasure(conn)?;
ticket::verify_sqlite_ticket_schema(conn).map_err(|error| {
Error::Store(format!(
"fresh Workspace Ticket schema verification failed: {error}"
@@ -15583,10 +15608,17 @@ fn create_workspace_schema_structure_from_sql(
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
conn.execute_batch(&schema_sql.replace(
"-- Ticket-owned assignment schema is composed by store.rs.",
ticket::TICKET_ASSIGNMENT_SCHEMA_SQL,
))?;
conn.execute_batch(
&schema_sql
.replace(
"-- Ticket-owned assignment schema is composed by store.rs.",
ticket::TICKET_ASSIGNMENT_SCHEMA_SQL,
)
.replace(
"-- Subject deletion schema is composed by store.rs.",
include_str!("subject_deletion_schema.sql"),
),
)?;
let composed_sql = r#"
ALTER TABLE typed_tickets RENAME TO typed_tickets_legacy_v66;
CREATE TABLE typed_tickets (
@@ -15809,6 +15841,9 @@ fn verify_workspace_resource_constraints(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? >= 96 {
notification_routing::jobs::verify(conn)?;
}
if current_schema_version(conn)? >= 105 {
subject_deletion_store::verify(conn)?;
}
Ok(())
}
@@ -16223,6 +16258,8 @@ fn current_worker_singleton_owner(
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r JOIN subject_deletion_workers d ON d.workspace_id=r.workspace_id AND d.runtime_id=r.runtime_id AND d.worker_id=r.worker_id WHERE r.workspace_id=owner.workspace_id AND r.runtime_id=owner.runtime_id AND r.worker_id=owner.worker_id AND r.state='succeeded' AND owner.singleton_key='subjektiv:'||d.subject_id AND (d.prior_removal_operation_id=r.operation_id OR (d.prior_removal_operation_id IS NULL AND d.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.metadata_disposition='tombstone'))) \
AND NOT EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=owner.workspace_id AND d.runtime_id=owner.runtime_id AND d.worker_id=owner.worker_id AND owner.singleton_key='subjektiv:'||d.subject_id AND d.originals_absent=1) \
THEN 'created' ELSE reservation.state END \
FROM worker_singleton_owners owner \
LEFT JOIN worker_create_reservations reservation \
@@ -19265,6 +19302,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
WorkspaceSchemaMigrationStep {version:103, name:"Passkey metadata and atomic addition results".into()},
WorkspaceSchemaMigrationStep {version:104, name:"Converge Passkey results and Job input retention".into()},
WorkspaceSchemaMigrationStep {version:105, name:"Durable Subject deletion snapshots and Job admission fences".into()},
]
);
@@ -19379,6 +19417,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(102, "Durable failed Subject creation recovery".to_string()),
(103, "Passkey metadata and atomic addition results".to_string()),
(104, "Converge Passkey results and Job input retention".to_string()),
(105, "Durable Subject deletion snapshots and Job admission fences".to_string()),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -22664,7 +22703,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into()),
(103, "Passkey metadata and atomic addition results".into()),
(104, "Converge Passkey results and Job input retention".into())]);
(104, "Converge Passkey results and Job input retention".into()),
(105, "Durable Subject deletion snapshots and Job admission fences".into())]);
for (table, rows) in unchanged.iter().zip(before.iter()) {
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
}
@@ -26857,12 +26897,19 @@ INSERT INTO worker_registry (
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap();
store.with_conn(|conn| {
// This fixture deliberately reconstructs schema 102 from fresh.
// Remove the newer Subject fence composition, not just its receipt.
let triggers=conn.prepare("SELECT name,tbl_name FROM sqlite_schema WHERE type='trigger' AND tbl_name LIKE 'notification_%' AND name NOT LIKE 'subject_deleted_%' AND sql LIKE '%subject_deletion_jobs%'")?
.query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
for (name,table) in triggers {conn.execute_batch(&format!("DROP TRIGGER {name}; CREATE TRIGGER {name} BEFORE DELETE ON {table} BEGIN SELECT RAISE(ABORT,'Notification evidence is immutable'); END;"))?;}
for name in ["subject_deleted_job_insert","subject_deleted_job_result","subject_deleted_attempt_insert","subject_deleted_attempt_result","subject_deleted_worker_reservation","subject_deleted_notification_source","subject_deleted_notification_payload","subject_deleted_notification_send","subject_deletion_worker_prior_insert","subject_deletion_worker_identity_immutable"] {conn.execute_batch(&format!("DROP TRIGGER {name}"))?;}
conn.execute_batch("DROP TABLE subject_deletion_allocations; DROP TABLE subject_deletion_jobs; DROP TABLE subject_deletion_workers; DROP TABLE subject_deletions;")?;
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
DROP TRIGGER passkey_metadata_insert;
DROP TABLE passkey_addition_receipts;
DROP TABLE passkey_metadata;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=105;
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
}).unwrap();
drop(store);
@@ -0,0 +1,73 @@
-- Content-free, monotonic main-DB fence. The Host installs the domain fence first
-- and holds its Subject gate across snapshot/admission. No Subject contents live here.
CREATE TABLE subject_deletions (
workspace_id TEXT NOT NULL,
subject_id TEXT NOT NULL,
operation_id TEXT NOT NULL CHECK(length(operation_id)>0),
completed INTEGER NOT NULL DEFAULT 0 CHECK(completed IN (0,1)),
jobs_erased INTEGER NOT NULL DEFAULT 0 CHECK(jobs_erased IN (0,1)),
PRIMARY KEY(workspace_id,subject_id),
UNIQUE(workspace_id,operation_id),
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
);
CREATE TABLE subject_deletion_workers (
workspace_id TEXT NOT NULL,
subject_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
is_job INTEGER NOT NULL CHECK(is_job IN (0,1)),
-- Exact canonical success already present at prepare, never a later receipt.
prior_removal_operation_id TEXT REFERENCES worker_removal_operations(operation_id),
-- Confirmed by the scoped Runtime storage owner, not catalog absence or Stop ACK.
originals_absent INTEGER NOT NULL DEFAULT 0 CHECK(originals_absent IN (0,1)),
PRIMARY KEY(workspace_id,subject_id,runtime_id,worker_id),
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
);
CREATE TRIGGER subject_deletion_worker_prior_insert BEFORE INSERT ON subject_deletion_workers
WHEN NEW.prior_removal_operation_id IS NOT NULL AND NOT EXISTS(
SELECT 1 FROM worker_removal_operations r WHERE r.operation_id=NEW.prior_removal_operation_id
AND r.workspace_id=NEW.workspace_id AND r.runtime_id=NEW.runtime_id
AND r.worker_id=NEW.worker_id AND r.state='succeeded')
BEGIN SELECT RAISE(ABORT,'Subject deletion prior removal must be an exact canonical success'); END;
CREATE TRIGGER subject_deletion_worker_identity_immutable BEFORE UPDATE OF
workspace_id,subject_id,runtime_id,worker_id,prior_removal_operation_id ON subject_deletion_workers
BEGIN SELECT RAISE(ABORT,'Subject deletion Worker snapshot identity is immutable'); END;
CREATE TRIGGER subject_deletion_original_absence_monotonic BEFORE UPDATE OF originals_absent ON subject_deletion_workers
WHEN OLD.originals_absent=1 AND NEW.originals_absent=0
BEGIN SELECT RAISE(ABORT,'Subject deletion original absence proof is permanent'); END;
-- Job/attempt identities remain after payload erasure, preventing stale allocation
-- or caller-selected Job-id reuse from recreating the deleted Subject's inputs.
CREATE TABLE subject_deletion_jobs (
workspace_id TEXT NOT NULL,
subject_id TEXT NOT NULL,
job_id TEXT NOT NULL,
PRIMARY KEY(workspace_id,job_id),
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
);
CREATE TABLE subject_deletion_allocations (
workspace_id TEXT NOT NULL,
subject_id TEXT NOT NULL,
allocation_key TEXT NOT NULL,
PRIMARY KEY(workspace_id,allocation_key),
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
);
CREATE TRIGGER subject_deleted_job_insert BEFORE INSERT ON backend_jobs
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
(d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_journal.subject_id') OR
d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_consolidation.subject_id')))
OR EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job admission fenced'); END;
CREATE TRIGGER subject_deleted_job_result BEFORE UPDATE OF state,result_json ON backend_jobs
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job result fenced'); END;
CREATE TRIGGER subject_deleted_attempt_insert BEFORE INSERT ON backend_job_attempts
WHEN EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job retry fenced'); END;
CREATE TRIGGER subject_deleted_attempt_result BEFORE UPDATE OF state,result_json,runtime_id,worker_id,runtime_run_id ON backend_job_attempts
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job attempt fenced'); END;
CREATE TRIGGER subject_deleted_worker_reservation BEFORE INSERT ON worker_create_reservations
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
NEW.singleton_key IN ('subjektiv:'||d.subject_id,'subjektiv-consolidation:'||d.subject_id))
OR EXISTS(SELECT 1 FROM subject_deletion_allocations d WHERE d.workspace_id=NEW.workspace_id AND d.allocation_key=NEW.allocation_key)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Worker allocation fenced'); END;
File diff suppressed because it is too large Load Diff
@@ -94,6 +94,12 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"workdir_registry",
"workdir_removal_operations",
"worker_control_grants",
// Snapshot children precede their fence parent and canonical Worker removal
// receipts (subject_deletion_workers.prior_removal_operation_id).
"subject_deletion_allocations",
"subject_deletion_jobs",
"subject_deletion_workers",
"subject_deletions",
"subject_worker_releases",
"subject_create_recoveries",
"worker_create_reservations",
@@ -1276,6 +1282,134 @@ mod tests {
}
}
#[test]
fn workspace_purge_removes_subject_deletion_snapshots_and_preserves_other_workspace() {
let (store, owner, workspace_id) = setup();
let tables = [
"subject_deletion_allocations",
"subject_deletion_jobs",
"subject_deletion_workers",
"subject_deletions",
"worker_removal_operations",
];
// Check the explicit FK ordering as well as execution (the purge defers FKs).
let position = |table| {
WORKSPACE_DELETION_PURGE_TABLES
.iter()
.position(|name| *name == table)
.unwrap()
};
for child in &tables[..3] {
assert!(position(*child) < position("subject_deletions"));
}
assert!(position("subject_deletion_workers") < position("worker_removal_operations"));
for workspace in ["workspace-a", "workspace-b"] {
store.with_conn(|conn| {
conn.execute("INSERT INTO worker_removal_operations(
operation_id,plan_id,input_fingerprint,workspace_id,runtime_id,worker_id,
worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition,
archive_retention_kind,diagnostics_disposition,blockers_json,state,reason,created_at,updated_at)
VALUES(?1,?1,'fingerprint',?1,'runtime','worker','1','policy','digest',
'purge','purge','forever','purge','[]','succeeded','removed','1','1')", [workspace])?;
conn.execute("INSERT INTO subject_deletions(workspace_id,subject_id,operation_id,completed,jobs_erased)
VALUES(?1,'subject','subject-delete',1,1)", [workspace])?;
conn.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id,originals_absent)
VALUES(?1,'subject','runtime','worker',1,?1,1)", [workspace])?;
conn.execute("INSERT INTO subject_deletion_jobs VALUES(?1,'subject','job')", [workspace])?;
conn.execute("INSERT INTO subject_deletion_allocations VALUES(?1,'subject','allocation')", [workspace])?;
Ok(())
}).unwrap();
}
let rows = |workspace: &str| {
store
.with_conn(|conn| {
tables
.iter()
.map(|table| {
let mut statement = conn
.prepare(&format!("SELECT * FROM {table} WHERE workspace_id=?1"))?;
let count = statement.column_count();
Ok(statement
.query_map([workspace], |row| {
(0..count)
.map(|index| row.get::<_, rusqlite::types::Value>(index))
.collect::<rusqlite::Result<Vec<_>>>()
})?
.collect::<rusqlite::Result<Vec<_>>>()?)
})
.collect::<Result<Vec<_>>>()
})
.unwrap()
};
let before = rows("workspace-a");
let other = rows("workspace-b");
assert!(before.iter().all(|table| table.len() == 1));
let preflight = store
.workspace_deletion_preflight(&owner, &workspace_id)
.unwrap();
let operation = store
.reserve_workspace_deletion(
&owner,
&workspace_id,
&WorkspaceDeletionRequest {
operation_id: "purge-subject-snapshots".into(),
expected_workspace_updated_at: preflight.expected_workspace_updated_at,
confirmation: "Alpha".into(),
},
)
.unwrap()
.operation;
store
.with_conn(|conn| {
conn.execute_batch(
"CREATE TRIGGER reject_subject_snapshot_purge BEFORE DELETE ON workspaces
BEGIN SELECT RAISE(ABORT,'injected purge failure'); END;",
)?;
Ok(())
})
.unwrap();
assert!(
store
.finalize_workspace_deletion(&operation.operation_id)
.is_err()
);
assert_eq!(
rows("workspace-a"),
before,
"rollback restores snapshots and canonical receipt"
);
assert_eq!(rows("workspace-b"), other);
store
.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER reject_subject_snapshot_purge")?;
Ok(())
})
.unwrap();
let completed = store
.finalize_workspace_deletion(&operation.operation_id)
.unwrap();
assert_eq!(completed.state, WorkspaceDeletionState::Succeeded);
assert!(rows("workspace-a").iter().all(|table| table.is_empty()));
assert_eq!(rows("workspace-b"), other);
assert_eq!(
store
.finalize_workspace_deletion(&operation.operation_id)
.unwrap(),
completed
);
store
.with_conn(|conn| {
assert_eq!(
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| {
row.get::<_, i64>(0)
})?,
0
);
Ok(())
})
.unwrap();
}
#[test]
fn deletion_is_idempotent_and_removes_workspace_scoped_rows() {
let (store, owner, workspace_id) = setup();