feat: replace Subject retirement with durable explicit deletion
This commit is contained in:
@@ -478,7 +478,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
(101, "Durable explicit Subject body release".into()),
|
||||
(102, "Durable failed Subject creation recovery".into()),
|
||||
(103, "Passkey metadata and atomic addition results".into()),
|
||||
(104, "Converge Passkey results and Job input retention".into())
|
||||
(104, "Converge Passkey results and Job input retention".into()),
|
||||
(105, "Durable Subject deletion snapshots and Job admission fences".into())
|
||||
]
|
||||
);
|
||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||
|
||||
@@ -1292,6 +1292,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
_request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
|
||||
Err("Subject deletion fencing is unavailable on this Runtime".into())
|
||||
}
|
||||
|
||||
fn recover_subject_create(
|
||||
&self,
|
||||
_request: runtime_api::SubjectCreateRecoveryRequest,
|
||||
@@ -2269,6 +2276,21 @@ impl RuntimeRegistry {
|
||||
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
|
||||
}
|
||||
|
||||
pub fn subject_delete_worker(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, RuntimeRegistryError> {
|
||||
validate_backend_identifier("runtime_id", runtime_id)?;
|
||||
self.runtime(runtime_id)?
|
||||
.subject_delete_worker(request)
|
||||
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: runtime_id.into(),
|
||||
code: "subject_delete_worker_unproven".into(),
|
||||
message,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn recover_subject_create(
|
||||
&self,
|
||||
runtime_id: &str,
|
||||
@@ -3367,6 +3389,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
|
||||
if request.workspace_id != self.workspace_id {
|
||||
return Err("Subject deletion Workspace mismatch".into());
|
||||
}
|
||||
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
|
||||
self.runtime
|
||||
.subject_delete_worker_scoped(&scope, request)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn recover_subject_create(
|
||||
&self,
|
||||
request: runtime_api::SubjectCreateRecoveryRequest,
|
||||
@@ -4255,7 +4290,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
|
||||
}
|
||||
if path.contains("/retention/")
|
||||
|| (path.starts_with("/v1/workers/") && method == "DELETE")
|
||||
|| (path == "/v1/subject-create-recovery" && method == "POST")
|
||||
|| (matches!(
|
||||
path,
|
||||
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
|
||||
) && method == "POST")
|
||||
{
|
||||
return "workers:delete";
|
||||
}
|
||||
@@ -5656,6 +5694,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
request: runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
|
||||
self.run_runtime_api(
|
||||
self.request_timeout,
|
||||
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
|
||||
move |client| async move { client.subject_delete_worker(request).await },
|
||||
)
|
||||
.map_err(|diagnostic| diagnostic.message)
|
||||
}
|
||||
|
||||
fn recover_subject_create(
|
||||
&self,
|
||||
request: runtime_api::SubjectCreateRecoveryRequest,
|
||||
@@ -8507,6 +8557,57 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_subject_delete_uses_generated_exact_identity_transport() {
|
||||
assert_eq!(
|
||||
workspace_runtime_operation("POST", "/v1/subject-delete-worker"),
|
||||
"workers:delete"
|
||||
);
|
||||
let request = runtime_api::SubjectDeleteWorkerRequest {
|
||||
workspace_id: "workspace-test".into(),
|
||||
worker_id: EmbeddedWorkerId::now_v7(),
|
||||
create_fingerprint: "sha256:reserved-create".into(),
|
||||
operation_id: "delete-worker-1".into(),
|
||||
};
|
||||
let response = runtime_api::SubjectDeleteWorkerResponse {
|
||||
originals_absent: true,
|
||||
workspace_id: request.workspace_id.clone(),
|
||||
worker_id: request.worker_id,
|
||||
create_fingerprint: request.create_fingerprint.clone(),
|
||||
operation_id: request.operation_id.clone(),
|
||||
};
|
||||
// Only remote transport is substituted; actual Controller shutdown and
|
||||
// signed admission are covered by worker-runtime's independent suites.
|
||||
let (base_url, server) = serve_mock_http(vec![mock_response(
|
||||
"POST",
|
||||
"/v1/subject-delete-worker",
|
||||
true,
|
||||
200,
|
||||
serde_json::to_string(&response).unwrap(),
|
||||
)]);
|
||||
let registry = RuntimeRegistry::new(Vec::new());
|
||||
registry.register(
|
||||
RemoteWorkerRuntime::new(
|
||||
RemoteRuntimeConfig::new(
|
||||
"remote:delete",
|
||||
"Deletion transport fixture",
|
||||
base_url,
|
||||
Some("secret-token".into()),
|
||||
),
|
||||
"workspace-test".into(),
|
||||
"http://127.0.0.1:8787".into(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
assert_eq!(
|
||||
registry
|
||||
.subject_delete_worker("remote:delete", request)
|
||||
.unwrap(),
|
||||
response
|
||||
);
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_subject_create_recovery_uses_generated_management_transport() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -367,14 +367,54 @@ async fn implementation_result_rejects_forged_evidence_and_rolls_back_without_no
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
|
||||
let (dir, store) = tracking_store_fixture().await;
|
||||
tracking_store_assign(&store, "delegation", "assign");
|
||||
let worker = routing_worker_assign(&store, "implementation").worker;
|
||||
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
|
||||
store.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event;
|
||||
// Preserve seeded assignment evidence while rebuilding the actual pre-business
|
||||
// capture composition. Changing only a receipt leaves schema105 admission fences
|
||||
// and erasure guards in an otherwise legacy fixture.
|
||||
fn reconstruct_implementation_schema96(conn: &Connection) -> Result<()> {
|
||||
assert_eq!(current_schema_version(conn)?, 105);
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("retained96.db");
|
||||
prepare_retained_schema(&path, 96);
|
||||
let retained = Connection::open(path)?;
|
||||
for trigger in [
|
||||
"notification_sources_no_delete",
|
||||
"notification_decisions_no_delete",
|
||||
"notification_requests_no_delete",
|
||||
"notification_acknowledgements_no_delete",
|
||||
"notification_failures_no_delete",
|
||||
"notification_job_targets_no_delete",
|
||||
"notification_job_legacy_no_delete",
|
||||
"notification_source_preparation_failures_no_delete",
|
||||
] {
|
||||
let definition: String = retained.query_row(
|
||||
"SELECT sql FROM sqlite_schema WHERE type='trigger' AND name=?1",
|
||||
[trigger],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
conn.execute_batch(&format!("DROP TRIGGER {trigger}; {definition};"))?;
|
||||
}
|
||||
for trigger in [
|
||||
"subject_deleted_job_insert",
|
||||
"subject_deleted_job_result",
|
||||
"subject_deleted_attempt_insert",
|
||||
"subject_deleted_attempt_result",
|
||||
"subject_deleted_worker_reservation",
|
||||
"subject_deleted_notification_source",
|
||||
"subject_deleted_notification_payload",
|
||||
"subject_deleted_notification_send",
|
||||
] {
|
||||
conn.execute_batch(&format!("DROP TRIGGER {trigger};"))?;
|
||||
}
|
||||
// Children before parents; dropping the tables also drops snapshot guards.
|
||||
conn.execute_batch(
|
||||
"DROP TABLE subject_deletion_allocations;
|
||||
DROP TABLE subject_deletion_jobs;
|
||||
DROP TABLE subject_deletion_workers;
|
||||
DROP TABLE subject_deletions;
|
||||
UPDATE __yoi_schema_migrations SET version=104 WHERE version=105;",
|
||||
)?;
|
||||
conn.execute_batch(
|
||||
"DROP TRIGGER notification_mr_event;
|
||||
DROP TRIGGER notification_mr_source;
|
||||
DROP TRIGGER notification_implementation_result;
|
||||
DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
@@ -396,7 +436,43 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
|
||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;",
|
||||
)?;
|
||||
assert_eq!(current_schema_version(conn)?, 96);
|
||||
assert_eq!(conn.query_row(
|
||||
"SELECT count(*) FROM sqlite_schema WHERE name LIKE 'subject_delet%' OR sql LIKE '%subject_deletion%'",
|
||||
[], |row| row.get::<_, i64>(0),
|
||||
)?, 0);
|
||||
for table in [
|
||||
"notification_sources",
|
||||
"notification_decisions",
|
||||
"notification_requests",
|
||||
"notification_acknowledgements",
|
||||
"notification_failures",
|
||||
"notification_job_targets",
|
||||
"notification_job_legacy_receipts",
|
||||
"notification_source_preparation_failures",
|
||||
"backend_jobs",
|
||||
"backend_job_attempts",
|
||||
"worker_create_reservations",
|
||||
] {
|
||||
assert_eq!(
|
||||
state_contract_schema_signature(conn, table),
|
||||
state_contract_schema_signature(&retained, table),
|
||||
"schema96 {table}"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
|
||||
let (dir, store) = tracking_store_fixture().await;
|
||||
tracking_store_assign(&store, "delegation", "assign");
|
||||
let worker = routing_worker_assign(&store, "implementation").worker;
|
||||
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
|
||||
store.with_conn(|conn| {
|
||||
reconstruct_implementation_schema96(conn)?;
|
||||
migrate_business_notifications_v96_to_v97(conn)?;
|
||||
assert_eq!(current_schema_version(conn)?, 97);
|
||||
Ok(())
|
||||
@@ -522,24 +598,7 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
|
||||
store.with_conn(|conn| {
|
||||
conn.execute("INSERT INTO ticket_assignment_operation_fences VALUES('w','operation-known-implementation','delegation','r','worker','workspace-orchestrator','tracking-binding',?1)",[serde_json::to_string(&tracking_store_source()).unwrap()])?;
|
||||
// Reconstruct the prior composition before its new business capture existed.
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TRIGGER passkey_metadata_insert;
|
||||
DROP TABLE passkey_addition_receipts;
|
||||
DROP TABLE passkey_metadata;
|
||||
DROP TRIGGER backend_job_input_retirement;
|
||||
DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||
DROP TABLE subject_create_recoveries;
|
||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
|
||||
reconstruct_implementation_schema96(conn)?;
|
||||
conn.execute_batch("INSERT INTO merge_requests VALUES('w','historical-mr','main','open','refs/heads/work','refs/heads/main','3','3');
|
||||
INSERT INTO merge_request_ticket_relations VALUES('w','historical-mr','tracking','implements','3');
|
||||
INSERT INTO merge_request_thread_events VALUES('w','historical-mr','approved-event',1,'review','{\"subject_ref\":\"source-a\",\"decision\":\"approve\"}',NULL,'3');
|
||||
@@ -691,24 +750,7 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
|
||||
];
|
||||
store.with_conn(|conn| {
|
||||
if historical {
|
||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||
DROP TRIGGER passkey_metadata_insert;
|
||||
DROP TABLE passkey_addition_receipts;
|
||||
DROP TABLE passkey_metadata;
|
||||
DROP TRIGGER backend_job_input_retirement;
|
||||
DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||
DROP TABLE subject_create_recoveries;
|
||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||
DROP TABLE subject_worker_releases;
|
||||
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
|
||||
DROP TABLE user_activations;
|
||||
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
|
||||
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
|
||||
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
|
||||
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
|
||||
reconstruct_implementation_schema96(conn)?;
|
||||
}
|
||||
conn.execute_batch("INSERT INTO merge_requests VALUES('w','actor-mr','main','open','work','main','3','3');
|
||||
INSERT INTO merge_request_ticket_relations VALUES('w','actor-mr','tracking','implements','3');")?;
|
||||
|
||||
@@ -1757,3 +1757,5 @@ WHEN (OLD.runtime_id != NEW.runtime_id OR OLD.worker_id != NEW.worker_id OR OLD.
|
||||
WHERE workspace_id = OLD.workspace_id AND runtime_id = OLD.runtime_id
|
||||
AND worker_id = OLD.worker_id)
|
||||
BEGIN SELECT RAISE(ABORT, 'Worker has unfinished Ticket work'); END;
|
||||
|
||||
-- Subject deletion schema is composed by store.rs.
|
||||
|
||||
@@ -444,6 +444,28 @@ fn job_result_facts(
|
||||
|
||||
pub(super) fn deliver_job_result(api: &WorkspaceApi, acceptance: &BackendJobResultAcceptance) {
|
||||
use crate::store::JobNotificationFacts;
|
||||
// A recovered/captured result is not delivery authority. Serialize even the
|
||||
// Runtime send with deletion, so no prepared bytes escape after its fence.
|
||||
let subject = subjektiv_deletion::job_subject(&acceptance.job.request);
|
||||
let gate = match subject
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(api, subject))
|
||||
.transpose()
|
||||
{
|
||||
Ok(gate) => gate,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "Subject Job notification gate unavailable");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _guard = match gate.as_ref().map(|gate| gate.lock()).transpose() {
|
||||
Ok(guard) => guard,
|
||||
Err(_) => return,
|
||||
};
|
||||
if let Some(subject) = subject
|
||||
&& subjektiv_deletion::require_available(api, subject).is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let Some(_permit) = api.notification_delivery_lifecycle.begin() else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -849,7 +849,16 @@ async fn notification_routing_fresh_and_upgraded_schema_and_guards_match() {
|
||||
let snapshot = |conn: &Connection| -> Vec<(String, String)> {
|
||||
conn.prepare("SELECT name,sql FROM sqlite_schema WHERE name LIKE 'notification_%' OR name LIKE 'ticket_assignment_content%' OR name IN ('ticket_assignment_title_input','ticket_assignment_initial_post_input') ORDER BY name").unwrap().query_map([],|r|Ok((r.get(0)?,r.get(1)?))).unwrap().collect::<rusqlite::Result<_>>().unwrap()
|
||||
};
|
||||
let upgraded = snapshot(&conn);
|
||||
assert_eq!(current_schema_version(&conn).unwrap(), 98);
|
||||
drop(conn);
|
||||
// Compare the complete current composition, not upgraded98 against fresh105.
|
||||
let upgraded_store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||
let upgraded = upgraded_store
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(current_schema_version(conn)?, 105);
|
||||
Ok(snapshot(conn))
|
||||
})
|
||||
.unwrap();
|
||||
let (fresh_directory, fresh) = tracking_store_fixture().await;
|
||||
fresh
|
||||
.with_conn(|conn| {
|
||||
|
||||
@@ -333,6 +333,15 @@ impl SqliteWorkspaceStore {
|
||||
return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()}));
|
||||
}
|
||||
};
|
||||
if crate::store::subject_deletion_worker_kind(&tx, &req.workspace_id, &req.worker)?.is_some() {
|
||||
let prior = tx.prepare("SELECT plan_id FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('executing','failed')")?
|
||||
.query_map(params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|r.get::<_,String>(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
for id in prior {
|
||||
let prior=load_plan(&tx,&id)?.ok_or_else(||StoreError::Store("removal plan disappeared".into()))?;
|
||||
require_current_subject_deletion_plan(&tx,&prior)?;
|
||||
}
|
||||
}
|
||||
let mut blockers=Vec::new();
|
||||
if worker.retention_state=="pinned" { blockers.push(WorkerRemovalBlocker::Hold); }
|
||||
if let Some((assignment_id,ticket_id))=tx.query_row("SELECT a.assignment_id,a.ticket_id FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3",params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()? {
|
||||
@@ -363,6 +372,7 @@ impl SqliteWorkspaceStore {
|
||||
let mut plan=load_plan(&tx,plan_id)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:plan missing")))?;
|
||||
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
|
||||
if plan.input_fingerprint!=fp{return Err(StoreError::InvalidInput(format!("fingerprint:{}",plan.operation_id)));}
|
||||
require_current_subject_deletion_plan(&tx, &plan)?;
|
||||
if plan.state==WorkerRemovalPlanState::Succeeded{tx.commit()?;return Ok(plan);}
|
||||
if plan.state==WorkerRemovalPlanState::Blocked {
|
||||
return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));
|
||||
@@ -420,6 +430,8 @@ impl SqliteWorkspaceStore {
|
||||
WorkerRetentionError::Invalid("Worker id must be a canonical UUIDv7".to_string())
|
||||
})?;
|
||||
let removed_at = plan.created_at.clone();
|
||||
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
|
||||
.map_err(map_error)?;
|
||||
let prior_failure_category = plan.failure_category.clone();
|
||||
Ok(PreparedWorkerRemoval {
|
||||
runtime_request: WorkerRetentionExecutionRequest {
|
||||
@@ -459,8 +471,8 @@ impl SqliteWorkspaceStore {
|
||||
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
|
||||
)
|
||||
)
|
||||
ORDER BY CASE state WHEN 'succeeded' THEN 0 ELSE 1 END,
|
||||
created_at DESC LIMIT 1",
|
||||
ORDER BY CASE WHEN EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=worker_removal_operations.workspace_id AND d.runtime_id=worker_removal_operations.runtime_id AND d.worker_id=worker_removal_operations.worker_id AND d.prior_removal_operation_id=worker_removal_operations.operation_id) THEN 0 WHEN state='succeeded' THEN 1 ELSE 2 END,
|
||||
created_at DESC,operation_id DESC LIMIT 1",
|
||||
params![workspace_id, worker.runtime_id, worker.worker_id],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
@@ -474,6 +486,8 @@ impl SqliteWorkspaceStore {
|
||||
let Some(plan) = plan else {
|
||||
return Ok(None);
|
||||
};
|
||||
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
|
||||
.map_err(map_error)?;
|
||||
let prior_failure_category = plan.failure_category.clone();
|
||||
let worker_id = plan
|
||||
.worker
|
||||
@@ -659,6 +673,7 @@ impl SqliteWorkspaceStore {
|
||||
let mut plan=load_plan_op(&tx,operation_id)?.ok_or_else(||StoreError::InvalidInput("operation missing".into()))?;
|
||||
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
|
||||
if plan.input_fingerprint!=fp||result.input_fingerprint!=fp||result.operation_id!=operation_id{return Err(StoreError::InvalidInput(format!("fingerprint:{operation_id}")));}
|
||||
require_current_subject_deletion_plan(&tx, &plan)?;
|
||||
if plan.state==WorkerRemovalPlanState::Succeeded{
|
||||
// Repair projection fences left by the pre-atomic implementation and
|
||||
// return a replayable change for a broadcast missed after commit.
|
||||
@@ -1024,6 +1039,49 @@ fn load_removal_policy(
|
||||
let Some(mut policy) = load_policy(c, w)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// Snapshot authority outranks normal policy and Journal cleanup policy. Body
|
||||
// and delegated Sessions must never inherit the ephemeral Job exception.
|
||||
if let Some(is_job) = crate::store::subject_deletion_worker_kind(c, w, worker)? {
|
||||
let update = WorkerRetentionPolicyUpdate {
|
||||
policy_id: if is_job {
|
||||
"subject-deletion-job-v1"
|
||||
} else {
|
||||
"subject-deletion-conversation-v1"
|
||||
}
|
||||
.into(),
|
||||
session_disposition: if is_job {
|
||||
SessionDisposition::Purge
|
||||
} else {
|
||||
SessionDisposition::Archive
|
||||
},
|
||||
metadata_disposition: MetadataDisposition::Tombstone,
|
||||
archive_retention: ArchiveRetention::Forever,
|
||||
// Runtime calls diagnostic archiving Retain, not Archive. Preserve
|
||||
// the Workspace's duration, or thirty days when it normally purges.
|
||||
diagnostics_disposition: if is_job {
|
||||
DiagnosticsDisposition::Purge
|
||||
} else {
|
||||
DiagnosticsDisposition::Retain
|
||||
},
|
||||
diagnostics_retention_seconds: if is_job {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
policy
|
||||
.diagnostics_retention_seconds
|
||||
.unwrap_or(30 * 24 * 60 * 60),
|
||||
)
|
||||
},
|
||||
};
|
||||
policy.policy_id = update.policy_id.clone();
|
||||
policy.policy_digest = worker_retention_policy_digest(&update);
|
||||
policy.session_disposition = update.session_disposition;
|
||||
policy.metadata_disposition = update.metadata_disposition;
|
||||
policy.archive_retention = update.archive_retention;
|
||||
policy.diagnostics_disposition = update.diagnostics_disposition;
|
||||
policy.diagnostics_retention_seconds = update.diagnostics_retention_seconds;
|
||||
return Ok(Some(policy));
|
||||
}
|
||||
if erased_journal || crate::store::terminal_journal_job_worker(c, w, worker)? {
|
||||
let update = WorkerRetentionPolicyUpdate {
|
||||
policy_id: if erased_journal {
|
||||
@@ -1047,6 +1105,39 @@ fn load_removal_policy(
|
||||
}
|
||||
Ok(Some(policy))
|
||||
}
|
||||
fn require_current_subject_deletion_plan(
|
||||
c: &Connection,
|
||||
plan: &WorkerRemovalPlan,
|
||||
) -> crate::Result<()> {
|
||||
if plan.state == WorkerRemovalPlanState::Succeeded
|
||||
&& crate::store::subject_deletion_prior_removal_matches(
|
||||
c,
|
||||
&plan.workspace_id,
|
||||
&plan.worker,
|
||||
&plan.operation_id,
|
||||
)?
|
||||
{
|
||||
// An exact successful receipt pinned before Subject deletion owns its
|
||||
// independent original/archive lifecycle. Never retarget its policy.
|
||||
return Ok(());
|
||||
}
|
||||
if crate::store::subject_deletion_worker_kind(c, &plan.workspace_id, &plan.worker)?.is_some() {
|
||||
let current = load_removal_policy(c, &plan.workspace_id, &plan.worker, false)?
|
||||
.ok_or_else(|| StoreError::InvalidInput("Subject deletion policy missing".into()))?;
|
||||
if plan.session_disposition != current.session_disposition
|
||||
|| plan.metadata_disposition != current.metadata_disposition
|
||||
|| plan.archive_retention != current.archive_retention
|
||||
|| plan.diagnostics_disposition != current.diagnostics_disposition
|
||||
{
|
||||
return Err(stale_error(
|
||||
plan,
|
||||
"Subject deletion retention differs from saved plan; reconcile canonical Runtime removal before retry",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn load_policy(c: &Connection, w: &str) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
||||
c.query_row(
|
||||
"SELECT p.policy_id,p.policy_digest,r.session_disposition,r.metadata_disposition,
|
||||
@@ -2736,4 +2827,78 @@ mod tests {
|
||||
);
|
||||
assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_deletion_snapshot_overrides_policy_without_changing_normal_removal() {
|
||||
let store = setup();
|
||||
let ordinary = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||
store
|
||||
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
|
||||
.unwrap();
|
||||
let body = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||
assert_eq!(body.session_disposition, SessionDisposition::Archive);
|
||||
assert_eq!(body.archive_retention, ArchiveRetention::Forever);
|
||||
assert_eq!(body.metadata_disposition, MetadataDisposition::Tombstone);
|
||||
assert_eq!(body.diagnostics_disposition, DiagnosticsDisposition::Retain);
|
||||
assert!(body.diagnostics_retention_seconds.is_some());
|
||||
assert_ne!(body.policy_id, ordinary.policy_id);
|
||||
// A saved plan that would discard diagnostics cannot silently stand in
|
||||
// for the Subject conversation archive plan.
|
||||
assert!(matches!(
|
||||
store.begin_worker_removal("w", &ordinary.plan_id, &ordinary.input_fingerprint),
|
||||
Err(WorkerRetentionError::StalePlan { .. })
|
||||
));
|
||||
let ordinary_store = setup();
|
||||
assert_eq!(
|
||||
ordinary_store
|
||||
.plan_worker_removal(&req(), &inv())
|
||||
.unwrap()
|
||||
.policy_id,
|
||||
ordinary.policy_id
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subject_deletion_job_purge_rejects_recovery_of_old_executing_archive() {
|
||||
let store = setup();
|
||||
let old = store.plan_worker_removal(&req(), &inv()).unwrap();
|
||||
store
|
||||
.prepare_worker_removal_execution("w", &old.plan_id, &old.input_fingerprint)
|
||||
.unwrap();
|
||||
store
|
||||
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
|
||||
.unwrap();
|
||||
store
|
||||
.with_conn(|c| {
|
||||
c.execute(
|
||||
"UPDATE subject_deletion_workers SET is_job=1 WHERE workspace_id='w'",
|
||||
[],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
store.recover_worker_removal_execution("w", &req().worker),
|
||||
Err(WorkerRetentionError::StalePlan { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
store.plan_worker_removal(&req(), &inv()),
|
||||
Err(WorkerRetentionError::StalePlan { .. })
|
||||
));
|
||||
let fresh = store
|
||||
.with_conn(|c| load_removal_policy(c, "w", &req().worker, false))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(fresh.session_disposition, SessionDisposition::Purge);
|
||||
assert_eq!(fresh.diagnostics_disposition, DiagnosticsDisposition::Purge);
|
||||
assert_eq!(fresh.metadata_disposition, MetadataDisposition::Tombstone);
|
||||
assert_eq!(
|
||||
store
|
||||
.with_conn(|c| load_plan(c, &old.plan_id))
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.state,
|
||||
WorkerRemovalPlanState::Executing
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,10 +91,16 @@ mod schema_convergence {
|
||||
assert_eq!(plan.current_schema_version, 103);
|
||||
assert_eq!(
|
||||
plan.migrations,
|
||||
vec![WorkspaceSchemaMigrationStep {
|
||||
version: 104,
|
||||
name: "Converge Passkey results and Job input retention".into(),
|
||||
}]
|
||||
vec![
|
||||
WorkspaceSchemaMigrationStep {
|
||||
version: 104,
|
||||
name: "Converge Passkey results and Job input retention".into(),
|
||||
},
|
||||
WorkspaceSchemaMigrationStep {
|
||||
version: 105,
|
||||
name: "Durable Subject deletion snapshots and Job admission fences".into(),
|
||||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
schema_rows(&conn),
|
||||
@@ -109,6 +115,7 @@ mod schema_convergence {
|
||||
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||
let mut expected = history.clone();
|
||||
expected.push((104, "Converge Passkey results and Job input retention".into()));
|
||||
expected.push((105, "Durable Subject deletion snapshots and Job admission fences".into()));
|
||||
assert_eq!(workspace_schema_migration_history(conn)?, expected);
|
||||
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
|
||||
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
|
||||
@@ -138,7 +145,7 @@ mod schema_convergence {
|
||||
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||
assert_eq!(
|
||||
workspace_schema_migration_history(conn)?.len(),
|
||||
history.len() + 1
|
||||
history.len() + 2
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
@@ -176,7 +183,7 @@ mod schema_convergence {
|
||||
upgraded
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||
assert_eq!(current_schema_version(conn)?, 104);
|
||||
assert_eq!(current_schema_version(conn)?, 105);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
@@ -28,6 +28,7 @@ use subjektiv::api::*;
|
||||
mod backend_job_profile;
|
||||
mod external_operation_audit;
|
||||
mod subject_authorization;
|
||||
mod subjektiv_deletion;
|
||||
mod subjektiv_job;
|
||||
mod subjektiv_journal;
|
||||
mod subjektiv_journal_browser;
|
||||
@@ -4577,6 +4578,12 @@ impl WorkspaceApi {
|
||||
.run(move || recover_workdir_removals(&workdir_recovery_api))
|
||||
.await?;
|
||||
recover_runtime_removals(&api).await;
|
||||
let deletion_api = api.clone();
|
||||
let deletion_runtime = tokio::runtime::Handle::current();
|
||||
api.host_executors
|
||||
.host()?
|
||||
.run(move || deletion_runtime.block_on(subjektiv_deletion::recover(&deletion_api)))
|
||||
.await?;
|
||||
api.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
@@ -4604,9 +4611,23 @@ impl WorkspaceApi {
|
||||
) -> Result<BackendJobReservation> {
|
||||
self.resolve_backend_job_profile(request)?;
|
||||
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let reservation =
|
||||
let reservation = {
|
||||
let gate = subjektiv_deletion::job_subject(request)
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||
.transpose()?;
|
||||
let _guard = gate
|
||||
.as_ref()
|
||||
.map(|gate| {
|
||||
gate.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subjektiv_deletion::job_subject(request) {
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
self.store
|
||||
.reserve_backend_job(&self.config.workspace_id, request, &now)?;
|
||||
.reserve_backend_job(&self.config.workspace_id, request, &now)?
|
||||
};
|
||||
let result = self.dispatch_backend_job_reservation(reservation);
|
||||
if result.is_err() {
|
||||
self.drain_reserved_backend_jobs();
|
||||
@@ -4621,9 +4642,27 @@ impl WorkspaceApi {
|
||||
/// immutable request.
|
||||
pub fn retry_backend_job(&self, job_id: &str) -> Result<BackendJobReservation> {
|
||||
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let reservation =
|
||||
let reservation = {
|
||||
let current = self
|
||||
.store
|
||||
.get_backend_job(self.workspace_id(), job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput("unknown Backend Job".into()))?;
|
||||
let gate = subjektiv_deletion::job_subject(¤t.request)
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||
.transpose()?;
|
||||
let _guard = gate
|
||||
.as_ref()
|
||||
.map(|gate| {
|
||||
gate.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subjektiv_deletion::job_subject(¤t.request) {
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
self.store
|
||||
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?;
|
||||
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?
|
||||
};
|
||||
let result = self.dispatch_backend_job_reservation(reservation);
|
||||
if result.is_err() {
|
||||
self.drain_reserved_backend_jobs();
|
||||
@@ -4637,13 +4676,8 @@ impl WorkspaceApi {
|
||||
) -> Result<BackendJobReservation> {
|
||||
// Reread even a terminal/stale reservation before deciding whether it can
|
||||
// dispatch or replay: an explicit erasure may have retired its saved input.
|
||||
let journal_gate = reservation
|
||||
.job
|
||||
.request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
.map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id))
|
||||
let journal_gate = subjektiv_deletion::job_subject(&reservation.job.request)
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||
.transpose()?;
|
||||
let _journal_guard = journal_gate
|
||||
.as_ref()
|
||||
@@ -4652,6 +4686,9 @@ impl WorkspaceApi {
|
||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subjektiv_deletion::job_subject(&reservation.job.request) {
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
let current_job = self
|
||||
.store
|
||||
.get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)?
|
||||
@@ -5571,6 +5608,27 @@ impl WorkspaceApi {
|
||||
source_worker: &RuntimeWorkerRef,
|
||||
submission: &BackendJobResultSubmission,
|
||||
) -> Result<BackendJobResultAcceptance> {
|
||||
let result_job = self
|
||||
.store
|
||||
.get_backend_job(self.workspace_id(), &submission.job_id)?;
|
||||
let subject_gate = result_job
|
||||
.as_ref()
|
||||
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
|
||||
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
|
||||
.transpose()?;
|
||||
let subject_guard = subject_gate
|
||||
.as_ref()
|
||||
.map(|gate| {
|
||||
gate.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = result_job
|
||||
.as_ref()
|
||||
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
|
||||
{
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
|
||||
let mut verified_submission = submission.clone();
|
||||
if let Some(job) = self
|
||||
@@ -5696,6 +5754,7 @@ impl WorkspaceApi {
|
||||
&verified_submission,
|
||||
&accepted_at,
|
||||
)?;
|
||||
drop(subject_guard);
|
||||
if let Some(job) = self
|
||||
.store
|
||||
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
|
||||
@@ -5899,15 +5958,21 @@ impl WorkspaceApi {
|
||||
.map(|assignment| assignment.operation_id.clone())
|
||||
})
|
||||
.unwrap_or_else(|| format!("manual:{}", WorkerId::now_v7()));
|
||||
let subject = singleton_key
|
||||
let body_subject = singleton_key
|
||||
.as_deref()
|
||||
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
|
||||
if let Some(subject) = subject
|
||||
if let Some(subject) = body_subject
|
||||
&& let Err(error) = subjektiv_journal::recover_failed_subject_create(self, subject)
|
||||
{
|
||||
let diagnostics = cleanup_spawn_created_workdirs(self, &spawned_workdir_ids);
|
||||
return Err(ApiError::with_diagnostics(error, diagnostics));
|
||||
}
|
||||
let subject = body_subject.or_else(|| {
|
||||
request
|
||||
.resolved_execution_grant
|
||||
.as_ref()
|
||||
.map(|grant| grant.subject_id.as_str())
|
||||
});
|
||||
let subject_gate = subject
|
||||
.map(|s| subjektiv_journal::lifecycle_lock(self, s))
|
||||
.transpose()?;
|
||||
@@ -5919,6 +5984,9 @@ impl WorkspaceApi {
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(subject) = subject {
|
||||
subjektiv_deletion::require_available(self, subject)?;
|
||||
}
|
||||
if let Some(subject) = body_subject {
|
||||
subjektiv_journal::check_start_allowed(self, subject)?;
|
||||
}
|
||||
let reservation = match self.config_store.reserve_worker_create(
|
||||
@@ -5943,11 +6011,11 @@ impl WorkspaceApi {
|
||||
return Err(ApiError::with_diagnostics(error, diagnostics));
|
||||
}
|
||||
};
|
||||
drop(subject_guard);
|
||||
let worker_id = reservation.worker_id;
|
||||
if let Some(grant) = request.resolved_execution_grant.as_ref() {
|
||||
subject_authorization::bind_reserved_grant(self, grant, &worker_id.to_string())?;
|
||||
}
|
||||
drop(subject_guard);
|
||||
// A completed exact-source allocation is historical acceptance, not a new
|
||||
// launch. Source/grant authorization and the full fingerprint were checked
|
||||
// above. Do not reserve its already-linked attachments again, redispatch
|
||||
@@ -7458,7 +7526,7 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou
|
||||
.merge(server_api::server_api_axum::subjektiv_memory_changes(
|
||||
service.clone(),
|
||||
))
|
||||
.merge(server_api::server_api_axum::subjektiv_subject_retire(
|
||||
.merge(server_api::server_api_axum::subjektiv_subject_delete(
|
||||
service.clone(),
|
||||
))
|
||||
.merge(server_api::server_api_axum::subjektiv_stage_candidate(
|
||||
@@ -9724,22 +9792,25 @@ impl server_api::ServerApi for ServerApiContractService {
|
||||
.map_err(ApiError::into_repository_api_error)
|
||||
}
|
||||
|
||||
async fn subjektiv_subject_retire(
|
||||
async fn subjektiv_subject_delete(
|
||||
&self,
|
||||
context: server_api::ServerRequestContext,
|
||||
workspace_id: String,
|
||||
subject_id: String,
|
||||
) -> std::result::Result<server_api::SubjektivSubjectResponse, server_api::RepositoryApiError>
|
||||
{
|
||||
scoped_retire_subjektiv_subject(
|
||||
State(self.workspace_api()?.clone()),
|
||||
AxumPath(ScopedSubjektivSubjectPath {
|
||||
workspace_id,
|
||||
subject_id,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map(|Json(response)| response)
|
||||
.map_err(ApiError::into_repository_api_error)
|
||||
request: server_api::SubjektivSubjectDeleteRequest,
|
||||
) -> std::result::Result<
|
||||
server_api::SubjektivSubjectDeleteResponse,
|
||||
server_api::RepositoryApiError,
|
||||
> {
|
||||
let api = self.workspace_api()?.clone();
|
||||
let actor = subject_authorization::administration_actor(&context)
|
||||
.map_err(ApiError::into_repository_api_error)?;
|
||||
require_workspace_owner(&api, &workspace_id, actor, "Subject deletion")
|
||||
.await
|
||||
.map_err(ApiError::into_repository_api_error)?;
|
||||
subjektiv_deletion::delete(api, workspace_id, subject_id, request)
|
||||
.await
|
||||
.map_err(ApiError::into_repository_api_error)
|
||||
}
|
||||
|
||||
async fn subjektiv_stage_candidate(
|
||||
@@ -22674,7 +22745,9 @@ fn subjektiv_subject_response(
|
||||
memory_fingerprint,
|
||||
state: match subject.state {
|
||||
crate::subjektiv::SubjectState::Active => server_api::SubjektivSubjectState::Active,
|
||||
crate::subjektiv::SubjectState::Retired => server_api::SubjektivSubjectState::Retired,
|
||||
crate::subjektiv::SubjectState::LegacyRetired => {
|
||||
server_api::SubjektivSubjectState::LegacyRetired
|
||||
}
|
||||
},
|
||||
created_at: subject.created_at,
|
||||
updated_at: subject.updated_at,
|
||||
@@ -22962,24 +23035,6 @@ async fn scoped_list_subjektiv_memory_changes(
|
||||
.await
|
||||
}
|
||||
|
||||
async fn scoped_retire_subjektiv_subject(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedSubjektivSubjectPath>,
|
||||
) -> ApiResult<Json<server_api::SubjektivSubjectResponse>> {
|
||||
api.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
validate_workspace_scope(&api, &path.workspace_id)?;
|
||||
let subject = open_subjektiv_store(&api)?
|
||||
.retire_subject(&path.subject_id)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
// Retirement deliberately does not stop or remove the current Worker. The
|
||||
// Worker lease and subject lifecycle are independent authorities.
|
||||
Ok(Json(subjektiv_subject_response(&api, subject)?))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn scoped_stage_subjektiv_candidate(
|
||||
State(api): State<WorkspaceApi>,
|
||||
AxumPath(path): AxumPath<ScopedWorkspacePath>,
|
||||
@@ -24564,6 +24619,7 @@ fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error {
|
||||
Error::WorkspacePermissionDenied(error.to_string())
|
||||
}
|
||||
crate::subjektiv::SubjektivError::SubjectPermissionsConflict(_)
|
||||
| crate::subjektiv::SubjektivError::SubjectDeletionConflict(_)
|
||||
| crate::subjektiv::SubjektivError::PermissionOperationConflict(_)
|
||||
| crate::subjektiv::SubjektivError::WorkerExecutionGrantConflict(_) => {
|
||||
Error::RepositoryConflict(error.to_string())
|
||||
@@ -31543,6 +31599,7 @@ async fn workspace_response(
|
||||
manage_secrets: is_owner,
|
||||
manage_runtimes: is_owner,
|
||||
delete_workspace: is_owner,
|
||||
delete_subjects: is_owner,
|
||||
},
|
||||
extension_points: WorkspaceExtensionPoints {
|
||||
store: "sqlite".to_string(),
|
||||
@@ -37777,6 +37834,7 @@ mod tests {
|
||||
mod restore_workdir_tests;
|
||||
mod restored_parent_control_tests;
|
||||
mod subject_authorization_tests;
|
||||
mod subject_deletion_tests;
|
||||
mod subject_spawn_tests;
|
||||
mod subjektiv_jobs_tests;
|
||||
mod subjektiv_journal_tests;
|
||||
@@ -46241,7 +46299,13 @@ mod tests {
|
||||
let retired = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("retired").unwrap())
|
||||
.unwrap();
|
||||
store.retire_subject(&retired.id).unwrap();
|
||||
// Frozen lifecycle fixture: migration preserves old inactive Subjects; there is
|
||||
// intentionally no public operation that creates this historical state anymore.
|
||||
api.feature_storage.open(&api.subjektiv_registration).unwrap()
|
||||
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||
tx.execute("UPDATE subjects SET state='retired', record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&retired.id])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
|
||||
let request = |subject_id: Option<&str>, profile: &str, singleton_key: Option<&str>| {
|
||||
CreateWorkspaceWorkerRequest {
|
||||
@@ -48843,6 +48907,7 @@ mod tests {
|
||||
>,
|
||||
materializer: worker_runtime::working_directory::RuntimeGitMaterializer,
|
||||
spawn_failure: std::sync::Mutex<Option<String>>,
|
||||
subject_delete_failure: std::sync::Mutex<Option<String>>,
|
||||
recovery_failure: std::sync::Mutex<Option<String>>,
|
||||
recovery_requests: std::sync::Mutex<Vec<runtime_api::SubjectCreateRecoveryRequest>>,
|
||||
recovery_hook: std::sync::Mutex<Option<Box<dyn FnOnce() + Send>>>,
|
||||
@@ -48887,6 +48952,7 @@ mod tests {
|
||||
std::env::temp_dir().join(unique),
|
||||
),
|
||||
spawn_failure: std::sync::Mutex::new(None),
|
||||
subject_delete_failure: std::sync::Mutex::new(None),
|
||||
recovery_failure: std::sync::Mutex::new(None),
|
||||
recovery_requests: std::sync::Mutex::new(Vec::new()),
|
||||
recovery_hook: std::sync::Mutex::new(None),
|
||||
@@ -49242,6 +49308,24 @@ mod tests {
|
||||
.map(|execution| execution.snapshot())
|
||||
}
|
||||
|
||||
fn preflight_subject_delete_worker(
|
||||
&self,
|
||||
_request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> std::result::Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn subject_delete_worker(
|
||||
&self,
|
||||
request: &runtime_api::SubjectDeleteWorkerRequest,
|
||||
) -> std::result::Result<(), String> {
|
||||
if let Some(message) = self.subject_delete_failure.lock().unwrap().clone() {
|
||||
return Err(message);
|
||||
}
|
||||
self.stop_worker(&worker_runtime::identity::WorkerRef::new(request.worker_id));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stop_worker(
|
||||
&self,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
|
||||
@@ -149,6 +149,13 @@ pub(super) fn worker_permissions(
|
||||
api: &WorkspaceApi,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ApiResult<Option<Vec<ScopedPermission>>> {
|
||||
if subjektiv_deletion::deleting_worker(api, worker)?
|
||||
|| open_subjektiv_store(api)?
|
||||
.worker_subject_deleted(&worker.runtime_id, &worker.worker_id)
|
||||
.map_err(subjektiv_store_error)?
|
||||
{
|
||||
return Err(denied("Worker belonged to an explicitly deleted Subject"));
|
||||
}
|
||||
if let Some(grant) = live_child_grant(api, worker)? {
|
||||
return Ok(Some(grant.permissions));
|
||||
}
|
||||
@@ -226,6 +233,8 @@ pub(super) fn prepare_delegation(
|
||||
singleton_key: Option<&str>,
|
||||
subject_connection: bool,
|
||||
) -> ApiResult<Option<WorkerExecutionGrant>> {
|
||||
// A deleted grant must never become an ordinary non-Subject delegation source.
|
||||
worker_permissions(api, source)?;
|
||||
let child = live_child_grant(api, source)?;
|
||||
let body = if child.is_none() {
|
||||
current_body_subject(api, source)?
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
//! Explicit Subject destruction is independent of ordinary Worker retention. The domain
|
||||
//! fence commits before external effects, remains after failures, and rejects cached grants.
|
||||
//! Worker originals are archived; Job execution copies are purged. Existing archives, Workdirs,
|
||||
//! Tickets and other Subjects are not erased. Archive contents can include previously injected
|
||||
//! Memory: their independent retention/access policy still applies. This is logical content
|
||||
//! deletion, not a promise to erase backups, external providers or already exported material.
|
||||
//!
|
||||
//! Child removal/create-fence receipts, not acceptance, Idle or missing projections, prove
|
||||
//! quiescence. A retry carries the same scope/Subject/operation; no new child identity is issued.
|
||||
//! The domain receipt is authoritative completion. Backend inventories retain only identifiers
|
||||
//! needed for cleanup and fail-closed authorization, never a copy of Subject content.
|
||||
use super::*;
|
||||
use rusqlite::params;
|
||||
|
||||
pub(super) fn job_subject(request: &BackendJobRequest) -> Option<&str> {
|
||||
request
|
||||
.grants
|
||||
.subjektiv_journal
|
||||
.as_ref()
|
||||
.map(|g| g.subject_id.as_str())
|
||||
.or_else(|| {
|
||||
request
|
||||
.grants
|
||||
.subjektiv_consolidation
|
||||
.as_ref()
|
||||
.map(|g| g.subject_id.as_str())
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn require_available(api: &WorkspaceApi, subject: &str) -> Result<()> {
|
||||
let store = open_subjektiv_store(api).map_err(|error| error.error)?;
|
||||
if store
|
||||
.subject(subject)
|
||||
.map_err(|e| Error::Store(e.to_string()))?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::SubjektivSubjectNotFound(subject.into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn pending(api: &WorkspaceApi, subject: &str) -> Result<bool> {
|
||||
api.config_store.with_conn(|connection| {
|
||||
Ok(connection.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND subject_id=?2)",
|
||||
params![api.workspace_id(), subject], |row| row.get(0),
|
||||
)?)
|
||||
})
|
||||
}
|
||||
|
||||
/// This is internal cleanup evidence, never a user-supplied removal flag. It exists only
|
||||
/// after the independently durable domain fence and exact-target inventory commit.
|
||||
pub(super) fn deleting_worker(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Result<bool> {
|
||||
api.config_store.with_conn(|connection| {
|
||||
Ok(connection.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM subject_deletion_workers WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3)",
|
||||
params![api.workspace_id(), worker.runtime_id, worker.worker_id], |row| row.get(0),
|
||||
)?)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn delete(
|
||||
api: WorkspaceApi,
|
||||
workspace: String,
|
||||
subject: String,
|
||||
request: server_api::SubjektivSubjectDeleteRequest,
|
||||
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
api.host_executors
|
||||
.host()?
|
||||
.run(move || {
|
||||
validate_workspace_scope(&api, &workspace)?;
|
||||
handle.block_on(resume(&api, &subject, &request.operation_id))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn recover(api: &WorkspaceApi) -> Result<()> {
|
||||
let pending = open_subjektiv_store(api)
|
||||
.map_err(|error| error.error)?
|
||||
.pending_subject_deletions()
|
||||
.map_err(|e| Error::Store(e.to_string()))?;
|
||||
for deletion in pending {
|
||||
if let Err(error) = resume(api, &deletion.subject_id, &deletion.operation_id).await {
|
||||
tracing::warn!(%error.error, subject_id=%deletion.subject_id,
|
||||
"Subject deletion restart recovery remains incomplete; same request can resume");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resume(
|
||||
api: &WorkspaceApi,
|
||||
subject: &str,
|
||||
operation: &str,
|
||||
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
|
||||
let store = open_subjektiv_store(api)?;
|
||||
let gate = subjektiv_journal::lifecycle_lock(api, subject)?;
|
||||
let receipt = {
|
||||
let _guard = gate
|
||||
.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
|
||||
let receipt = store
|
||||
.begin_subject_deletion(subject, operation)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
if !receipt.completed {
|
||||
let delegated = store
|
||||
.deletion_subject_workers(subject)
|
||||
.map_err(subjektiv_store_error)?
|
||||
.into_iter()
|
||||
.map(|(runtime, worker)| RuntimeWorkerRef::new(runtime, worker))
|
||||
.collect::<Vec<_>>();
|
||||
api.config_store.prepare_subject_deletion(
|
||||
api.workspace_id(),
|
||||
subject,
|
||||
operation,
|
||||
&delegated,
|
||||
)?;
|
||||
}
|
||||
receipt
|
||||
};
|
||||
let response = |completed| server_api::SubjektivSubjectDeleteResponse {
|
||||
subject_id: subject.into(),
|
||||
operation_id: operation.into(),
|
||||
completed,
|
||||
};
|
||||
if receipt.completed {
|
||||
return Ok(response(true));
|
||||
}
|
||||
// Never hold the Host callback gate across a Runtime call. A create/restore can be
|
||||
// waiting for the callback that now has to reject this permanently fenced Subject.
|
||||
let workers =
|
||||
api.config_store
|
||||
.list_subject_deletion_workers(api.workspace_id(), subject, operation)?;
|
||||
for (worker, _is_job) in workers {
|
||||
if let Err(error) = remove_worker(api, subject, operation, &worker).await {
|
||||
tracing::warn!(%error.error, subject_id=subject, operation_id=operation, "Subject deletion remains incomplete; exact retry required");
|
||||
return Ok(response(false));
|
||||
}
|
||||
}
|
||||
{
|
||||
let _guard = gate
|
||||
.lock()
|
||||
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
|
||||
api.config_store
|
||||
.erase_subject_deleted_jobs(api.workspace_id(), subject, operation)?;
|
||||
// Finish Backend cleanup before the authoritative domain receipt. If domain
|
||||
// purge fails, its separately committed fence and pending operation survive.
|
||||
// No retry can report domain completion while occupancy/Job cleanup is unproven.
|
||||
api.config_store
|
||||
.finish_subject_deletion_record(api.workspace_id(), subject, operation)?;
|
||||
store
|
||||
.finish_subject_deletion(subject, operation)
|
||||
.map_err(subjektiv_store_error)?;
|
||||
}
|
||||
Ok(response(true))
|
||||
}
|
||||
|
||||
async fn remove_worker(
|
||||
api: &WorkspaceApi,
|
||||
subject: &str,
|
||||
operation: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ApiResult<()> {
|
||||
let request = api.config_store.subject_deletion_worker_request(
|
||||
api.workspace_id(),
|
||||
subject,
|
||||
operation,
|
||||
worker,
|
||||
)?;
|
||||
let fence = api
|
||||
.runtime
|
||||
.subject_delete_worker(&worker.runtime_id, request.clone())
|
||||
.map_err(RuntimeRegistryError::into_error)?;
|
||||
if fence.workspace_id != request.workspace_id
|
||||
|| fence.worker_id != request.worker_id
|
||||
|| fence.create_fingerprint != request.create_fingerprint
|
||||
|| fence.operation_id != request.operation_id
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Subject Worker deletion fence receipt mismatch".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
// Replay the canonical child even when physical removal already happened:
|
||||
// its retained receipt can finish the Backend catalog midpoint. The Runtime
|
||||
// fence above also prevents an older create from arriving after that removal.
|
||||
if api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(api.workspace_id(), worker)
|
||||
.map_err(|error| Error::Store(error.to_string()))?
|
||||
.is_some()
|
||||
{
|
||||
return execute_removal(api, operation, worker).await;
|
||||
}
|
||||
match api.runtime.worker(worker) {
|
||||
Ok(detail) => {
|
||||
if !detail.state.eq_ignore_ascii_case("stopped") {
|
||||
let stopped = api
|
||||
.runtime
|
||||
.stop_worker(
|
||||
worker,
|
||||
WorkerLifecycleRequest {
|
||||
reason: Some(format!("Explicit Subject deletion {operation}")),
|
||||
ticket_assignment: None,
|
||||
},
|
||||
)
|
||||
.map_err(RuntimeRegistryError::into_error)?;
|
||||
if stopped.state != InternalWorkerOperationState::Accepted {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Subject Worker stop was not accepted".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
let actual = api
|
||||
.runtime
|
||||
.worker(worker)
|
||||
.map_err(RuntimeRegistryError::into_error)?;
|
||||
if !actual.state.eq_ignore_ascii_case("stopped") {
|
||||
return Err(
|
||||
Error::RepositoryConflict("Subject Worker stop is unconfirmed".into()).into(),
|
||||
);
|
||||
}
|
||||
if api
|
||||
.store
|
||||
.get_worker_registry(api.workspace_id(), worker)?
|
||||
.is_none()
|
||||
{
|
||||
// The Runtime fence authenticated this exact reserved fingerprint. A create
|
||||
// can have reached Runtime before its Backend publication failed. Publish
|
||||
// only its stopped record so the normal archive/Purge owner can remove it.
|
||||
record_worker_summary(
|
||||
api,
|
||||
&actual,
|
||||
&actual.display_name,
|
||||
actual.profile.clone(),
|
||||
WorkerRegistryDisplayNamePolicy::PreserveExisting,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Err(RuntimeRegistryError::UnknownWorker { .. }) => {
|
||||
if api
|
||||
.store
|
||||
.get_worker_registry(api.workspace_id(), worker)?
|
||||
.is_some()
|
||||
{
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Runtime object missing before canonical original archival".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
api.config_store.confirm_subject_deletion_absence(
|
||||
api.workspace_id(),
|
||||
subject,
|
||||
operation,
|
||||
worker,
|
||||
&fence,
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) => return Err(error.into_error().into()),
|
||||
}
|
||||
execute_removal(api, operation, worker).await
|
||||
}
|
||||
|
||||
async fn execute_removal(
|
||||
api: &WorkspaceApi,
|
||||
operation: &str,
|
||||
worker: &RuntimeWorkerRef,
|
||||
) -> ApiResult<()> {
|
||||
let response = WorkerRemovalService::new(api)
|
||||
.execute_target_removal_on_host(
|
||||
api.runtime.as_ref(),
|
||||
worker,
|
||||
&format!("Explicit Subject deletion {operation}"),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::Store)?;
|
||||
if response.status != StatusCode::OK.as_u16() {
|
||||
return Err(Error::RepositoryConflict(
|
||||
"Subject Worker archival/Purge remains unconfirmed".into(),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -446,6 +446,7 @@ pub(super) fn bind_host_session(
|
||||
/// Runs under the same Subject gate as final revoke/singleton release. A failed create or
|
||||
/// legacy removal may have released its Backend reservation but cannot hide a durable writer.
|
||||
pub(super) fn check_start_allowed(api: &WorkspaceApi, subject: &str) -> Result<()> {
|
||||
subjektiv_deletion::require_available(api, subject)?;
|
||||
let store = store(api)?;
|
||||
let key = crate::subjektiv::subject_worker_singleton_key(subject)
|
||||
.map_err(|e| Error::InvalidInput(e.to_string()))?;
|
||||
@@ -1747,6 +1748,13 @@ pub(super) fn prepare_worker_removal(
|
||||
runtime: &RuntimeRegistry,
|
||||
target: &RuntimeWorkerRef,
|
||||
) -> Result<Option<crate::retention::ErasedJournalSession>> {
|
||||
if let Some(api) = service.journal_dispatch_api.upgrade()
|
||||
&& subjektiv_deletion::deleting_worker(&api, target)?
|
||||
{
|
||||
// Full Subject deletion has a permanent domain fence. It deliberately drops
|
||||
// Journal continuity, but retains source originals through its archive policy.
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(attr) = removal_attribution(service, target)? else {
|
||||
if service
|
||||
.store
|
||||
@@ -1809,6 +1817,13 @@ pub(super) fn erased_archive_removal_proof(
|
||||
target: &RuntimeWorkerRef,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
) -> Result<Option<crate::retention::RuntimeJournalErasureAck>> {
|
||||
if let Some(api) = service.journal_dispatch_api.upgrade()
|
||||
&& subjektiv_deletion::deleting_worker(&api, target)?
|
||||
{
|
||||
// Resume the exact canonical retention request after full Subject fencing.
|
||||
// Subject deletion retains originals; it is not per-Session source erasure.
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(attr) = removal_attribution(service, target)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -1852,6 +1867,13 @@ pub(super) fn validate_removal_retention(
|
||||
target: &RuntimeWorkerRef,
|
||||
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||
) -> Result<()> {
|
||||
if let Some(api) = service.journal_dispatch_api.upgrade()
|
||||
&& subjektiv_deletion::deleting_worker(&api, target)?
|
||||
{
|
||||
// The plan-local retention policy and canonical Runtime receipt remain required.
|
||||
// Neither unfinished Journals nor a stopped old Controller are deletion blockers.
|
||||
return Ok(());
|
||||
}
|
||||
let Some(attr) = removal_attribution(service, target)? else {
|
||||
if service
|
||||
.store
|
||||
@@ -2066,6 +2088,9 @@ pub(super) fn session_erased(api: &WorkspaceApi, request: &BackendJobRequest) ->
|
||||
let Some(grant) = &request.grants.subjektiv_journal else {
|
||||
return Ok(false);
|
||||
};
|
||||
if subjektiv_deletion::pending(api, &grant.subject_id)? {
|
||||
return Ok(true);
|
||||
}
|
||||
let store = store(api)?;
|
||||
store
|
||||
.journal_session_erased(&host(api, &store, grant)?)
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
//! Real router/authentication, feature store and Backend DB boundaries. Runtime
|
||||
//! execution/unknown and restart fences are covered in worker-runtime and the
|
||||
//! canonical removal suites, rather than claiming HTTP acceptance proves shutdown.
|
||||
use super::*;
|
||||
|
||||
async fn delete_request(
|
||||
api: &WorkspaceApi,
|
||||
subject: &str,
|
||||
key: &str,
|
||||
token: Option<&str>,
|
||||
workspace: &str,
|
||||
) -> (StatusCode, Value) {
|
||||
let mut builder = Request::builder()
|
||||
.method("DELETE")
|
||||
.uri(format!("/api/w/{workspace}/subjektiv/subjects/{subject}"))
|
||||
.header(CONTENT_TYPE, "application/json");
|
||||
if let Some(token) = token {
|
||||
builder = builder.header("authorization", format!("Bearer {token}"));
|
||||
}
|
||||
let response = build_router(api.clone())
|
||||
.oneshot(
|
||||
builder
|
||||
.body(Body::from(json!({"operation_id":key}).to_string()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let bytes = to_bytes(response.into_body(), 2 * 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
let body = serde_json::from_slice(&bytes)
|
||||
.unwrap_or_else(|_| json!({"text":String::from_utf8_lossy(&bytes)}));
|
||||
(status, body)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deletion_owner_scope_exact_retry_and_subject_isolation() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let api = test_api(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("delete me").unwrap(),
|
||||
"private behavior".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let other = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("keep me").unwrap(),
|
||||
"unchanged".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-owner").await;
|
||||
let stranger = seed_test_api_token(api.store.as_ref(), "subject-delete-stranger");
|
||||
for (token, workspace, expected) in [
|
||||
(None, TEST_WORKSPACE_ID, StatusCode::UNAUTHORIZED),
|
||||
(
|
||||
Some(stranger.as_str()),
|
||||
TEST_WORKSPACE_ID,
|
||||
StatusCode::FORBIDDEN,
|
||||
),
|
||||
(
|
||||
Some(owner.as_str()),
|
||||
"another-workspace",
|
||||
StatusCode::NOT_FOUND,
|
||||
),
|
||||
] {
|
||||
let (status, body) =
|
||||
delete_request(&api, &subject.id, "same-operation", token, workspace).await;
|
||||
assert_eq!(status, expected, "{body}");
|
||||
assert!(store.subject(&subject.id).unwrap().is_some());
|
||||
}
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"same-operation",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||
assert_eq!(
|
||||
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||
"unchanged"
|
||||
);
|
||||
assert_eq!(store.list_subjects(100).unwrap().items.len(), 1);
|
||||
let (status, replay) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"same-operation",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{replay}");
|
||||
assert_eq!(replay, body);
|
||||
let (status, _) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"different-operation",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::CONFLICT);
|
||||
let (status, _) = delete_request(
|
||||
&api,
|
||||
&other.id,
|
||||
"same-operation",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::CONFLICT);
|
||||
assert!(store.subject(&other.id).unwrap().is_some());
|
||||
let fresh = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("fresh").unwrap())
|
||||
.unwrap();
|
||||
assert_ne!(fresh.id, subject.id);
|
||||
assert!(store.subject_permissions(&fresh.id).unwrap().is_none());
|
||||
assert_eq!(
|
||||
store
|
||||
.pending_staging_candidates(&fresh.id, 100)
|
||||
.unwrap()
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_inactive_subject_is_preserved_until_explicit_delete_and_retire_route_is_gone() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let api = test_api(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("legacy fixture").unwrap(),
|
||||
"retained original".into(),
|
||||
)
|
||||
.unwrap();
|
||||
api.feature_storage.open(&api.subjektiv_registration).unwrap()
|
||||
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||
tx.execute("UPDATE subjects SET state='retired',record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&subject.id])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert_eq!(
|
||||
store.subject(&subject.id).unwrap().unwrap().state,
|
||||
crate::subjektiv::SubjectState::LegacyRetired
|
||||
);
|
||||
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-legacy").await;
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!(
|
||||
"/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/retire",
|
||||
subject.id
|
||||
))
|
||||
.header("authorization", format!("Bearer {owner}"))
|
||||
.body(Body::empty())
|
||||
.unwrap();
|
||||
let old = build_router(api.clone()).oneshot(request).await.unwrap();
|
||||
assert_eq!(old.status(), StatusCode::METHOD_NOT_ALLOWED);
|
||||
assert_eq!(
|
||||
store.subject(&subject.id).unwrap().unwrap().behavior_md,
|
||||
"retained original"
|
||||
);
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"explicit-legacy-delete",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn unconfirmed_worker_shutdown_is_not_completion_and_exact_retry_converges() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(dir.path());
|
||||
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("delete connected body").unwrap(),
|
||||
"private behavior".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let Json(created) = create_workspace_worker(
|
||||
State(api.clone()),
|
||||
HeaderMap::new(),
|
||||
Json(CreateWorkspaceWorkerRequest {
|
||||
runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(),
|
||||
display_name: "Subject deletion boundary".into(),
|
||||
singleton_key: None,
|
||||
profile: Some("builtin:companion".into()),
|
||||
ticket_assignment: None,
|
||||
initial_submit: vec![],
|
||||
workdir_attachments: vec![],
|
||||
feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest {
|
||||
subjektiv: Some(server_api::SubjektivWorkerConnectionRequest {
|
||||
subject_id: subject.id.clone(),
|
||||
}),
|
||||
},
|
||||
control_operation_id: None,
|
||||
execution_grant: None,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id);
|
||||
seed_worker_session_for_cleanup(dir.path(), &worker);
|
||||
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unknown").await;
|
||||
*execution.subject_delete_failure.lock().unwrap() = Some("shutdown completion unknown".into());
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"unknown-delete",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], false);
|
||||
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||
assert!(
|
||||
!store
|
||||
.subject_deletion(&subject.id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.completed
|
||||
);
|
||||
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
*execution.subject_delete_failure.lock().unwrap() = None;
|
||||
api.config_store.with_conn(|conn| {
|
||||
conn.execute_batch("CREATE TRIGGER injected_worker_catalog_failure BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected catalog failure'); END;")?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
let (status, midpoint) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"unknown-delete",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{midpoint}");
|
||||
assert_eq!(midpoint["completed"], false);
|
||||
assert!(matches!(
|
||||
api.runtime.worker(&worker),
|
||||
Err(RuntimeRegistryError::UnknownWorker { .. })
|
||||
));
|
||||
assert!(
|
||||
api.store
|
||||
.get_worker_registry(TEST_WORKSPACE_ID, &worker)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
api.config_store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER injected_worker_catalog_failure;")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"unknown-delete",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true, "{body}");
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let removal = api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
removal.plan.state,
|
||||
crate::retention::WorkerRemovalPlanState::Succeeded
|
||||
);
|
||||
assert_eq!(
|
||||
removal.runtime_request.session_disposition,
|
||||
worker_runtime::retention::SessionDisposition::Archive
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.subject_deletion(&subject.id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.completed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn captured_subject_job_result_cannot_deliver_after_domain_fence() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("notification fence").unwrap())
|
||||
.unwrap();
|
||||
let target = spawn_ticket_check_source(&api, "subject-notification-target");
|
||||
let worker = spawn_ticket_check_source(&api, "subject-notification-result");
|
||||
let request = BackendJobRequest {
|
||||
job_id: "captured-subject-notification".into(),
|
||||
purpose: "subject_notification_contract".into(),
|
||||
input_ref: "subject:notification".into(),
|
||||
input: json!({"private":"input"}),
|
||||
instruction: "Read input".into(),
|
||||
profile: "builtin:backend-job".into(),
|
||||
grants: crate::backend_job::BackendJobGrants {
|
||||
subjektiv_consolidation: Some(crate::backend_job::SubjektivConsolidationGrant {
|
||||
subject_id: subject.id.clone(),
|
||||
candidate_ids: vec!["candidate".into()],
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
serialization_key: None,
|
||||
source_worker: Some(target.clone()),
|
||||
notification_target: Some(target),
|
||||
limits: Default::default(),
|
||||
};
|
||||
// Enter at the committed result boundary: execution/result validation is
|
||||
// outside this notification test, but routing and Runtime transport are real.
|
||||
let now = Utc::now().to_rfc3339();
|
||||
let reserved = api
|
||||
.store
|
||||
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &now)
|
||||
.unwrap();
|
||||
let attempt = &reserved.attempt.attempt_id;
|
||||
api.store
|
||||
.claim_backend_job_attempt_dispatch(TEST_WORKSPACE_ID, &request.job_id, attempt, &now)
|
||||
.unwrap();
|
||||
api.store
|
||||
.bind_backend_job_attempt_worker(
|
||||
TEST_WORKSPACE_ID,
|
||||
&request.job_id,
|
||||
attempt,
|
||||
&worker,
|
||||
Some("run"),
|
||||
&now,
|
||||
)
|
||||
.unwrap();
|
||||
let captured = api
|
||||
.store
|
||||
.accept_backend_job_result(
|
||||
TEST_WORKSPACE_ID,
|
||||
&worker,
|
||||
&BackendJobResultSubmission {
|
||||
job_id: request.job_id.clone(),
|
||||
attempt_id: attempt.clone(),
|
||||
input_digest: request.input_digest().unwrap(),
|
||||
result: json!({"private":"result"}),
|
||||
},
|
||||
&now,
|
||||
)
|
||||
.unwrap();
|
||||
let delivery = crate::backend_job::delivery_id(&request.job_id, attempt);
|
||||
assert!(
|
||||
api.store
|
||||
.job_notification_source(TEST_WORKSPACE_ID, &request.job_id, attempt)
|
||||
.is_ok()
|
||||
);
|
||||
execution.take_inputs();
|
||||
store
|
||||
.begin_subject_deletion(&subject.id, "notification-delete")
|
||||
.unwrap();
|
||||
notification_delivery::deliver_job_result(&api, &captured);
|
||||
notification_delivery::recover_job_notifications(&api);
|
||||
assert!(
|
||||
execution.take_inputs().is_empty(),
|
||||
"captured or recovered result must not escape the domain fence"
|
||||
);
|
||||
assert!(
|
||||
api.store
|
||||
.job_notification_request(TEST_WORKSPACE_ID, &delivery)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn domain_purge_failure_keeps_fence_and_backend_reopen_resumes_exact_operation() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let api = test_api(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("retry purge").unwrap(),
|
||||
"do not resurrect".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let other = store
|
||||
.create_subject_with_behavior(
|
||||
crate::subjektiv::SubjectRole::new("untouched").unwrap(),
|
||||
"keep".into(),
|
||||
)
|
||||
.unwrap();
|
||||
let database = api
|
||||
.feature_storage
|
||||
.open(&api.subjektiv_registration)
|
||||
.unwrap();
|
||||
database.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||
tx.execute_batch("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected domain purge failure'); END;")?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
let owner =
|
||||
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-purge-failure").await;
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"purge-retry",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert!(status.is_server_error(), "{status}: {body}");
|
||||
assert!(
|
||||
!store
|
||||
.subject_deletion(&subject.id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.completed
|
||||
);
|
||||
assert!(store.subject(&subject.id).unwrap().is_none());
|
||||
assert_eq!(
|
||||
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||
"keep"
|
||||
);
|
||||
database
|
||||
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
|
||||
tx.execute_batch("DROP TRIGGER injected_subject_purge_failure;")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let mut reopened = api.clone();
|
||||
let config_store =
|
||||
Arc::new(SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap());
|
||||
reopened.config_store = config_store.clone();
|
||||
reopened.store = config_store;
|
||||
let recovery_api = reopened.clone();
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
reopened
|
||||
.host_executors
|
||||
.host()
|
||||
.unwrap()
|
||||
.run(move || handle.block_on(subjektiv_deletion::recover(&recovery_api)))
|
||||
.await
|
||||
.unwrap();
|
||||
let recovered = open_subjektiv_store(&reopened)
|
||||
.unwrap()
|
||||
.subject_deletion(&subject.id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(recovered.operation_id, "purge-retry");
|
||||
assert!(recovered.completed);
|
||||
let (status, body) = delete_request(
|
||||
&reopened,
|
||||
&subject.id,
|
||||
"purge-retry",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
assert_eq!(
|
||||
store.subject(&other.id).unwrap().unwrap().behavior_md,
|
||||
"keep"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn pending_consolidation_is_purged_and_captured_dispatch_and_late_result_are_rejected() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let subject = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("delete running Job").unwrap())
|
||||
.unwrap();
|
||||
let candidate: memory::extract::StagingRecord = serde_json::from_value(json!({
|
||||
"schema_version": 2, "id": "delete-job-candidate", "extract_run_id": "extract",
|
||||
"source": {"segment_id":"segment", "range":[1,2]}, "kind":"constraint",
|
||||
"claim":"private candidate marker", "why_useful":"retained only until explicit erasure",
|
||||
"evidence": [], "source_refs": []
|
||||
}))
|
||||
.unwrap();
|
||||
store
|
||||
.stage_candidate(crate::subjektiv::SubjectStagingRecord::attach(
|
||||
&subject.id,
|
||||
candidate,
|
||||
))
|
||||
.unwrap();
|
||||
let started = start_subjektiv_staging_consolidation(
|
||||
api.clone(),
|
||||
&subject.id,
|
||||
MemoryConsolidateStagingOperation { force: true },
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(started.status, "started");
|
||||
let job = api
|
||||
.store
|
||||
.find_active_backend_job_for_resource(
|
||||
TEST_WORKSPACE_ID,
|
||||
&crate::backend_job::subjektiv_serialization_key(&subject.id),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let attempt = api
|
||||
.store
|
||||
.get_backend_job_attempt(
|
||||
TEST_WORKSPACE_ID,
|
||||
&job.request.job_id,
|
||||
&crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let worker = attempt.worker.clone().unwrap();
|
||||
seed_worker_session_for_cleanup(dir.path(), &worker);
|
||||
let captured = BackendJobReservation {
|
||||
job: job.clone(),
|
||||
attempt: attempt.clone(),
|
||||
replayed: true,
|
||||
resource_reused: false,
|
||||
};
|
||||
let owner =
|
||||
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-consolidation").await;
|
||||
*execution.subject_delete_failure.lock().unwrap() = Some("completion unknown".into());
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"delete-job",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], false);
|
||||
assert!(
|
||||
api.dispatch_backend_job_reservation(captured.clone())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
api.accept_backend_job_result(
|
||||
&worker,
|
||||
&BackendJobResultSubmission {
|
||||
job_id: job.request.job_id.clone(),
|
||||
attempt_id: attempt.attempt_id.clone(),
|
||||
input_digest: job.input_digest.clone(),
|
||||
result: json!({"late":true}),
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
*execution.subject_delete_failure.lock().unwrap() = None;
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
"delete-job",
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
assert!(
|
||||
api.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
api.store
|
||||
.get_backend_job_attempt(TEST_WORKSPACE_ID, &job.request.job_id, &attempt.attempt_id)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(api.dispatch_backend_job_reservation(captured).is_err());
|
||||
assert!(api.dispatch_backend_job(&job.request).is_err());
|
||||
let removal = api
|
||||
.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
removal.plan.state,
|
||||
crate::retention::WorkerRemovalPlanState::Succeeded
|
||||
);
|
||||
assert_eq!(
|
||||
removal.runtime_request.session_disposition,
|
||||
worker_runtime::retention::SessionDisposition::Purge
|
||||
);
|
||||
assert_eq!(
|
||||
removal.runtime_request.diagnostics_disposition,
|
||||
worker_runtime::retention::DiagnosticsDisposition::Purge
|
||||
);
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join(".test-embedded-runtime-store/workers")
|
||||
.join(&worker.worker_id)
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unmaterialized_body_and_job_reservations_finish_only_with_runtime_storage_absence_proof() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (api, _) = test_api_with_recording_backend(dir.path()).await;
|
||||
let store = open_subjektiv_store(&api).unwrap();
|
||||
let owner =
|
||||
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unmaterialized").await;
|
||||
for is_job in [false, true] {
|
||||
let subject = store
|
||||
.create_subject(crate::subjektiv::SubjectRole::new("reserved before deletion").unwrap())
|
||||
.unwrap();
|
||||
let singleton = (!is_job).then(|| format!("subjektiv:{}", subject.id));
|
||||
let allocation = if is_job {
|
||||
let request = BackendJobRequest {
|
||||
job_id: "unmaterialized-subject-job".into(),
|
||||
purpose: "subject_reservation_contract".into(),
|
||||
input_ref: "subject:reserved".into(),
|
||||
input: json!({"private":"reserved input"}),
|
||||
instruction: "read".into(),
|
||||
profile: "builtin:backend-job".into(),
|
||||
grants: crate::backend_job::BackendJobGrants {
|
||||
subjektiv_consolidation: Some(
|
||||
crate::backend_job::SubjektivConsolidationGrant {
|
||||
subject_id: subject.id.clone(),
|
||||
candidate_ids: vec!["candidate".into()],
|
||||
},
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
serialization_key: None,
|
||||
source_worker: None,
|
||||
notification_target: None,
|
||||
limits: Default::default(),
|
||||
};
|
||||
let reserved = api
|
||||
.store
|
||||
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &Utc::now().to_rfc3339())
|
||||
.unwrap();
|
||||
crate::backend_job::allocation_key(&reserved.attempt.attempt_id)
|
||||
} else {
|
||||
"unmaterialized-subject-body".into()
|
||||
};
|
||||
let settings = api
|
||||
.config_store
|
||||
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
|
||||
.unwrap();
|
||||
let reserved = api
|
||||
.config_store
|
||||
.reserve_worker_create(
|
||||
TEST_WORKSPACE_ID,
|
||||
EMBEDDED_WORKER_RUNTIME_ID,
|
||||
&allocation,
|
||||
"unmaterialized-fingerprint",
|
||||
singleton.as_deref(),
|
||||
&settings,
|
||||
)
|
||||
.unwrap();
|
||||
let worker =
|
||||
RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, reserved.worker_id.to_string());
|
||||
assert!(matches!(
|
||||
api.runtime.worker(&worker),
|
||||
Err(RuntimeRegistryError::UnknownWorker { .. })
|
||||
));
|
||||
let (status, body) = delete_request(
|
||||
&api,
|
||||
&subject.id,
|
||||
&format!("delete-reserved-{is_job}"),
|
||||
Some(&owner),
|
||||
TEST_WORKSPACE_ID,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::OK, "{body}");
|
||||
assert_eq!(body["completed"], true);
|
||||
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
|
||||
let proved = api.config_store.with_conn(|conn| Ok(conn.query_row("SELECT originals_absent FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4", rusqlite::params![TEST_WORKSPACE_ID,subject.id,worker.runtime_id,worker.worker_id], |row|row.get::<_,bool>(0))?)).unwrap();
|
||||
assert!(proved);
|
||||
assert!(
|
||||
api.config_store
|
||||
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
|
||||
.unwrap()
|
||||
.is_none(),
|
||||
"absence is not a fabricated Purge/archive receipt"
|
||||
);
|
||||
assert!(
|
||||
api.config_store
|
||||
.reserve_worker_create(
|
||||
TEST_WORKSPACE_ID,
|
||||
EMBEDDED_WORKER_RUNTIME_ID,
|
||||
&allocation,
|
||||
"unmaterialized-fingerprint",
|
||||
singleton.as_deref(),
|
||||
&settings
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
if let Some(singleton) = singleton {
|
||||
assert!(
|
||||
api.store
|
||||
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1143,6 +1143,8 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
|
||||
&Utc::now().to_rfc3339(),
|
||||
)
|
||||
.unwrap();
|
||||
// Discard only the original delivery, before recovery can automatically retry.
|
||||
f.execution.take_inputs();
|
||||
f.api.recover_backend_job_worker_cleanups().unwrap();
|
||||
wait_for_backend_job_worker_cleanup(
|
||||
&f.api,
|
||||
@@ -1151,23 +1153,64 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
|
||||
BackendJobWorkerCleanupState::Completed,
|
||||
)
|
||||
.await;
|
||||
// Cleanup commits before domain reconciliation; wait for its authoritative outcome.
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
// Cleanup commits before domain reconciliation. Job Failed is only transient:
|
||||
// reconcile_unknown fences this attempt, then automatically reserves/queues its
|
||||
// retry. Observe the immutable attempt outcome and the delivered new generation,
|
||||
// not the Job's transient state; a manual retry would race the automatic one.
|
||||
let reconciliation = tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
loop {
|
||||
let job = f
|
||||
.api
|
||||
.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
if job.state == BackendJobState::Failed {
|
||||
break;
|
||||
let original = f.api.store.get_backend_job_attempt(
|
||||
TEST_WORKSPACE_ID,
|
||||
&f.request.job_id,
|
||||
&dispatched.attempt.attempt_id,
|
||||
).unwrap().unwrap();
|
||||
if original.state == BackendJobAttemptState::Failed {
|
||||
assert_eq!(original.failure_category.as_deref(), Some("journal_attempt_fenced"));
|
||||
assert_eq!(original.worker_cleanup_state, Some(BackendJobWorkerCleanupState::Completed));
|
||||
assert!(original.worker_cleanup_completed_at.is_some());
|
||||
if let Some(generation) = f.store.active_journal_generation(&f.host, &f.id).unwrap()
|
||||
&& generation.job.attempt_id != original.attempt_id
|
||||
&& let Some(retry) = f.api.store.get_backend_job_attempt(
|
||||
TEST_WORKSPACE_ID,
|
||||
&f.request.job_id,
|
||||
&generation.job.attempt_id,
|
||||
).unwrap()
|
||||
&& retry.state == BackendJobAttemptState::Dispatched
|
||||
&& let Some(resumed) = &retry.worker
|
||||
// Worker binding precedes input delivery/attestation. ReadState
|
||||
// succeeds only once the new generation's delivery is attested.
|
||||
&& subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok()
|
||||
{
|
||||
break (original, retry, generation);
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
.await;
|
||||
assert!(
|
||||
reconciliation.is_ok(),
|
||||
"post-cleanup reconciliation timed out: job={:?}, original_attempt={:?}, active_generation={:?}",
|
||||
f.api
|
||||
.store
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
|
||||
.unwrap(),
|
||||
f.api
|
||||
.store
|
||||
.get_backend_job_attempt(
|
||||
TEST_WORKSPACE_ID,
|
||||
&f.request.job_id,
|
||||
&dispatched.attempt.attempt_id
|
||||
)
|
||||
.unwrap(),
|
||||
f.store.active_journal_generation(&f.host, &f.id).unwrap(),
|
||||
);
|
||||
let (fenced, retry, generation) = reconciliation.unwrap();
|
||||
assert_eq!(fenced.worker, dispatched.attempt.worker);
|
||||
assert_eq!(retry.attempt, dispatched.attempt.attempt + 1);
|
||||
assert_eq!(generation.job.job_id, f.request.job_id);
|
||||
assert_eq!(generation.job.attempt_id, retry.attempt_id);
|
||||
assert_eq!(retry.input_digest, f.request.input_digest().unwrap());
|
||||
f.assert_purged(worker);
|
||||
assert!(f.store.journal_retention_required(&f.host).unwrap());
|
||||
assert_eq!(
|
||||
@@ -1187,25 +1230,25 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
|
||||
1,
|
||||
"source pin survives failed/partial Job cleanup"
|
||||
);
|
||||
f.execution.take_inputs();
|
||||
let retry = f
|
||||
let job = f
|
||||
.api
|
||||
.store
|
||||
.reserve_backend_job_retry(
|
||||
TEST_WORKSPACE_ID,
|
||||
&f.request.job_id,
|
||||
&Utc::now().to_rfc3339(),
|
||||
)
|
||||
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(retry.job.request.input, f.request.input);
|
||||
assert_eq!(retry.job.input_digest, f.request.input_digest().unwrap());
|
||||
let retry = f.api.dispatch_backend_job_reservation(retry).unwrap();
|
||||
let resumed = retry.attempt.worker.as_ref().unwrap();
|
||||
assert_eq!(job.state, BackendJobState::Pending);
|
||||
assert_eq!(job.current_attempt, retry.attempt);
|
||||
assert_eq!(
|
||||
job.load_executable_request().unwrap().input,
|
||||
f.request.input
|
||||
);
|
||||
assert_eq!(job.input_digest, f.request.input_digest().unwrap());
|
||||
let resumed = retry.worker.as_ref().unwrap();
|
||||
let delivered = f.execution.take_inputs();
|
||||
assert_eq!(delivered.len(), 1);
|
||||
assert_eq!(
|
||||
delivered[0].1,
|
||||
f.request.worker_input(&retry.attempt.attempt_id).unwrap()
|
||||
f.request.worker_input(&retry.attempt_id).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
f.store
|
||||
|
||||
@@ -92,6 +92,13 @@ use crate::backend_job::{
|
||||
use crate::workspace_deletion::WorkspaceDeletionStore;
|
||||
use crate::{Error, Result};
|
||||
|
||||
#[path = "subject_deletion_store.rs"]
|
||||
mod subject_deletion_store;
|
||||
pub(crate) use subject_deletion_store::{
|
||||
prior_removal_matches as subject_deletion_prior_removal_matches,
|
||||
worker_kind as subject_deletion_worker_kind,
|
||||
};
|
||||
|
||||
#[path = "activation.rs"]
|
||||
mod activation;
|
||||
pub use activation::ActivationCodeIssue;
|
||||
@@ -119,7 +126,7 @@ pub use ticket::{
|
||||
};
|
||||
|
||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 104;
|
||||
const LATEST_SCHEMA_VERSION: i64 = 105;
|
||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||
@@ -447,6 +454,11 @@ const MIGRATIONS: &[Migration] = &[
|
||||
name: "Converge Passkey results and Job input retention",
|
||||
apply: migrate_schema_103_variants_to_v104,
|
||||
},
|
||||
Migration {
|
||||
version: 105,
|
||||
name: "Durable Subject deletion snapshots and Job admission fences",
|
||||
apply: subject_deletion_store::migrate,
|
||||
},
|
||||
];
|
||||
|
||||
// Both schema-103 identities shipped. Keep the alternate implementation and accept
|
||||
@@ -3000,6 +3012,7 @@ impl SqliteWorkspaceStore {
|
||||
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
subject_deletion_store::require_worker_admission(&tx, workspace_id, allocation_key, singleton_key)?;
|
||||
let fixed_worker_id = prepare(&tx)?;
|
||||
let existing = tx
|
||||
.query_row(
|
||||
@@ -7455,6 +7468,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let deadline = backend_job_deadline(now, request.limits.timeout_seconds)?;
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, request)?;
|
||||
if let Some(job) = read_backend_job(&tx, workspace_id, &request.job_id)? {
|
||||
if job.intent_fingerprint != fingerprint {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
@@ -7532,7 +7546,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
return Ok(job);
|
||||
}
|
||||
if reason == JobInputRetirementReason::Completed {
|
||||
job.load_executable_request()?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))?
|
||||
.ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?;
|
||||
if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed
|
||||
@@ -7562,6 +7577,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let mut job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let previous_id = attempt_id(job_id, job.current_attempt);
|
||||
let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)?
|
||||
@@ -7701,6 +7717,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
||||
@@ -7764,6 +7781,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
|
||||
@@ -7827,6 +7845,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
|
||||
.ok_or_else(|| {
|
||||
@@ -7880,6 +7899,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{}`", submission.job_id)))?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)?
|
||||
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
|
||||
if submission.input_digest != job.input_digest
|
||||
|| attempt.input_digest != submission.input_digest
|
||||
@@ -7908,6 +7928,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
tx.commit()?;
|
||||
return Ok(BackendJobResultAcceptance { job, attempt, replayed: true });
|
||||
}
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
job.load_executable_request()?;
|
||||
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
|
||||
return Err(Error::InvalidInput(
|
||||
@@ -7972,6 +7993,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?;
|
||||
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
|
||||
let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?;
|
||||
// Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence.
|
||||
if &attempt != expected || job.current_attempt != attempt.attempt
|
||||
@@ -8277,6 +8299,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
) -> Result<(BackendJobDeliveryRecord, bool)> {
|
||||
self.with_conn_mut(|conn| {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let deleted:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_jobs WHERE workspace_id=?1 AND job_id=?2)",params![workspace_id,job_id],|r|r.get(0))?;
|
||||
if deleted {return Err(Error::RepositoryConflict("subject_deleted: Job delivery fenced".into()));}
|
||||
if let Some(existing) = read_backend_job_delivery(&tx, workspace_id, delivery_id)? {
|
||||
if existing.job_id != job_id || existing.target != *target {
|
||||
return Err(Error::InvalidInput(
|
||||
@@ -15545,6 +15569,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
||||
notification_routing::jobs::install(conn)?;
|
||||
notification_routing::install_business_events(conn)?;
|
||||
notification_routing::upgrade_result_reports(conn)?;
|
||||
subject_deletion_store::install_notification_erasure(conn)?;
|
||||
ticket::verify_sqlite_ticket_schema(conn).map_err(|error| {
|
||||
Error::Store(format!(
|
||||
"fresh Workspace Ticket schema verification failed: {error}"
|
||||
@@ -15583,10 +15608,17 @@ fn create_workspace_schema_structure_from_sql(
|
||||
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
|
||||
|
||||
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
|
||||
conn.execute_batch(&schema_sql.replace(
|
||||
"-- Ticket-owned assignment schema is composed by store.rs.",
|
||||
ticket::TICKET_ASSIGNMENT_SCHEMA_SQL,
|
||||
))?;
|
||||
conn.execute_batch(
|
||||
&schema_sql
|
||||
.replace(
|
||||
"-- Ticket-owned assignment schema is composed by store.rs.",
|
||||
ticket::TICKET_ASSIGNMENT_SCHEMA_SQL,
|
||||
)
|
||||
.replace(
|
||||
"-- Subject deletion schema is composed by store.rs.",
|
||||
include_str!("subject_deletion_schema.sql"),
|
||||
),
|
||||
)?;
|
||||
let composed_sql = r#"
|
||||
ALTER TABLE typed_tickets RENAME TO typed_tickets_legacy_v66;
|
||||
CREATE TABLE typed_tickets (
|
||||
@@ -15809,6 +15841,9 @@ fn verify_workspace_resource_constraints(conn: &Connection) -> Result<()> {
|
||||
if current_schema_version(conn)? >= 96 {
|
||||
notification_routing::jobs::verify(conn)?;
|
||||
}
|
||||
if current_schema_version(conn)? >= 105 {
|
||||
subject_deletion_store::verify(conn)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -16223,6 +16258,8 @@ fn current_worker_singleton_owner(
|
||||
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
||||
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
||||
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
||||
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r JOIN subject_deletion_workers d ON d.workspace_id=r.workspace_id AND d.runtime_id=r.runtime_id AND d.worker_id=r.worker_id WHERE r.workspace_id=owner.workspace_id AND r.runtime_id=owner.runtime_id AND r.worker_id=owner.worker_id AND r.state='succeeded' AND owner.singleton_key='subjektiv:'||d.subject_id AND (d.prior_removal_operation_id=r.operation_id OR (d.prior_removal_operation_id IS NULL AND d.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.metadata_disposition='tombstone'))) \
|
||||
AND NOT EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=owner.workspace_id AND d.runtime_id=owner.runtime_id AND d.worker_id=owner.worker_id AND owner.singleton_key='subjektiv:'||d.subject_id AND d.originals_absent=1) \
|
||||
THEN 'created' ELSE reservation.state END \
|
||||
FROM worker_singleton_owners owner \
|
||||
LEFT JOIN worker_create_reservations reservation \
|
||||
@@ -19265,6 +19302,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
|
||||
WorkspaceSchemaMigrationStep {version:103, name:"Passkey metadata and atomic addition results".into()},
|
||||
WorkspaceSchemaMigrationStep {version:104, name:"Converge Passkey results and Job input retention".into()},
|
||||
WorkspaceSchemaMigrationStep {version:105, name:"Durable Subject deletion snapshots and Job admission fences".into()},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -19379,6 +19417,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(102, "Durable failed Subject creation recovery".to_string()),
|
||||
(103, "Passkey metadata and atomic addition results".to_string()),
|
||||
(104, "Converge Passkey results and Job input retention".to_string()),
|
||||
(105, "Durable Subject deletion snapshots and Job admission fences".to_string()),
|
||||
]
|
||||
);
|
||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||
@@ -22664,7 +22703,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
||||
(101, "Durable explicit Subject body release".into()),
|
||||
(102, "Durable failed Subject creation recovery".into()),
|
||||
(103, "Passkey metadata and atomic addition results".into()),
|
||||
(104, "Converge Passkey results and Job input retention".into())]);
|
||||
(104, "Converge Passkey results and Job input retention".into()),
|
||||
(105, "Durable Subject deletion snapshots and Job admission fences".into())]);
|
||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||
}
|
||||
@@ -26857,12 +26897,19 @@ INSERT INTO worker_registry (
|
||||
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
|
||||
.unwrap();
|
||||
store.with_conn(|conn| {
|
||||
// This fixture deliberately reconstructs schema 102 from fresh.
|
||||
// Remove the newer Subject fence composition, not just its receipt.
|
||||
let triggers=conn.prepare("SELECT name,tbl_name FROM sqlite_schema WHERE type='trigger' AND tbl_name LIKE 'notification_%' AND name NOT LIKE 'subject_deleted_%' AND sql LIKE '%subject_deletion_jobs%'")?
|
||||
.query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
for (name,table) in triggers {conn.execute_batch(&format!("DROP TRIGGER {name}; CREATE TRIGGER {name} BEFORE DELETE ON {table} BEGIN SELECT RAISE(ABORT,'Notification evidence is immutable'); END;"))?;}
|
||||
for name in ["subject_deleted_job_insert","subject_deleted_job_result","subject_deleted_attempt_insert","subject_deleted_attempt_result","subject_deleted_worker_reservation","subject_deleted_notification_source","subject_deleted_notification_payload","subject_deleted_notification_send","subject_deletion_worker_prior_insert","subject_deletion_worker_identity_immutable"] {conn.execute_batch(&format!("DROP TRIGGER {name}"))?;}
|
||||
conn.execute_batch("DROP TABLE subject_deletion_allocations; DROP TABLE subject_deletion_jobs; DROP TABLE subject_deletion_workers; DROP TABLE subject_deletions;")?;
|
||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||
DROP TRIGGER passkey_metadata_insert;
|
||||
DROP TABLE passkey_addition_receipts;
|
||||
DROP TABLE passkey_metadata;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=105;
|
||||
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
|
||||
}).unwrap();
|
||||
drop(store);
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
-- Content-free, monotonic main-DB fence. The Host installs the domain fence first
|
||||
-- and holds its Subject gate across snapshot/admission. No Subject contents live here.
|
||||
CREATE TABLE subject_deletions (
|
||||
workspace_id TEXT NOT NULL,
|
||||
subject_id TEXT NOT NULL,
|
||||
operation_id TEXT NOT NULL CHECK(length(operation_id)>0),
|
||||
completed INTEGER NOT NULL DEFAULT 0 CHECK(completed IN (0,1)),
|
||||
jobs_erased INTEGER NOT NULL DEFAULT 0 CHECK(jobs_erased IN (0,1)),
|
||||
PRIMARY KEY(workspace_id,subject_id),
|
||||
UNIQUE(workspace_id,operation_id),
|
||||
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TABLE subject_deletion_workers (
|
||||
workspace_id TEXT NOT NULL,
|
||||
subject_id TEXT NOT NULL,
|
||||
runtime_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
is_job INTEGER NOT NULL CHECK(is_job IN (0,1)),
|
||||
-- Exact canonical success already present at prepare, never a later receipt.
|
||||
prior_removal_operation_id TEXT REFERENCES worker_removal_operations(operation_id),
|
||||
-- Confirmed by the scoped Runtime storage owner, not catalog absence or Stop ACK.
|
||||
originals_absent INTEGER NOT NULL DEFAULT 0 CHECK(originals_absent IN (0,1)),
|
||||
PRIMARY KEY(workspace_id,subject_id,runtime_id,worker_id),
|
||||
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TRIGGER subject_deletion_worker_prior_insert BEFORE INSERT ON subject_deletion_workers
|
||||
WHEN NEW.prior_removal_operation_id IS NOT NULL AND NOT EXISTS(
|
||||
SELECT 1 FROM worker_removal_operations r WHERE r.operation_id=NEW.prior_removal_operation_id
|
||||
AND r.workspace_id=NEW.workspace_id AND r.runtime_id=NEW.runtime_id
|
||||
AND r.worker_id=NEW.worker_id AND r.state='succeeded')
|
||||
BEGIN SELECT RAISE(ABORT,'Subject deletion prior removal must be an exact canonical success'); END;
|
||||
CREATE TRIGGER subject_deletion_worker_identity_immutable BEFORE UPDATE OF
|
||||
workspace_id,subject_id,runtime_id,worker_id,prior_removal_operation_id ON subject_deletion_workers
|
||||
BEGIN SELECT RAISE(ABORT,'Subject deletion Worker snapshot identity is immutable'); END;
|
||||
CREATE TRIGGER subject_deletion_original_absence_monotonic BEFORE UPDATE OF originals_absent ON subject_deletion_workers
|
||||
WHEN OLD.originals_absent=1 AND NEW.originals_absent=0
|
||||
BEGIN SELECT RAISE(ABORT,'Subject deletion original absence proof is permanent'); END;
|
||||
-- Job/attempt identities remain after payload erasure, preventing stale allocation
|
||||
-- or caller-selected Job-id reuse from recreating the deleted Subject's inputs.
|
||||
CREATE TABLE subject_deletion_jobs (
|
||||
workspace_id TEXT NOT NULL,
|
||||
subject_id TEXT NOT NULL,
|
||||
job_id TEXT NOT NULL,
|
||||
PRIMARY KEY(workspace_id,job_id),
|
||||
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TABLE subject_deletion_allocations (
|
||||
workspace_id TEXT NOT NULL,
|
||||
subject_id TEXT NOT NULL,
|
||||
allocation_key TEXT NOT NULL,
|
||||
PRIMARY KEY(workspace_id,allocation_key),
|
||||
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TRIGGER subject_deleted_job_insert BEFORE INSERT ON backend_jobs
|
||||
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
|
||||
(d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_journal.subject_id') OR
|
||||
d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_consolidation.subject_id')))
|
||||
OR EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job admission fenced'); END;
|
||||
CREATE TRIGGER subject_deleted_job_result BEFORE UPDATE OF state,result_json ON backend_jobs
|
||||
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job result fenced'); END;
|
||||
CREATE TRIGGER subject_deleted_attempt_insert BEFORE INSERT ON backend_job_attempts
|
||||
WHEN EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job retry fenced'); END;
|
||||
CREATE TRIGGER subject_deleted_attempt_result BEFORE UPDATE OF state,result_json,runtime_id,worker_id,runtime_run_id ON backend_job_attempts
|
||||
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job attempt fenced'); END;
|
||||
CREATE TRIGGER subject_deleted_worker_reservation BEFORE INSERT ON worker_create_reservations
|
||||
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
|
||||
NEW.singleton_key IN ('subjektiv:'||d.subject_id,'subjektiv-consolidation:'||d.subject_id))
|
||||
OR EXISTS(SELECT 1 FROM subject_deletion_allocations d WHERE d.workspace_id=NEW.workspace_id AND d.allocation_key=NEW.allocation_key)
|
||||
BEGIN SELECT RAISE(ABORT,'subject_deleted: Worker allocation fenced'); END;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -94,6 +94,12 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
|
||||
"workdir_registry",
|
||||
"workdir_removal_operations",
|
||||
"worker_control_grants",
|
||||
// Snapshot children precede their fence parent and canonical Worker removal
|
||||
// receipts (subject_deletion_workers.prior_removal_operation_id).
|
||||
"subject_deletion_allocations",
|
||||
"subject_deletion_jobs",
|
||||
"subject_deletion_workers",
|
||||
"subject_deletions",
|
||||
"subject_worker_releases",
|
||||
"subject_create_recoveries",
|
||||
"worker_create_reservations",
|
||||
@@ -1276,6 +1282,134 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_purge_removes_subject_deletion_snapshots_and_preserves_other_workspace() {
|
||||
let (store, owner, workspace_id) = setup();
|
||||
let tables = [
|
||||
"subject_deletion_allocations",
|
||||
"subject_deletion_jobs",
|
||||
"subject_deletion_workers",
|
||||
"subject_deletions",
|
||||
"worker_removal_operations",
|
||||
];
|
||||
// Check the explicit FK ordering as well as execution (the purge defers FKs).
|
||||
let position = |table| {
|
||||
WORKSPACE_DELETION_PURGE_TABLES
|
||||
.iter()
|
||||
.position(|name| *name == table)
|
||||
.unwrap()
|
||||
};
|
||||
for child in &tables[..3] {
|
||||
assert!(position(*child) < position("subject_deletions"));
|
||||
}
|
||||
assert!(position("subject_deletion_workers") < position("worker_removal_operations"));
|
||||
for workspace in ["workspace-a", "workspace-b"] {
|
||||
store.with_conn(|conn| {
|
||||
conn.execute("INSERT INTO worker_removal_operations(
|
||||
operation_id,plan_id,input_fingerprint,workspace_id,runtime_id,worker_id,
|
||||
worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition,
|
||||
archive_retention_kind,diagnostics_disposition,blockers_json,state,reason,created_at,updated_at)
|
||||
VALUES(?1,?1,'fingerprint',?1,'runtime','worker','1','policy','digest',
|
||||
'purge','purge','forever','purge','[]','succeeded','removed','1','1')", [workspace])?;
|
||||
conn.execute("INSERT INTO subject_deletions(workspace_id,subject_id,operation_id,completed,jobs_erased)
|
||||
VALUES(?1,'subject','subject-delete',1,1)", [workspace])?;
|
||||
conn.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id,originals_absent)
|
||||
VALUES(?1,'subject','runtime','worker',1,?1,1)", [workspace])?;
|
||||
conn.execute("INSERT INTO subject_deletion_jobs VALUES(?1,'subject','job')", [workspace])?;
|
||||
conn.execute("INSERT INTO subject_deletion_allocations VALUES(?1,'subject','allocation')", [workspace])?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
}
|
||||
let rows = |workspace: &str| {
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
tables
|
||||
.iter()
|
||||
.map(|table| {
|
||||
let mut statement = conn
|
||||
.prepare(&format!("SELECT * FROM {table} WHERE workspace_id=?1"))?;
|
||||
let count = statement.column_count();
|
||||
Ok(statement
|
||||
.query_map([workspace], |row| {
|
||||
(0..count)
|
||||
.map(|index| row.get::<_, rusqlite::types::Value>(index))
|
||||
.collect::<rusqlite::Result<Vec<_>>>()
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?)
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()
|
||||
})
|
||||
.unwrap()
|
||||
};
|
||||
let before = rows("workspace-a");
|
||||
let other = rows("workspace-b");
|
||||
assert!(before.iter().all(|table| table.len() == 1));
|
||||
let preflight = store
|
||||
.workspace_deletion_preflight(&owner, &workspace_id)
|
||||
.unwrap();
|
||||
let operation = store
|
||||
.reserve_workspace_deletion(
|
||||
&owner,
|
||||
&workspace_id,
|
||||
&WorkspaceDeletionRequest {
|
||||
operation_id: "purge-subject-snapshots".into(),
|
||||
expected_workspace_updated_at: preflight.expected_workspace_updated_at,
|
||||
confirmation: "Alpha".into(),
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.operation;
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch(
|
||||
"CREATE TRIGGER reject_subject_snapshot_purge BEFORE DELETE ON workspaces
|
||||
BEGIN SELECT RAISE(ABORT,'injected purge failure'); END;",
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.finalize_workspace_deletion(&operation.operation_id)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
rows("workspace-a"),
|
||||
before,
|
||||
"rollback restores snapshots and canonical receipt"
|
||||
);
|
||||
assert_eq!(rows("workspace-b"), other);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER reject_subject_snapshot_purge")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let completed = store
|
||||
.finalize_workspace_deletion(&operation.operation_id)
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, WorkspaceDeletionState::Succeeded);
|
||||
assert!(rows("workspace-a").iter().all(|table| table.is_empty()));
|
||||
assert_eq!(rows("workspace-b"), other);
|
||||
assert_eq!(
|
||||
store
|
||||
.finalize_workspace_deletion(&operation.operation_id)
|
||||
.unwrap(),
|
||||
completed
|
||||
);
|
||||
store
|
||||
.with_conn(|conn| {
|
||||
assert_eq!(
|
||||
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| {
|
||||
row.get::<_, i64>(0)
|
||||
})?,
|
||||
0
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_is_idempotent_and_removes_workspace_scoped_rows() {
|
||||
let (store, owner, workspace_id) = setup();
|
||||
|
||||
Reference in New Issue
Block a user