From a06f6bfb3de0bdc0d87897a0ea26a5b1997f2ca3 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 28 Oct 2026 11:18:43 +0900 Subject: [PATCH] feat: replace Subject retirement with durable explicit deletion --- crates/runtime-api/src/lib.rs | 58 + crates/server-api/src/lib.rs | 41 +- .../tests/subject_delete_contract.rs | 49 + crates/subjektiv/README.md | 46 + crates/subjektiv/src/api/context.rs | 1 + crates/subjektiv/src/api/tests.rs | 42 + crates/subjektiv/src/deletion.rs | 419 ++++++ crates/subjektiv/src/deletion/tests.rs | 489 +++++++ crates/subjektiv/src/journal/binding.rs | 9 + .../src/journal/memory_links/tests.rs | 66 + crates/subjektiv/src/journal/recording.rs | 4 + crates/subjektiv/src/journal/service.rs | 5 + crates/subjektiv/src/journal/store.rs | 2 + crates/subjektiv/src/journal/tests.rs | 332 ++++- crates/subjektiv/src/lib.rs | 91 +- crates/subjektiv/src/permissions.rs | 9 +- crates/subjektiv/src/permissions/tests.rs | 52 +- crates/worker-runtime/src/execution.rs | 31 + crates/worker-runtime/src/fs_store.rs | 76 +- crates/worker-runtime/src/http_server.rs | 18 +- .../src/http_server/runtime_management_api.rs | 12 + .../tests/session_material_tests.rs | 48 + .../src/retention/runtime_v5_tests.rs | 2 +- crates/worker-runtime/src/runtime.rs | 49 +- .../src/runtime/subject_delete.rs | 168 +++ .../src/runtime/subject_delete_tests.rs | 274 ++++ .../src/subject_delete_backend_tests.rs | 285 ++++ .../src/subject_delete_create_tests.rs | 101 ++ .../src/subject_delete_store_tests.rs | 104 ++ crates/worker-runtime/src/test_support.rs | 4 +- crates/worker-runtime/src/worker_backend.rs | 176 +++ .../src/assignment_tracking_tests.rs | 3 +- crates/workspace-server/src/hosts.rs | 103 +- .../src/implementation_result_tests.rs | 132 +- crates/workspace-server/src/latest_schema.sql | 2 + .../src/notification_delivery.rs | 22 + .../src/notification_routing_tests.rs | 11 +- crates/workspace-server/src/retention.rs | 169 ++- .../src/schema_convergence_tests.rs | 19 +- crates/workspace-server/src/server.rs | 180 ++- .../src/server/subject_authorization.rs | 9 + .../src/server/subjektiv_deletion.rs | 288 ++++ .../src/server/subjektiv_journal.rs | 25 + .../server/tests/subject_deletion_tests.rs | 729 ++++++++++ .../server/tests/subjektiv_journal_tests.rs | 93 +- crates/workspace-server/src/store.rs | 63 +- .../src/subject_deletion_schema.sql | 73 + .../src/subject_deletion_store.rs | 1178 +++++++++++++++++ .../src/workspace_deletion.rs | 134 ++ openapi/server-api.json | 283 ++-- tools/state-contract-name-exceptions.json | 8 + .../web-ux/scenarios/subject-delete-dark.json | 162 +++ .../scenarios/subject-delete-light.json | 162 +++ web/workspace/deno.json | 2 +- web/workspace/deno.lock | 7 + .../src/lib/generated/companion-api.ts | 2 +- .../src/lib/generated/legacy-server-api.ts | 4 + web/workspace/src/lib/generated/memory-api.ts | 8 + .../src/lib/generated/repository-api.ts | 2 +- .../src/lib/generated/runtime-api.ts | 2 +- web/workspace/src/lib/generated/ticket-api.ts | 2 +- .../src/lib/generated/worker-launch-api.ts | 2 +- .../src/lib/workspace/api/workspace-model.ts | 2 + .../lib/workspace/memory/SubjectDelete.svelte | 134 ++ web/workspace/src/lib/workspace/memory/api.ts | 66 + .../lib/workspace/memory/subject-delete.ts | 51 + .../w/[workspaceId]/memory/PageContent.svelte | 2 +- .../memory/[subjectId]/PageContent.svelte | 15 +- .../subject-delete-browser/fixture-server.ts | 142 ++ .../subject-delete.browser.test.ts | 257 ++++ web/workspace/tests/subject-delete.test.ts | 152 +++ 71 files changed, 7448 insertions(+), 315 deletions(-) create mode 100644 crates/server-api/tests/subject_delete_contract.rs create mode 100644 crates/subjektiv/src/deletion.rs create mode 100644 crates/subjektiv/src/deletion/tests.rs create mode 100644 crates/worker-runtime/src/runtime/subject_delete.rs create mode 100644 crates/worker-runtime/src/runtime/subject_delete_tests.rs create mode 100644 crates/worker-runtime/src/subject_delete_backend_tests.rs create mode 100644 crates/worker-runtime/src/subject_delete_create_tests.rs create mode 100644 crates/worker-runtime/src/subject_delete_store_tests.rs create mode 100644 crates/workspace-server/src/server/subjektiv_deletion.rs create mode 100644 crates/workspace-server/src/server/tests/subject_deletion_tests.rs create mode 100644 crates/workspace-server/src/subject_deletion_schema.sql create mode 100644 crates/workspace-server/src/subject_deletion_store.rs create mode 100644 tools/web-ux/scenarios/subject-delete-dark.json create mode 100644 tools/web-ux/scenarios/subject-delete-light.json create mode 100644 web/workspace/src/lib/workspace/memory/SubjectDelete.svelte create mode 100644 web/workspace/src/lib/workspace/memory/subject-delete.ts create mode 100644 web/workspace/test/subject-delete-browser/fixture-server.ts create mode 100644 web/workspace/test/subject-delete-browser/subject-delete.browser.test.ts create mode 100644 web/workspace/tests/subject-delete.test.ts diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index ea000bd2e..225045481 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -1193,8 +1193,40 @@ pub struct SubjectCreateRecoveryResponse { pub operation_id: String, } +/// Host requests this only after durably fencing the Subject in Backend authority. +/// Permanently exclude the Worker from creation and restoration, and confirm actual +/// execution shutdown. This does not delete Worker or Session records. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct SubjectDeleteWorkerRequest { + pub workspace_id: String, + pub worker_id: WorkerId, + pub create_fingerprint: String, + pub operation_id: String, +} + +/// Durable admission exclusion plus confirmed execution shutdown/absence. Errors, +/// timeouts and lost responses are unknown outcomes; retry the same full request. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct SubjectDeleteWorkerResponse { + pub workspace_id: String, + pub worker_id: WorkerId, + pub create_fingerprint: String, + pub operation_id: String, + /// The canonical live Worker/Session/diagnostic aggregate is physically absent + /// under the permanent creation fence. False is not a deletion receipt. + pub originals_absent: bool, +} + #[api(reqwest, axum)] pub trait RuntimeApi { + #[post("/v1/subject-delete-worker", status = 200, error_status = 400)] + async fn subject_delete_worker( + &self, + #[body] request: SubjectDeleteWorkerRequest, + ) -> Result; + #[post("/v1/subject-create-recovery", status = 200, error_status = 400)] async fn recover_subject_create( &self, @@ -1755,6 +1787,19 @@ mod tests { ) -> Result { Err(test_error(501)) } + async fn subject_delete_worker( + &self, + request: SubjectDeleteWorkerRequest, + ) -> Result { + Ok(SubjectDeleteWorkerResponse { + originals_absent: true, + workspace_id: request.workspace_id, + worker_id: request.worker_id, + create_fingerprint: request.create_fingerprint, + operation_id: request.operation_id, + }) + } + async fn recover_subject_create( &self, request: SubjectCreateRecoveryRequest, @@ -1934,6 +1979,19 @@ mod tests { assert_eq!(response.create_fingerprint, recovery.create_fingerprint); assert_eq!(response.operation_id, recovery.operation_id); + let deletion = SubjectDeleteWorkerRequest { + workspace_id: "workspace-a".into(), + worker_id: WorkerId::now_v7(), + create_fingerprint: "deleted-create".into(), + operation_id: "delete-1".into(), + }; + let receipt = client + .subject_delete_worker(deletion.clone()) + .await + .unwrap(); + let mut expected = serde_json::to_value(deletion).unwrap(); + expected["originals_absent"] = true.into(); + assert_eq!(serde_json::to_value(receipt).unwrap(), expected); let summary = client.runtime_summary().await.unwrap(); assert_eq!(summary.runtime.display_name.as_deref(), Some("round-trip")); let workers = client diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index 59b2d6b8f..0b4d55499 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -338,6 +338,8 @@ impl_openapi_schema!( MemoryConsolidationResponse, SubjektivSubjectCreateRequest, SubjektivSubjectBehaviorUpdateRequest, + SubjektivSubjectDeleteRequest, + SubjektivSubjectDeleteResponse, SubjektivSubjectListQuery, SubjektivSubjectListResponse, SubjektivSubjectResponse, @@ -1527,19 +1529,21 @@ pub trait ServerApi { #[path] memory_id: String, #[query] query: SubjektivMemoryChangesQuery, ) -> Result; - #[post( - "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire", + #[delete( + "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}", status = 200, error_status = 400, additional_error_statuses = [401, 403, 404, 409, 500], bearer_auth = true, browser_auth = true, )] - async fn subjektiv_subject_retire( + async fn subjektiv_subject_delete( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] subject_id: String, - ) -> Result; + #[body] request: SubjektivSubjectDeleteRequest, + ) -> Result; #[post( "/api/w/{workspace_id}/subjektiv/staging", status = 200, @@ -4955,6 +4959,8 @@ pub struct WorkspacePermissionSummary { pub manage_secrets: bool, pub manage_runtimes: bool, pub delete_workspace: bool, + /// Explicit Subject deletion authority for the current authenticated actor. + pub delete_subjects: bool, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] @@ -5350,7 +5356,25 @@ pub struct MemoryConsolidationResponse { #[serde(rename_all = "snake_case")] pub enum SubjektivSubjectState { Active, - Retired, + /// Saved compatibility only; not a current lifecycle operation. + #[serde(rename = "retired")] + LegacyRetired, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SubjektivSubjectDeleteRequest { + /// Stable identity retained for every retry, including unknown outcomes. + pub operation_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SubjektivSubjectDeleteResponse { + pub subject_id: String, + pub operation_id: String, + /// False confirms the deletion fence, not external stop or erasure completion. + pub completed: bool, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] @@ -10950,6 +10974,10 @@ pub fn memory_api_typescript() -> String { export type SubjektivSubjectCreateRequest = { role: string, behavior_md?: string, }; +export type SubjektivSubjectDeleteRequest = { operation_id: string, }; + +export type SubjektivSubjectDeleteResponse = { subject_id: string, operation_id: string, completed: boolean, }; + export type SubjektivSubjectBehaviorUpdateRequest = { expected_behavior_md: string, behavior_md: string, }; export type SubjektivSubjectResponse = { id: string, role: string, behavior_md: string, state: SubjektivSubjectState, memory_fingerprint: string, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, }; @@ -13483,7 +13511,8 @@ mod tests { "manage_repositories": true, "manage_secrets": true, "manage_runtimes": true, - "delete_workspace": true + "delete_workspace": true, + "delete_subjects": true }, "extension_points": { "store": "sqlite", diff --git a/crates/server-api/tests/subject_delete_contract.rs b/crates/server-api/tests/subject_delete_contract.rs new file mode 100644 index 000000000..c219933cf --- /dev/null +++ b/crates/server-api/tests/subject_delete_contract.rs @@ -0,0 +1,49 @@ +use serde_json::json; +use server_api::{ + SubjektivSubjectDeleteRequest, SubjektivSubjectDeleteResponse, SubjektivSubjectState, +}; + +#[test] +fn subject_delete_api_replaces_retire_and_requires_operation_identity() { + let document: serde_json::Value = serde_json::from_str( + &server_api::canonical_openapi_document() + .unwrap() + .to_json() + .unwrap(), + ) + .unwrap(); + let paths = &document["paths"]; + assert!( + paths + .get("/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire") + .is_none() + ); + let operation = &paths["/api/w/{workspace_id}/subjektiv/subjects/{subject_id}"]["delete"]; + assert_eq!(operation["operationId"], "subjektiv_subject_delete"); + assert_eq!(operation["requestBody"]["required"], true); + assert!(serde_json::from_value::(json!({})).is_err()); + assert!( + serde_json::from_value::( + json!({"operation_id":"op", "force":true}) + ) + .is_err() + ); + for completed in [false, true] { + let response = SubjektivSubjectDeleteResponse { + subject_id: "subject".into(), + operation_id: "op".into(), + completed, + }; + assert_eq!( + serde_json::to_value(response).unwrap(), + json!({"subject_id":"subject", "operation_id":"op", "completed":completed}) + ); + } +} + +#[test] +fn old_subject_state_is_explicit_saved_compatibility_not_a_new_operation() { + let state: SubjektivSubjectState = serde_json::from_value(json!("retired")).unwrap(); + assert_eq!(state, SubjektivSubjectState::LegacyRetired); + assert_eq!(serde_json::to_value(state).unwrap(), json!("retired")); +} diff --git a/crates/subjektiv/README.md b/crates/subjektiv/README.md index b798be40d..5212d8572 100644 --- a/crates/subjektiv/README.md +++ b/crates/subjektiv/README.md @@ -12,3 +12,49 @@ Hosts retain body ownership across inactivity, reserve actual execution quiescen - [`api`]: shared Host operations and bounded responses - [`job`]: consolidation selection rules and result validation - [`journal`]: individually committed activity operations, source coverage, retry receipts, and bounded reading + +## Explicit Subject deletion + +`begin_subject_deletion(subject_id, operation_id)` commits a permanent content-free +fence in this store **before** the Host stops execution or erases external content. +The operation ID is unique within this storage scope. Exact retries return current +saved state; a different key for that Subject or reuse for another Subject conflicts. +Pending fences survive restart and a failed purge. Normal reads, writes, saved-receipt +replay, cached Host/Journal contexts and authority lookup cannot use a fenced Subject. +Subject discovery hides fenced records before pagination. SQL INSERT/UPDATE guards +cover direct and indirect owners, checking both OLD and NEW ownership on UPDATE. +Deleting a grant never restores non-Subject fallback: content-free Worker identity +fences remain, and the Host must consult them before fallback authorization. + +The Host uses `subject_deletion` / `pending_subject_deletions` for recovery and +`deletion_subject_workers` / `deletion_subject_sessions` / `deletion_subject_jobs` +only as cleanup inventories, not authority or model-visible content. It must fence +creation reservations, quiesce Workers and in-flight delivery, purge Job inputs/results +and live Job execution copies, and clear resident/derived caches. Source Worker/Session +originals and pre-existing archives have independent retention lifecycles: the Workspace +Host archives source originals rather than erasing them. Those archives may contain +previously injected Memory; Workdirs, Tickets, exports and external provider copies are +also outside Subject content erasure. These boundaries must be explained to the user. +The domain rechecks after out-of-DB capture callbacks and before returning Job read +responses; already returned DTOs cannot be revoked by SQLite. Host lifecycle locking +must cover delivery/reinjection of previously fetched content as well as finish. + +Only after external cleanup succeeds may the Host call +`finish_subject_deletion(subject_id, operation_id)`. One transaction deletes Subject, +Memory/history, staging/evidence, Surface snapshots/runs, permission sources/receipts, +child grants, writer/session bindings, Journal content/generations/deliveries/receipts, +links, and the Subject's `legacy_record_archive` rows. A purge error rolls back all +those deletions without undoing the separately committed fence. Ordinary history +retention and explicit Journal Session-erasure rules are unchanged for other Subjects. +Retained records contain only deletion operation/time/completion and execution/session +identities needed to prevent reuse; existing content-free Session-erasure markers and +workspace-only migration metadata remain. Other Subjects and storage scopes are untouched. + +The deletion migration adds fences without erasing or reactivating existing data. Old `retired` +JSON/SQL records deserialize as `SubjectState::LegacyRetired` and remain readable +until explicitly deleted. There is no current retirement API. Normal Worker deletion +still does not implicitly delete Subject memory. + +Completion means logical erasure from domain and Host-managed storage, **not** secure +physical overwriting: SQLite free pages, WAL files, filesystem snapshots, backups and +external copies can retain bytes under their own retention/erasure policies. diff --git a/crates/subjektiv/src/api/context.rs b/crates/subjektiv/src/api/context.rs index bfea8ad1f..5528c5b45 100644 --- a/crates/subjektiv/src/api/context.rs +++ b/crates/subjektiv/src/api/context.rs @@ -116,6 +116,7 @@ impl HostOperationContext { "Host operation context belongs to another storage scope".into(), )); } + Self::require_subject(store, &self.subject_id)?; Ok(()) } diff --git a/crates/subjektiv/src/api/tests.rs b/crates/subjektiv/src/api/tests.rs index 00c3429f8..a816fac03 100644 --- a/crates/subjektiv/src/api/tests.rs +++ b/crates/subjektiv/src/api/tests.rs @@ -452,3 +452,45 @@ fn query_content_condition_ignores_behavior_and_detects_aba_memory_writes() { Err(OperationError::Conflict(_)) )); } + +#[test] +fn subject_deletion_revalidates_cached_body_empty_batch_and_receipt_contexts() { + let temp = tempfile::tempdir().unwrap(); + let (_manager, store) = open(temp.path(), "scope"); + let subject = store + .create_subject(SubjectRole::new("companion").unwrap()) + .unwrap(); + let body = HostOperationContext::validated_body(&store, &subject.id, None).unwrap(); + let empty = + HostOperationContext::validated_consolidation(&store, &subject.id, Some(vec![]), None) + .unwrap(); + store.begin_subject_deletion(&subject.id, "delete").unwrap(); + for context in [&body, &empty] { + for operation in [ + query(None), + Op::ListCandidates(server_api::SubjektivMemoryCandidateListRequest { limit: None }), + Op::ReceiptStatus(server_api::SubjektivMemoryReceiptStatusRequest { + receipt_id: "receipt".into(), + }), + Op::ReadCandidate(server_api::SubjektivMemoryCandidateReadRequest { + candidate_id: "candidate".into(), + }), + ] { + assert!(matches!( + execute(&store, context, operation), + Err(OperationError::SubjectNotFound(_)) + )); + } + } + store + .finish_subject_deletion(&subject.id, "delete") + .unwrap(); + assert!( + execute( + &store, + &empty, + Op::ListCandidates(server_api::SubjektivMemoryCandidateListRequest { limit: None }) + ) + .is_err() + ); +} diff --git a/crates/subjektiv/src/deletion.rs b/crates/subjektiv/src/deletion.rs new file mode 100644 index 000000000..937718794 --- /dev/null +++ b/crates/subjektiv/src/deletion.rs @@ -0,0 +1,419 @@ +//! Explicit Subject erasure is a two-phase Host operation. Beginning commits a permanent, +//! content-free fence before the Host stops/removes Workers and purges Jobs and execution copies. +//! Source Worker/Session originals remain under their separate archive lifecycle. +//! Failure leaves that fence and the cleanup inventory intact; the Host retries +//! the same operation ID. Finishing asserts that external cleanup succeeded and atomically erases +//! this store's content. This store cannot verify external Runtime/Job/Session cleanup itself. +//! +//! Tombstones contain no behavior, permission vectors, Memory, Journal or Session text. Worker +//! identities remain permanently fenced so removal of a grant cannot become non-Subject fallback +//! authority. Existing Journal Session erasure markers are retained to reject delayed admission; +//! they are content-free and are not a second copy of the deleted material. +use super::*; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SubjectDeletion { + pub subject_id: String, + pub operation_id: String, + pub completed: bool, + pub started_at: String, +} + +/// Host-only cleanup identity, never Job input, output, profile or authorization material. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeletionSubjectJob { + pub job_id: String, + pub attempt_id: String, +} + +// All content tables are enumerated deliberately. Indirect tables are erased before their +// owner rows, both for FK checks and so their fencing predicates remain resolvable. +const INDIRECT: &[(&str, &str)] = &[ + ( + "journal_generation_input_deliveries", + "generation_id IN (SELECT g.id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id={subject})", + ), + ( + "legacy_record_archive", + "json_extract(row_json, '$.subject_id')={subject}", + ), + ( + "journal_generation_deliveries", + "generation_id IN (SELECT g.id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id={subject})", + ), + ( + "journal_operations", + "journal_id IN (SELECT id FROM journals WHERE subject_id={subject})", + ), + ( + "journal_generations", + "journal_id IN (SELECT id FROM journals WHERE subject_id={subject})", + ), + ( + "journal_sources", + "journal_id IN (SELECT id FROM journals WHERE subject_id={subject})", + ), + ( + "journal_session_bindings", + "binding_id IN (SELECT binding_id FROM journal_writer_bindings WHERE subject_id={subject})", + ), +]; +const DIRECT: &[&str] = &[ + "memory_journal_links", + "journal_corrections", + "journal_entries", + "journals", + "journal_writer_bindings", + "subject_session_attributions", + "worker_execution_grants", + "subject_permission_receipts", + "subject_permissions", + "subject_permission_changes", + "candidate_decision_receipts", + "surface_generation_state", + "surface_generation_runs", + "surface_snapshot_refs", + "surface_snapshot_seals", + "surface_snapshots", + "staging_resolution_targets", + "staging_resolution_seals", + "staging_resolutions", + "memory_change_candidates", + "memory_change_derivations", + "memory_change_seals", + "memory_changes", + "memory_records", + "staging_records", + "subjects", +]; + +fn owner_expression(table: &str, row: &str) -> String { + match table { + "legacy_record_archive" => format!("json_extract({row}.row_json, '$.subject_id')"), + "journal_generation_input_deliveries" | "journal_generation_deliveries" => format!( + "(SELECT j.subject_id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE g.id={row}.generation_id)" + ), + "journal_operations" | "journal_generations" | "journal_sources" => { + format!("(SELECT subject_id FROM journals WHERE id={row}.journal_id)") + } + "journal_session_bindings" => format!( + "(SELECT subject_id FROM journal_writer_bindings WHERE binding_id={row}.binding_id)" + ), + _ => format!("{row}.subject_id"), + } +} + +pub(super) fn migrate(tx: &Transaction<'_>) -> feature_storage::Result<()> { + tx.execute_batch( + "CREATE TABLE subject_deletions ( + subject_id TEXT PRIMARY KEY, + operation_id TEXT NOT NULL UNIQUE, + completed INTEGER NOT NULL CHECK(completed IN (0,1)), + started_at TEXT NOT NULL + ); + CREATE TABLE subject_deleted_workers ( + subject_id TEXT NOT NULL REFERENCES subject_deletions(subject_id) ON DELETE RESTRICT, + runtime_id TEXT NOT NULL, + worker_id TEXT NOT NULL, + PRIMARY KEY(subject_id,runtime_id,worker_id) + ); + CREATE TABLE subject_deleted_sessions ( + subject_id TEXT NOT NULL REFERENCES subject_deletions(subject_id) ON DELETE RESTRICT, + session_id TEXT NOT NULL UNIQUE, + PRIMARY KEY(subject_id,session_id) + ); + CREATE TRIGGER subject_deleted_sessions_retained BEFORE DELETE ON subject_deleted_sessions + BEGIN SELECT RAISE(ABORT, 'deleted Session fences are permanent'); END; + CREATE TRIGGER subject_deleted_sessions_immutable BEFORE UPDATE ON subject_deleted_sessions + BEGIN SELECT RAISE(ABORT, 'deleted Session fences are immutable'); END; + CREATE INDEX subject_deleted_worker_identity ON subject_deleted_workers(runtime_id,worker_id); + CREATE TRIGGER subject_deletions_retained BEFORE DELETE ON subject_deletions + BEGIN SELECT RAISE(ABORT, 'Subject deletion fences are permanent'); END; + CREATE TRIGGER subject_deletions_identity BEFORE UPDATE ON subject_deletions + WHEN OLD.subject_id<>NEW.subject_id OR OLD.operation_id<>NEW.operation_id + OR OLD.started_at<>NEW.started_at OR OLD.completed>NEW.completed + BEGIN SELECT RAISE(ABORT, 'Subject deletion identity is permanent'); END; + CREATE TRIGGER subject_deleted_workers_retained BEFORE DELETE ON subject_deleted_workers + BEGIN SELECT RAISE(ABORT, 'deleted Worker fences are permanent'); END; + CREATE TRIGGER subject_deleted_workers_immutable BEFORE UPDATE ON subject_deleted_workers + BEGIN SELECT RAISE(ABORT, 'deleted Worker fences are immutable'); END;", + )?; + + // Preserve each existing history-retention condition, adding only the explicit Subject + // erasure exception. Frozen earlier migrations are not edited. Normal history deletion + // (including another Subject's history) remains forbidden. + let triggers = tx + .prepare("SELECT name,tbl_name,sql FROM sqlite_master WHERE type='trigger'")? + .query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + })? + .collect::, _>>()?; + for (name, table, sql) in triggers { + if !DIRECT.contains(&table.as_str()) && !INDIRECT.iter().any(|(t, _)| *t == table) { + continue; + } + let upper = sql.to_ascii_uppercase(); + if !upper.contains("BEFORE DELETE") { + continue; + } + let begin = upper.find("BEGIN").expect("trigger body"); + let (header, body) = sql.split_at(begin); + let exception = format!( + "NOT EXISTS(SELECT 1 FROM subject_deletions WHERE subject_id={})", + owner_expression(&table, "OLD") + ); + let header = if let Some(when) = header.to_ascii_uppercase().find("WHEN") { + format!( + "{} WHEN ({}) AND {exception} ", + &header[..when], + header[when + 4..].trim() + ) + } else { + format!("{header} WHEN {exception} ") + }; + tx.execute_batch(&format!("DROP TRIGGER \"{name}\"; {header}{body}"))?; + } + for table in DIRECT + .iter() + .copied() + .chain(INDIRECT.iter().map(|(t, _)| *t)) + .chain(["journal_erased_sessions"]) + { + for action in ["INSERT", "UPDATE"] { + let new_owner = owner_expression(table, "NEW"); + let predicate = if action == "UPDATE" { + format!( + "subject_id={new_owner} OR subject_id={}", + owner_expression(table, "OLD") + ) + } else { + format!("subject_id={new_owner}") + }; + tx.execute_batch(&format!( + "CREATE TRIGGER deletion_fence_{table}_{action} BEFORE {action} ON {table} + WHEN EXISTS(SELECT 1 FROM subject_deletions WHERE {predicate}) + BEGIN SELECT RAISE(ABORT, 'Subject deletion has begun'); END;" + ))?; + } + } + // Identity reuse must not reconnect purged content or turn a deleted child into + // an unscoped Worker. Check both old and new on ownership-changing UPDATEs. + for table in [ + "subject_session_attributions", + "journal_writer_bindings", + "worker_execution_grants", + ] { + for action in ["INSERT", "UPDATE"] { + let rows = if action == "UPDATE" { + vec!["OLD", "NEW"] + } else { + vec!["NEW"] + }; + let mut predicates = Vec::new(); + for row in rows { + predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id={row}.runtime_id AND worker_id={row}.worker_id)")); + if table == "subject_session_attributions" { + predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_sessions WHERE session_id={row}.session_id)")); + } + if table == "worker_execution_grants" { + predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=json_extract({row}.record_json,'$.origin_runtime_id') AND worker_id=json_extract({row}.record_json,'$.origin_worker_id'))")); + } + } + tx.execute_batch(&format!("CREATE TRIGGER deletion_identity_{table}_{action} BEFORE {action} ON {table} WHEN {} BEGIN SELECT RAISE(ABORT, 'deleted Subject execution identity'); END;", predicates.join(" OR ")))?; + } + } + Ok(()) +} + +pub(crate) fn is_deleting(connection: &Connection, subject: &str) -> Result { + // Older migration fixtures deliberately use current Rust reads before upgrade. Production + // opens only registered current stores and must fail closed if the fence table is absent. + #[cfg(test)] + if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deletions' AND type='table')", [], |r|r.get::<_,bool>(0))? { + return Ok(false); + } + Ok(connection.query_row( + "SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE subject_id=?1)", + [subject], + |r| r.get::<_, bool>(0), + )?) +} + +pub(crate) fn require_not_deleting(connection: &Connection, subject: &str) -> Result<()> { + if is_deleting(connection, subject)? { + return Err(SubjektivError::SubjectNotFound(subject.into())); + } + Ok(()) +} +pub(crate) fn require_live_worker( + connection: &Connection, + runtime: &str, + worker: &str, +) -> Result<()> { + #[cfg(test)] + if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deleted_workers' AND type='table')", [], |r|r.get::<_,bool>(0))? { return Ok(()); } + if connection.query_row( + "SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)", + params![runtime, worker], + |r| r.get::<_, bool>(0), + )? { + return Err(SubjektivError::AuthorizationDenied( + "Worker belongs to a deleted Subject".into(), + )); + } + Ok(()) +} + +pub(crate) fn deleted_session_subject( + connection: &Connection, + session: &str, +) -> Result> { + #[cfg(test)] + if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deleted_sessions' AND type='table')", [], |r|r.get::<_,bool>(0))? { return Ok(None); } + Ok(connection + .query_row( + "SELECT subject_id FROM subject_deleted_sessions WHERE session_id=?1", + [session], + |r| r.get(0), + ) + .optional()?) +} + +fn deletion(connection: &Connection, subject: &str) -> Result> { + Ok(connection.query_row("SELECT subject_id,operation_id,completed,started_at FROM subject_deletions WHERE subject_id=?1", [subject], |r|Ok(SubjectDeletion { subject_id:r.get(0)?, operation_id:r.get(1)?, completed:r.get(2)?, started_at:r.get(3)? })).optional()?) +} +fn require_deletion(connection: &Connection, subject: &str) -> Result { + deletion(connection, subject)?.ok_or_else(|| SubjektivError::SubjectNotFound(subject.into())) +} + +impl SubjektivStore { + /// Commits the read/write/authority fence, even for legacy-retired Subjects. Operation IDs + /// are unique within this store's scope; neither another key nor another Subject may reuse + /// a deletion. Exact retries return the saved current result, including completed deletions. + pub fn begin_subject_deletion( + &self, + subject: &str, + operation: &str, + ) -> Result { + validate_label("subject id", subject)?; + validate_label("deletion operation id", operation)?; + self.database.try_transaction(|tx| { + if let Some(saved) = deletion(tx,subject)? { + return if saved.operation_id == operation { Ok(saved) } else { Err(SubjektivError::SubjectDeletionConflict(operation.into())) }; + } + if tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE operation_id=?1)",[operation],|r|r.get::<_,bool>(0))? { + return Err(SubjektivError::SubjectDeletionConflict(operation.into())); + } + require_subject(tx,subject)?; + tx.execute("INSERT INTO subject_deletions(subject_id,operation_id,completed,started_at) VALUES(?1,?2,0,?3)",params![subject,operation,now()])?; + // Capture bodies, historical attribution, delegated children, and grant origins. + // This transaction also fences delayed grants/attributions, so no new identity can + // appear after the inventory is frozen. Runtime create reservations remain Host-owned. + tx.execute("INSERT INTO subject_deleted_workers(subject_id,runtime_id,worker_id) + SELECT subject_id,runtime_id,worker_id FROM subject_session_attributions WHERE subject_id=?1 + UNION SELECT subject_id,runtime_id,worker_id FROM journal_writer_bindings WHERE subject_id=?1 + UNION SELECT subject_id,runtime_id,worker_id FROM worker_execution_grants WHERE subject_id=?1 + UNION SELECT subject_id,json_extract(record_json,'$.origin_runtime_id'),json_extract(record_json,'$.origin_worker_id') FROM worker_execution_grants WHERE subject_id=?1",[subject])?; + tx.execute("INSERT INTO subject_deleted_sessions(subject_id,session_id) + SELECT subject_id,session_id FROM subject_session_attributions WHERE subject_id=?1 + UNION SELECT subject_id,source_session_id FROM journals WHERE subject_id=?1 + UNION SELECT subject_id,session_id FROM journal_erased_sessions WHERE subject_id=?1", [subject])?; + require_deletion(tx,subject) + }) + } + + /// Host-only, content-free cleanup inventory. Usable after begin, including exact finish + /// retries. Tombstones survive finish to prevent non-Subject permission fallback. + pub fn deletion_subject_workers(&self, subject: &str) -> Result> { + self.database.try_with_connection(|tx| { + require_deletion(tx,subject)?; + Ok(tx.prepare("SELECT runtime_id,worker_id FROM subject_deleted_workers WHERE subject_id=?1 ORDER BY runtime_id,worker_id")?.query_map([subject],|r|Ok((r.get(0)?,r.get(1)?)))?.collect::,_>>()?) + }) + } + + /// Host-only Session identities; retained source rows are never exposed through this path. + /// The Host must erase originals/archives before finish removes attribution. + pub fn deletion_subject_sessions(&self, subject: &str) -> Result> { + self.database.try_with_connection(|tx| { + require_deletion(tx,subject)?; + Ok(tx.prepare("SELECT session_id FROM subject_deleted_sessions WHERE subject_id=?1 ORDER BY session_id")?.query_map([subject],|r|r.get(0))?.collect::,_>>()?) + }) + } + + /// Host-only Job cleanup inventory. Job storage owns inputs, outcomes and Worker archives; + /// this store retains only these identities until Host cleanup has succeeded. + pub fn deletion_subject_jobs(&self, subject: &str) -> Result> { + self.database.try_with_connection(|tx| { + require_deletion(tx,subject)?; + Ok(tx.prepare("SELECT job_id,attempt_id FROM surface_generation_runs WHERE subject_id=?1 AND job_id IS NOT NULL AND attempt_id IS NOT NULL UNION SELECT g.job_id,g.attempt_id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id=?1 ORDER BY 1,2")?.query_map([subject],|r|Ok(DeletionSubjectJob {job_id:r.get(0)?,attempt_id:r.get(1)?}))?.collect::,_>>()?) + }) + } + + /// Host-only recovery state; this never exposes Subject content or authority. + pub fn subject_deletion(&self, subject: &str) -> Result> { + self.database + .try_with_connection(|tx| deletion(tx, subject)) + } + + /// Durable unfinished operations, including after Host restart. Completed tombstones + /// remain queryable individually but are not returned as pending work. + pub fn pending_subject_deletions(&self) -> Result> { + self.database.try_with_connection(|tx| { + Ok(tx.prepare("SELECT subject_id,operation_id,completed,started_at FROM subject_deletions WHERE completed=0 ORDER BY started_at,subject_id")? + .query_map([], |r| Ok(SubjectDeletion { subject_id:r.get(0)?, operation_id:r.get(1)?, completed:r.get(2)?, started_at:r.get(3)? }))? + .collect::,_>>()?) + }) + } + + pub fn worker_subject_deleted(&self, runtime: &str, worker: &str) -> Result { + self.database.try_with_connection(|tx| Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)",params![runtime,worker],|r|r.get(0))?)) + } + + /// Host confirmation that external cleanup is complete. All Subject content and completed + /// receipt commit together. Deferred FKs allow cyclic current/history references to disappear + /// without weakening FK enforcement outside this transaction. Failure rolls back all erasure. + pub fn finish_subject_deletion( + &self, + subject: &str, + operation: &str, + ) -> Result { + self.database.try_transaction(|tx| { + let saved = require_deletion(tx, subject)?; + if saved.operation_id != operation { + return Err(SubjektivError::SubjectDeletionConflict(operation.into())); + } + if saved.completed { + return Ok(saved); + } + tx.execute_batch("PRAGMA defer_foreign_keys=ON;")?; + for (table, predicate) in INDIRECT { + tx.execute( + &format!( + "DELETE FROM {table} WHERE {}", + predicate.replace("{subject}", "?1") + ), + [subject], + )?; + } + for table in DIRECT { + tx.execute( + &format!("DELETE FROM {table} WHERE subject_id=?1"), + [subject], + )?; + } + tx.execute( + "UPDATE subject_deletions SET completed=1 WHERE subject_id=?1", + [subject], + )?; + require_deletion(tx, subject) + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/subjektiv/src/deletion/tests.rs b/crates/subjektiv/src/deletion/tests.rs new file mode 100644 index 000000000..c0491162d --- /dev/null +++ b/crates/subjektiv/src/deletion/tests.rs @@ -0,0 +1,489 @@ +use super::*; +use crate::tests::{candidate, legacy_retired_fixture, open_store, role}; +use feature_storage::FeatureStorage; +use memory::extract::CandidateKind; + +fn draft() -> MemoryDraft { + MemoryDraft::active( + CandidateKind::Lesson, + "private claim", + "private body", + "useful", + "initial", + ) +} + +fn count(store: &SubjektivStore, table: &str) -> i64 { + store + .database + .try_with_connection::<_, SubjektivError>(|c| { + Ok(c.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0))?) + }) + .unwrap() +} + +#[test] +fn fence_only_denies_content_reads_writes_and_exact_replay_before_purge() { + let temp = tempfile::tempdir().unwrap(); + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + let subject = store + .create_subject_with_behavior(role(), "private behavior".into()) + .unwrap(); + let id = &subject.id; + let attribution = SubjectSessionAttribution::new(id, "runtime", "worker", "session").unwrap(); + store + .record_session_attribution(attribution.clone()) + .unwrap(); + let staged = candidate(id, "candidate", store.workspace_id()); + store.stage_candidate(staged.clone()).unwrap(); + let request = CandidateDecisionRequest { + request_id: "decision".into(), + candidate_id: "candidate".into(), + reason: "adopt".into(), + decision: CandidateDecision::Apply { + target: MemoryChangeTarget::Create, + draft: draft(), + }, + }; + let memory = store + .decide_candidate(id, request.clone()) + .unwrap() + .memory + .unwrap(); + let generation = store + .prepare_job_surface_generation(id, Some(("job", "attempt"))) + .unwrap(); + let snapshot = store + .publish_surface_generation( + id, + &generation.id, + vec![SurfacePoint { + body_md: "private surface".into(), + memory_refs: vec![MemoryChangeRef { + memory_id: memory.id.clone(), + change_id: memory.change_id.clone(), + }], + }], + ) + .unwrap(); + let fence = store.begin_subject_deletion(id, "delete").unwrap(); + assert!(!fence.completed); + // The fence must work while every body remains physically present. + assert_eq!(count(&store, "subjects"), 1); + assert_eq!(count(&store, "memory_records"), 1); + assert_eq!(count(&store, "candidate_decision_receipts"), 1); + assert!(store.subject(id).unwrap().is_none()); + assert!(store.list_subjects(1).unwrap().items.is_empty()); + macro_rules! denied { ($($result:expr),+ $(,)?) => { $(assert!($result.is_err(), stringify!($result));)+ }; } + denied!( + store.session_attribution("session"), + store.subject_session_attributions(id), + store.subject_session_attribution_page(id, None, "2999", None, 1), + store.record_session_attribution(attribution), + store.staging_candidate(id, "candidate"), + store.pending_staging_candidates(id, 1), + store.pending_staging_backlog(id), + store.staging_resolution(id, "candidate"), + store.memory(id, &memory.id), + store.memory_change(id, &memory.id, memory.change_id.clone()), + store.scoped_memory(id, &memory.id), + store.scoped_memory_change(id, &memory.id, memory.change_id.clone()), + store.list_memories(id), + store.list_memory_changes(id, &memory.id), + store.memory_fingerprint(id), + store.surface_snapshot(id, &snapshot.id), + store.resident_surface(id), + store.resident_context(id), + store.require_job_surface_generation(id, &generation.id, "job", "attempt"), + store.validate_job_surface_outcome( + id, + &generation.id, + generation.memory_fingerprint, + "ready", + Some(&snapshot.id), + None, + "job", + "attempt" + ), + store.prepare_surface_generation(id), + store.publish_surface_generation(id, &generation.id, vec![]), + store.fail_surface_generation(id, &generation.id, "failed"), + store.update_subject_behavior(id, &subject.behavior_md, "late behavior".into()), + store.stage_candidate(staged), + store.decide_candidate(id, request), + store.create_memory(id, draft()), + store.subject_permissions(id), + store.list_subject_worker_execution_grants(id), + ); + assert_eq!( + store.deletion_subject_workers(id).unwrap(), + vec![("runtime".into(), "worker".into())] + ); + assert_eq!( + store.deletion_subject_sessions(id).unwrap(), + vec!["session"] + ); + assert_eq!(store.deletion_subject_jobs(id).unwrap()[0].job_id, "job"); + assert!(store.worker_subject_deleted("runtime", "worker").unwrap()); + assert!( + store + .finish_subject_deletion(id, "delete") + .unwrap() + .completed + ); + for table in DIRECT { + assert_eq!(count(&store, table), 0, "{table}"); + } + assert!(store.worker_execution_grant("runtime", "worker").is_err()); + assert_eq!( + store.deletion_subject_sessions(id).unwrap(), + vec!["session"] + ); + assert_eq!( + store.finish_subject_deletion(id, "delete").unwrap(), + store.begin_subject_deletion(id, "delete").unwrap() + ); +} + +#[test] +fn purge_failure_rolls_back_content_but_not_the_committed_fence_and_restart_recovers() { + let temp = tempfile::tempdir().unwrap(); + let id: String; + let pending; + { + let (manager, scope, store) = open_store(temp.path(), "workspace-a"); + id = store.create_subject(role()).unwrap().id; + store.create_memory(&id, draft()).unwrap(); + pending = store.begin_subject_deletion(&id, "delete").unwrap(); + store.database.try_with_connection::<_, SubjektivError>(|c| { + c.execute_batch("CREATE TRIGGER injected_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT, 'simulated purge failure'); END;")?; + Ok(()) + }).unwrap(); + assert!(store.finish_subject_deletion(&id, "delete").is_err()); + assert_eq!(count(&store, "memory_records"), 1); + assert_eq!(count(&store, "memory_changes"), 1); + assert_eq!(store.subject_deletion(&id).unwrap(), Some(pending.clone())); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + } + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + assert_eq!( + store.pending_subject_deletions().unwrap(), + vec![pending.clone()] + ); + assert_eq!( + store.begin_subject_deletion(&id, "delete").unwrap(), + pending + ); + assert!(store.list_memories(&id).is_err()); + store + .database + .try_with_connection::<_, SubjektivError>(|c| { + c.execute_batch("DROP TRIGGER injected_failure")?; + Ok(()) + }) + .unwrap(); + assert!( + store + .finish_subject_deletion(&id, "delete") + .unwrap() + .completed + ); + assert!(store.pending_subject_deletions().unwrap().is_empty()); +} + +#[test] +fn deletion_identity_conflicts_are_scope_wide_but_do_not_cross_workspaces() { + let temp = tempfile::tempdir().unwrap(); + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + let a = store.create_subject(role()).unwrap().id; + let b = store.create_subject(role()).unwrap().id; + let saved = store.begin_subject_deletion(&a, "same-key").unwrap(); + assert_eq!(store.begin_subject_deletion(&a, "same-key").unwrap(), saved); + for result in [ + store.begin_subject_deletion(&a, "different"), + store.begin_subject_deletion(&b, "same-key"), + store.finish_subject_deletion(&a, "different"), + ] { + assert!(matches!( + result, + Err(SubjektivError::SubjectDeletionConflict(_)) + )); + } + assert!(store.begin_subject_deletion("absent", "new-key").is_err()); + assert!(store.finish_subject_deletion(&b, "same-key").is_err()); + let page = store.list_subjects(1).unwrap(); + assert_eq!(page.items[0].id, b); + assert!(!page.has_more); + store.finish_subject_deletion(&a, "same-key").unwrap(); + store.create_memory(&b, draft()).unwrap(); + let (_other_manager, _other_scope, other) = open_store(temp.path(), "workspace-b"); + assert!(other.subject_deletion(&a).unwrap().is_none()); + assert!(other.begin_subject_deletion(&b, "same-key").is_err()); + let c = other.create_subject(role()).unwrap().id; + other.begin_subject_deletion(&c, "same-key").unwrap(); + assert_eq!(store.list_memories(&b).unwrap().len(), 1); + let fresh = store.create_subject(role()).unwrap().id; + assert!(store.list_memories(&fresh).unwrap().is_empty()); + assert!(store.subject_permissions(&fresh).unwrap().is_none()); +} + +#[test] +fn concurrent_begin_and_write_serialize_and_exact_finish_is_replayable() { + let temp = tempfile::tempdir().unwrap(); + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + let id = store.create_subject(role()).unwrap().id; + let barrier = std::sync::Barrier::new(3); + std::thread::scope(|scope| { + let a = scope.spawn(|| { + barrier.wait(); + store.begin_subject_deletion(&id, "delete") + }); + let b = scope.spawn(|| { + barrier.wait(); + store.create_memory(&id, draft()) + }); + barrier.wait(); + assert!(!a.join().unwrap().unwrap().completed); + // A write either precedes begin and is later purged, or loses to the fence. + let _ = b.join().unwrap(); + }); + assert!(store.create_memory(&id, draft()).is_err()); + assert_eq!(store.pending_subject_deletions().unwrap().len(), 1); + store.finish_subject_deletion(&id, "delete").unwrap(); + assert_eq!(count(&store, "memory_records"), 0); +} + +#[test] +fn every_subject_owned_table_has_old_and_new_sql_fences_and_retention_is_scoped() { + let temp = tempfile::tempdir().unwrap(); + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + let a = store.create_subject(role()).unwrap().id; + let b = store.create_subject(role()).unwrap().id; + store + .stage_candidate(candidate(&a, "a", store.workspace_id())) + .unwrap(); + store + .stage_candidate(candidate(&b, "b", store.workspace_id())) + .unwrap(); + store.begin_subject_deletion(&a, "delete").unwrap(); + store.database.try_with_connection::<_, SubjektivError>(|c| { + let tables = c.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")?.query_map([], |r| r.get::<_,String>(0))?.collect::,_>>()?; + for table in tables { + let columns = c.prepare(&format!("PRAGMA table_info({table})"))?.query_map([], |r| r.get::<_,String>(1))?.collect::,_>>()?; + if !columns.iter().any(|v| v == "subject_id") || table.starts_with("subject_delet") { continue; } + assert!(DIRECT.contains(&table.as_str()) || table == "journal_erased_sessions", "unclassified Subject table: {table}"); + } + for table in DIRECT.iter().copied().chain(INDIRECT.iter().map(|v| v.0)).chain(["journal_erased_sessions"]) { + let sql: String = c.query_row("SELECT sql FROM sqlite_master WHERE name=?1", [format!("deletion_fence_{table}_UPDATE")], |r| r.get(0))?; + assert!(sql.contains("OLD.") && sql.contains("NEW."), "{table}: {sql}"); + } + // OLD fenced -> NEW unfenced must not evade a guard. + assert!(c.execute("UPDATE subjects SET subject_id=?2 WHERE subject_id=?1", params![a, "new-id"]).is_err()); + assert!(c.execute("UPDATE subjects SET subject_id=?2 WHERE subject_id=?1", params![b, a]).is_err()); + assert!(c.execute("DELETE FROM staging_records WHERE subject_id=?1", [&b]).is_err()); + assert!(c.execute("DELETE FROM subject_deletions WHERE subject_id=?1", [&a]).is_err()); + assert!(c.execute("UPDATE subject_deletions SET operation_id='changed' WHERE subject_id=?1", [&a]).is_err()); + Ok(()) + }).unwrap(); + store.finish_subject_deletion(&a, "delete").unwrap(); + assert!(store.staging_candidate(&b, "b").unwrap().is_some()); +} + +#[test] +fn legacy_retired_is_readable_serde_compatible_and_explicitly_deletable() { + let temp = tempfile::tempdir().unwrap(); + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + let id = store.create_subject(role()).unwrap().id; + store.create_memory(&id, draft()).unwrap(); + legacy_retired_fixture(&store, &id).unwrap(); + assert_eq!( + serde_json::from_str::("\"retired\"").unwrap(), + SubjectState::LegacyRetired + ); + assert_eq!( + serde_json::to_string(&SubjectState::LegacyRetired).unwrap(), + "\"retired\"" + ); + assert_eq!(store.list_memories(&id).unwrap().len(), 1); + assert!(store.create_memory(&id, draft()).is_err()); + store.begin_subject_deletion(&id, "delete-legacy").unwrap(); + store.finish_subject_deletion(&id, "delete-legacy").unwrap(); + assert_eq!(count(&store, "memory_changes"), 0); +} + +#[test] +fn migration_preserves_legacy_content_until_explicit_delete_including_archive() { + let temp = tempfile::tempdir().unwrap(); + let id: String; + { + let manager = FeatureStorage::new(temp.path()); + let scope = manager.workspace("workspace-a").unwrap(); + let registration = scope + .register(FeatureRegistration::new( + SUBJEKTIV_FEATURE_ID, + &MIGRATIONS[..6], + )) + .unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + id = "legacy-subject".into(); + store.database.try_transaction::<_, SubjektivError>(|tx| { + let raw = serde_json::json!({"schema_version":1,"id":id,"role":"workspace_companion","behavior_md":"legacy private behavior","state":"active","created_at":"2026-01-01","updated_at":"2026-01-01"}).to_string(); + tx.execute("INSERT INTO subjects(subject_id,role,state,store_revision,record_json,created_at,updated_at) VALUES(?1,'workspace_companion','active',0,?2,'2026-01-01','2026-01-01')", params![id,raw])?; + Ok(()) + }).unwrap(); + // Build a saved SQL fixture rather than using current-only APIs against an old schema. + store + .database + .try_transaction::<_, SubjektivError>(|tx| { + let raw: String = tx.query_row( + "SELECT record_json FROM subjects WHERE subject_id=?1", + [&id], + |r| r.get(0), + )?; + let mut value: serde_json::Value = serde_json::from_str(&raw)?; + value["state"] = "retired".into(); + tx.execute( + "UPDATE subjects SET state='retired',record_json=?2 WHERE subject_id=?1", + params![id, value.to_string()], + )?; + Ok(()) + }) + .unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + } + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + assert_eq!( + store.subject(&id).unwrap().unwrap().state, + SubjectState::LegacyRetired + ); + assert!(count(&store, "legacy_record_archive") > 1); + store.begin_subject_deletion(&id, "delete").unwrap(); + store.finish_subject_deletion(&id, "delete").unwrap(); + store.database.try_with_connection::<_, SubjektivError>(|c| { + let remaining: i64 = c.query_row("SELECT COUNT(*) FROM legacy_record_archive WHERE json_extract(row_json,'$.subject_id')=?1", [&id], |r|r.get(0))?; + assert_eq!(remaining, 0); + // Workspace-only metadata remains and is not a copy of the Subject's content. + assert_eq!(c.query_row("SELECT COUNT(*) FROM legacy_record_archive WHERE source_table='store_scope'", [], |r|r.get::<_,i64>(0))?, 1); + Ok(()) + }).unwrap(); +} + +#[test] +fn concurrent_deletions_on_independent_connections_save_one_request_identity() { + let temp = tempfile::tempdir().unwrap(); + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + let (_second_manager, _second_scope, second) = open_store(temp.path(), "workspace-a"); + let id = store.create_subject(role()).unwrap().id; + let barrier = std::sync::Barrier::new(3); + std::thread::scope(|scope| { + let a = scope.spawn(|| { + barrier.wait(); + store.begin_subject_deletion(&id, "first") + }); + let b = scope.spawn(|| { + barrier.wait(); + second.begin_subject_deletion(&id, "second") + }); + barrier.wait(); + let results = [a.join().unwrap(), b.join().unwrap()]; + assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .filter(|r| matches!(r, Err(SubjektivError::SubjectDeletionConflict(_)))) + .count(), + 1 + ); + }); + let saved = second.subject_deletion(&id).unwrap().unwrap(); + assert_eq!( + store + .begin_subject_deletion(&id, &saved.operation_id) + .unwrap(), + saved + ); + std::thread::scope(|scope| { + let a = scope.spawn(|| store.finish_subject_deletion(&id, &saved.operation_id)); + let b = scope.spawn(|| second.finish_subject_deletion(&id, &saved.operation_id)); + assert_eq!(a.join().unwrap().unwrap(), b.join().unwrap().unwrap()); + }); +} + +#[test] +fn deletion_migration_failure_rolls_back_fences_and_trigger_rewrites_then_retries() { + let temp = tempfile::tempdir().unwrap(); + let id; + { + let manager = FeatureStorage::new(temp.path()); + let scope = manager.workspace("workspace-a").unwrap(); + let registration = scope + .register(FeatureRegistration::new( + SUBJEKTIV_FEATURE_ID, + &MIGRATIONS[..13], + )) + .unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + id = store.create_subject(role()).unwrap().id; + store + .stage_candidate(candidate(&id, "preserved", store.workspace_id())) + .unwrap(); + store.database.try_with_connection::<_, SubjektivError>(|c| { + // Fail near the end, after retention triggers have already been rewritten. + c.execute_batch("CREATE TRIGGER deletion_fence_journal_sources_INSERT BEFORE INSERT ON journal_sources BEGIN SELECT 1; END;")?; + Ok(()) + }).unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + } + let manager = FeatureStorage::new(temp.path()); + let scope = manager.workspace("workspace-a").unwrap(); + let current = SubjektivStore::register(&scope).unwrap(); + assert!(SubjektivStore::open(&scope, ¤t).is_err()); + drop(scope); + manager.shutdown().unwrap(); + let manager = FeatureStorage::new(temp.path()); + let scope = manager.workspace("workspace-a").unwrap(); + // Reopen only the fixture's pre-deletion registration to inspect the failed migration. + let old = scope + .register(FeatureRegistration::new( + SUBJEKTIV_FEATURE_ID, + &MIGRATIONS[..13], + )) + .unwrap(); + let store = SubjektivStore::open(&scope, &old).unwrap(); + store + .database + .try_with_connection::<_, SubjektivError>(|c| { + assert!(!c.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deletions')", + [], + |r| r.get::<_, bool>(0) + )?); + let sql: String = c.query_row( + "SELECT sql FROM sqlite_master WHERE name='staging_records_no_delete'", + [], + |r| r.get(0), + )?; + assert!(!sql.contains("subject_deletions")); + assert!( + c.execute("DELETE FROM staging_records WHERE subject_id=?1", [&id]) + .is_err() + ); + c.execute_batch("DROP TRIGGER deletion_fence_journal_sources_INSERT;")?; + Ok(()) + }) + .unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + assert!(store.staging_candidate(&id, "preserved").unwrap().is_some()); + assert!(store.pending_subject_deletions().unwrap().is_empty()); + store.begin_subject_deletion(&id, "delete").unwrap(); + store.finish_subject_deletion(&id, "delete").unwrap(); +} diff --git a/crates/subjektiv/src/journal/binding.rs b/crates/subjektiv/src/journal/binding.rs index 3430a72cb..e80846ff0 100644 --- a/crates/subjektiv/src/journal/binding.rs +++ b/crates/subjektiv/src/journal/binding.rs @@ -39,6 +39,7 @@ fn denied() -> JournalError { } pub(super) fn check_writer(tx: &Connection, host: &JournalHostContext) -> JournalResult<()> { + crate::require_subject_in_connection(tx, &host.subject_id)?; let attr = host.attribution.as_ref().ok_or_else(denied)?; let binding = host.writer_binding_id.as_ref().ok_or_else(denied)?; if !tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings b JOIN journal_session_bindings s ON s.binding_id=b.binding_id WHERE b.binding_id=?1 AND b.subject_id=?2 AND b.runtime_id=?3 AND b.worker_id=?4 AND b.active=1 AND s.session_id=?5)", params![binding,host.subject_id,attr.runtime_id,attr.worker_id,attr.session_id], |r| r.get::<_,bool>(0))? { @@ -63,6 +64,7 @@ impl SubjektivStore { binding: &JournalWriterBinding, ) -> JournalResult> { self.database.try_with_connection(|tx| { + crate::require_subject_in_connection(tx, &binding.subject_id)?; Ok(tx.query_row("SELECT j.source_session_id FROM journals j JOIN journal_session_bindings s ON s.session_id=j.source_session_id WHERE j.subject_id=?1 AND s.binding_id=(SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=0 ORDER BY rowid DESC LIMIT 1) ORDER BY j.updated_at DESC,j.id DESC LIMIT 1", [&binding.subject_id], |r| r.get(0)).optional()?) }) } @@ -77,6 +79,7 @@ impl SubjektivStore { worker: &str, ) -> JournalResult { self.database.try_with_connection(|tx| { + crate::require_subject_in_connection(tx, subject)?; Ok(tx.query_row( "SELECT NOT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3) @@ -95,12 +98,14 @@ impl SubjektivStore { worker: &str, ) -> JournalResult { self.database.try_with_connection(|tx| { + crate::require_subject_in_connection(tx, subject)?; Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 AND active=0)", params![subject,runtime,worker], |r| r.get(0))?) }) } pub fn journal_writer_active(&self, binding: &JournalWriterBinding) -> JournalResult { self.database.try_with_connection(|tx| { + crate::require_subject_in_connection(tx, &binding.subject_id)?; Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4 AND active=1)", params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id], |r| r.get(0))?) }) } @@ -111,6 +116,7 @@ impl SubjektivStore { session: &str, ) -> JournalResult { self.database.try_with_connection(|tx| { + crate::require_subject_in_connection(tx, &binding.subject_id)?; Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_session_bindings WHERE binding_id=?1 AND session_id=?2)", params![binding.binding_id,session], |r| r.get(0))?) }) } @@ -135,6 +141,8 @@ impl SubjektivStore { } } self.database.try_transaction(|tx| { + crate::require_subject_in_connection(tx, subject)?; + crate::deletion::require_live_worker(tx, runtime, worker)?; let state: Option = tx.query_row("SELECT state FROM subjects WHERE subject_id=?1", [subject], |r| r.get(0)).optional()?; if state.as_deref() != Some("active") { return Err(JournalError::PermissionDenied("Subject is absent or retired".into())); } let current: Option = tx.query_row("SELECT subject_id,runtime_id,worker_id,binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=1", [subject], |r| Ok(JournalWriterBinding { subject_id:r.get(0)?,runtime_id:r.get(1)?,worker_id:r.get(2)?,binding_id:r.get(3)? })).optional()?; @@ -200,6 +208,7 @@ impl SubjektivStore { /// It never fixes an open entry or declares the represented work complete. pub fn release_journal_writer(&self, binding: &JournalWriterBinding) -> JournalResult<()> { self.database.try_transaction(|tx| { + crate::require_subject_in_connection(tx, &binding.subject_id)?; let active: Option = tx.query_row("SELECT active FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4",params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id],|r|r.get(0)).optional()?; let Some(active) = active else { return Err(denied()); }; if !active { return Ok(()); } diff --git a/crates/subjektiv/src/journal/memory_links/tests.rs b/crates/subjektiv/src/journal/memory_links/tests.rs index ddb9e4884..66605e943 100644 --- a/crates/subjektiv/src/journal/memory_links/tests.rs +++ b/crates/subjektiv/src/journal/memory_links/tests.rs @@ -1158,3 +1158,69 @@ fn failed_link_migration_rolls_back_ddl_and_preserves_existing_data_for_retry() assert_eq!(linked(&f.links(&selector, true).sources[0]).1, entry); assert_eq!(f.snapshot(), before); } + +#[test] +fn subject_deletion_denies_all_memory_journal_context_and_reconcile_entry_points() { + let f = Fixture::new(); + let staged = f.stage("candidate", &[0]); + let selector = f.apply(&staged); + let entry = f.fixed("entry", 0, vec![]); + f.links(&selector, true); + f.store() + .begin_subject_deletion(&f.host.subject_id, "delete") + .unwrap(); + for request in [ + MemoryJournalRequest::Links { + selector: selector.clone(), + }, + MemoryJournalRequest::Reconcile { + selector: selector.clone(), + }, + MemoryJournalRequest::LinksPage { + selector: selector.clone(), + source_offset: 0, + limit: 1, + }, + MemoryJournalRequest::ReconcilePage { + selector: selector.clone(), + source_offset: 0, + limit: 1, + }, + MemoryJournalRequest::Read { + selector, + source_index: 0, + journal_id: f.journal_id.clone(), + entry_id: entry.clone(), + expected_token: None, + byte_offset: 0, + max_bytes: 8, + }, + MemoryJournalRequest::Related { + journal_id: f.journal_id.clone(), + entry_id: entry, + after: None, + limit: 1, + }, + ] { + assert!(matches!( + f.store().execute_memory_journal(&f.host, request), + Err(JournalError::Subject( + crate::SubjektivError::SubjectNotFound(_) + )) + )); + } + f.store() + .finish_subject_deletion(&f.host.subject_id, "delete") + .unwrap(); + f.store() + .database + .try_with_connection::<_, JournalError>(|c| { + assert_eq!( + c.query_row("SELECT COUNT(*) FROM memory_journal_links", [], |r| r + .get::<_, i64>(0))?, + 0 + ); + Ok(()) + }) + .unwrap(); +} diff --git a/crates/subjektiv/src/journal/recording.rs b/crates/subjektiv/src/journal/recording.rs index 63947b90d..76985d940 100644 --- a/crates/subjektiv/src/journal/recording.rs +++ b/crates/subjektiv/src/journal/recording.rs @@ -63,6 +63,10 @@ impl JournalInput { // below operates on this value only, never by making another live capture request. let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope); input.capture = capture_once(journal.process_order, journal.process_byte, budget)?; + // Capture runs outside SQLite; a delayed callback cannot deliver deleted content. + store + .database + .try_with_connection(|tx| super::store::check_host(store, tx, host, true))?; input.selected_ranges = capture_ranges(&input.capture)?; if input.selected_ranges.is_empty() { return Ok(input); diff --git a/crates/subjektiv/src/journal/service.rs b/crates/subjektiv/src/journal/service.rs index 1829a0e18..ae0634207 100644 --- a/crates/subjektiv/src/journal/service.rs +++ b/crates/subjektiv/src/journal/service.rs @@ -469,6 +469,11 @@ fn execute_inner( Ok::<_, JournalError>(()) })?; } + // Also check empty source reads and Initial snapshot responses (no delivery write). + store.database.try_with_connection(|tx| { + super::store::job(store, tx, context, false)?; + Ok::<_, JournalError>(()) + })?; Ok(response) } diff --git a/crates/subjektiv/src/journal/store.rs b/crates/subjektiv/src/journal/store.rs index 72008f1b1..019c90a7e 100644 --- a/crates/subjektiv/src/journal/store.rs +++ b/crates/subjektiv/src/journal/store.rs @@ -105,6 +105,7 @@ pub(super) fn check_session_host( "different storage scope".into(), )); } + crate::require_subject_in_connection(tx, &host.subject_id)?; let attr = host .attribution .as_ref() @@ -134,6 +135,7 @@ pub(super) fn check_host( "different storage scope".into(), )); } + crate::require_subject_in_connection(tx, &host.subject_id)?; if write { let state: String = tx.query_row( "SELECT state FROM subjects WHERE subject_id=?1", diff --git a/crates/subjektiv/src/journal/tests.rs b/crates/subjektiv/src/journal/tests.rs index 2457fdea7..c20707123 100644 --- a/crates/subjektiv/src/journal/tests.rs +++ b/crates/subjektiv/src/journal/tests.rs @@ -349,7 +349,7 @@ mod input_delivery { open_action("must not save"), f.ranges.clone(), ); - f.store().retire_subject(&f.host.subject_id).unwrap(); + crate::tests::legacy_retired_fixture(f.store(), &f.host.subject_id).unwrap(); f.reopen(); assert!(matches!( service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges), @@ -1210,6 +1210,21 @@ mod snapshot_inputs { service::finish(f.store(), &recovery).unwrap(), saved_completion ); + // Deletion, unlike writer release, invalidates even exact saved input/completion replay. + f.store() + .begin_subject_deletion(&host.subject_id, "delete") + .unwrap(); + subject_deleted( + f.store() + .prepare_journal_input(&host, &input, generation.job.clone()), + ); + subject_deleted(service::finish(f.store(), &recovery)); + subject_deleted(JournalJobContext::validated_recovery( + f.store(), + &host, + &generation.id, + generation.job.clone(), + )); } #[test] @@ -2833,7 +2848,7 @@ fn job_service_source_reads_are_generation_bound_and_stop_at_supplied_byte_end() #[test] fn explicit_erasure_remains_available_for_retired_subject_without_reopening_recording() { let f = Fixture::new(); - f.store().retire_subject(&f.host.subject_id).unwrap(); + crate::tests::legacy_retired_fixture(f.store(), &f.host.subject_id).unwrap(); assert!(f.store().journal_retention_required(&f.host).unwrap()); f.store() .erase_journal_session(&f.host, "erase-retired") @@ -4086,10 +4101,7 @@ fn retired_subject_keeps_reader_access_but_rejects_existing_write_context() { }, vec![], ); - fixture - .store() - .retire_subject(&fixture.host.subject_id) - .unwrap(); + crate::tests::legacy_retired_fixture(fixture.store(), &fixture.host.subject_id).unwrap(); assert!(matches!( fixture.store().execute_journal(&ctx, late), Err(JournalError::PermissionDenied(_)) @@ -5745,3 +5757,311 @@ fn late_gap_admission_and_partial_updates_reject_stale_source_page_cursors() { Err(JournalError::InvalidInput(_)) )); } + +fn subject_deleted(result: JournalResult) { + assert!( + matches!( + result, + Err(JournalError::Subject( + crate::SubjektivError::SubjectNotFound(_) + )) + ), + "unexpected result: {result:?}" + ); +} + +#[test] +fn subject_deletion_fence_denies_all_journal_read_write_binding_and_recovery_paths() { + use transport::{ + JournalJobRequest as Request, JournalSnapshot, JournalTextRead, JournalTextTarget, + }; + let mut f = Fixture::new(); + let (generation, ctx) = f.generation("delete-job", f.ranges.clone()); + let operation = f.operation( + "saved", + open_action("private body"), + vec![f.ranges[0].clone()], + ); + let receipt = f.store().execute_journal(&ctx, operation.clone()).unwrap(); + let entry = receipt.entry_id.unwrap(); + let reader = JournalHostContext::validated_reader(f.store(), &f.host.subject_id).unwrap(); + let writer = f + .store() + .claim_journal_writer(&f.host.subject_id, "runtime", "worker") + .unwrap(); + let digest = format!("sha256:{}", "a".repeat(64)); + service::attest_input_delivery(f.store(), &ctx, &digest, &f.ranges).unwrap(); + f.store().database.try_transaction::<_, JournalError>(|tx| { + tx.execute("INSERT INTO journal_generation_deliveries(generation_id,source_order,ranges_json) VALUES(?1,0,'[]')", [&generation.id])?; + Ok(()) + }).unwrap(); + f.store().database.try_transaction::<_, JournalError>(|tx| { + tx.execute("INSERT INTO legacy_record_archive(source_table,row_position,row_json) VALUES('subjects',0,?1)", [serde_json::json!({"subject_id":f.host.subject_id,"record_json":"archived private body"}).to_string()])?; + Ok(()) + }).unwrap(); + f.store() + .begin_subject_deletion(&f.host.subject_id, "delete") + .unwrap(); + // Exercise actual SQL owner resolution for *every* indirect content table, + // not only table-name inventory. Fence errors must precede PK/immutable/FK errors. + f.store() + .database + .try_with_connection::<_, JournalError>(|tx| { + for (table, field, new_owner) in [ + ("journal_sources", "journal_id", "other-journal"), + ("journal_operations", "journal_id", "other-journal"), + ("journal_generations", "journal_id", "other-journal"), + ("journal_session_bindings", "binding_id", "other-binding"), + ( + "journal_generation_deliveries", + "generation_id", + "other-generation", + ), + ( + "journal_generation_input_deliveries", + "generation_id", + "other-generation", + ), + ( + "legacy_record_archive", + "row_json", + "{\"subject_id\":\"other-subject\"}", + ), + ] { + assert!( + tx.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r + .get::<_, i64>(0))? + > 0, + "{table}" + ); + let insert = tx + .execute(&format!("INSERT INTO {table} SELECT * FROM {table}"), []) + .unwrap_err(); + assert!( + insert.to_string().contains("Subject deletion has begun"), + "{table}: {insert}" + ); + let update = tx + .execute(&format!("UPDATE {table} SET {field}=?1"), [new_owner]) + .unwrap_err(); + assert!( + update.to_string().contains("Subject deletion has begun"), + "{table}: {update}" + ); + } + Ok(()) + }) + .unwrap(); + macro_rules! denied { ($($result:expr),+ $(,)?) => { $(subject_deleted($result);)+ }; } + for host in [&f.host, &reader] { + denied!( + f.store().list_journals(host, None, 1), + f.store().read_journal(host, &f.journal_id), + f.store().list_journal_entries(host, &f.journal_id, None, 1), + f.store().read_journal_text( + host, + &f.journal_id, + Some(&entry), + JournalTextField::Body, + None, + 0, + 8 + ), + f.store().read_journal_source(host, &f.journal_id, 0), + f.store().list_journal_sources(host, &f.journal_id, None, 1), + f.store() + .query_journal_operation(host, &f.journal_id, "saved"), + f.store() + .read_journal_operation_status(host, &f.journal_id, "saved"), + f.store() + .read_journal_operation_coverage(host, &f.journal_id, "saved", 0, 1), + f.store() + .list_journal_operations(host, &f.journal_id, None, 1), + f.store().journal_entry_summary(host, &f.journal_id, &entry), + f.store() + .journal_correction_targets(host, &f.journal_id, &entry), + f.store() + .browse_journal_entries(host, None, None, None, None, 1), + f.store() + .journal_has_unreflected_sources(host, &f.journal_id), + f.store().journal_startup_snapshot(host, None), + ); + } + denied!( + f.store().journal_compaction_snapshot(&f.host), + f.store().journal_session_erased(&f.host), + f.store().journal_for_session(&f.host), + f.store().journal_retention_required(&f.host), + f.store().erase_journal_session(&f.host, "erase"), + f.store().release_erased_journal_body(&f.host), + f.store().journal_predecessor_session(&writer), + f.store() + .journal_body_is_unpublished(&f.host.subject_id, "runtime", "worker"), + f.store() + .journal_worker_was_released(&f.host.subject_id, "runtime", "worker"), + f.store().journal_writer_active(&writer), + f.store() + .journal_session_writer_matches(&writer, "source-session"), + f.store() + .claim_journal_writer(&f.host.subject_id, "runtime", "worker"), + f.store() + .bind_journal_session(&writer, f.host.attribution.clone().unwrap()), + f.store().release_journal_writer(&writer), + f.store().accept_journal_sources(&f.host, "late", &f.ranges), + f.store() + .mark_journal_sources_unavailable(&f.host, &f.journal_id, &f.ranges), + f.store() + .mark_journal_sources_available(&f.host, &f.journal_id, &f.ranges), + f.store().active_journal_generation(&f.host, &f.journal_id), + f.store().prepare_journal_generation( + &f.host, + &f.journal_id, + generation.job.clone(), + f.ranges.clone() + ), + f.store() + .close_journal_generation(&f.host, &generation.id, &generation.job), + f.store().read_journal_generation(&f.host, &generation.id), + f.store().journal_resume_state(&f.host, &f.journal_id), + f.store().execute_journal(&ctx, operation.clone()), + JournalJobContext::validated_recovery( + f.store(), + &f.host, + &generation.id, + generation.job.clone() + ), + service::attest_input_delivery(f.store(), &ctx, &digest, &f.ranges), + service::finish(f.store(), &ctx), + service::execute_snapshot(f.store(), &ctx, &digest, Request::ReadState), + service::execute_snapshot( + f.store(), + &ctx, + &digest, + Request::QueryOperation { + request_id: "saved".into() + } + ), + service::execute_snapshot( + f.store(), + &ctx, + &digest, + Request::ReadText { + read: JournalTextRead { + target: JournalTextTarget::Position { + snapshot: JournalSnapshot::Initial + }, + offset: 0, + limit: 8, + content_token: None + } + } + ), + ); + // Failed purge and restart must not revive any cached context or exact receipt. + f.sql("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected'); END;").unwrap(); + assert!( + f.store() + .finish_subject_deletion(&f.host.subject_id, "delete") + .is_err() + ); + f.reopen(); + subject_deleted(f.store().execute_journal(&ctx, operation)); + f.sql("DROP TRIGGER injected_subject_purge_failure;") + .unwrap(); + f.store() + .finish_subject_deletion(&f.host.subject_id, "delete") + .unwrap(); + f.store() + .database + .try_with_connection::<_, JournalError>(|tx| { + for table in [ + "journals", + "journal_entries", + "journal_corrections", + "journal_sources", + "journal_generations", + "journal_operations", + "journal_generation_deliveries", + "journal_generation_input_deliveries", + "journal_writer_bindings", + "journal_session_bindings", + ] { + assert_eq!( + tx.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r + .get::<_, i64>(0))?, + 0, + "{table}" + ); + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn subject_deletion_during_capture_rejects_even_empty_delayed_input() { + use session_store::capture::CaptureDto; + let f = Fixture::new(); + let result = recording::JournalInput::build(f.store(), &f.host, &f.journal_id, |_, _, _| { + f.store() + .begin_subject_deletion(&f.host.subject_id, "delete")?; + Ok(CaptureDto { + version: 1, + session_id: "source-session".into(), + segment_id: "segment".into(), + view_entry_count: 0, + entries: vec![], + }) + }); + subject_deleted(result); +} + +#[test] +fn subject_deletion_during_source_callback_cannot_return_or_attest_cached_material() { + use transport::{JournalJobRequest as Request, JournalTextRead, JournalTextTarget}; + let f = Fixture::new(); + let (_, ctx) = f.generation("delayed-source", f.ranges.clone()); + let result = service::execute_bounded( + f.store(), + &ctx, + Request::ReadText { + read: JournalTextRead { + target: JournalTextTarget::Source { + pointer: f.ranges[0].source.pointer.clone(), + }, + offset: 0, + limit: 8, + content_token: None, + }, + }, + |source, start, _| { + f.store() + .begin_subject_deletion(&f.host.subject_id, "delete")?; + Ok(recording::JournalMaterial { + range: JournalSourceRange { + source: source.clone(), + bytes: JournalByteRange { + start, + end: start + 8, + }, + }, + text: "private!".into(), + }) + }, + ); + subject_deleted(result); + f.store() + .database + .try_with_connection::<_, JournalError>(|tx| { + assert_eq!( + tx.query_row( + "SELECT COUNT(*) FROM journal_generation_deliveries", + [], + |r| r.get::<_, i64>(0) + )?, + 0 + ); + Ok(()) + }) + .unwrap(); +} diff --git a/crates/subjektiv/src/lib.rs b/crates/subjektiv/src/lib.rs index 43bd67689..2928aab4a 100644 --- a/crates/subjektiv/src/lib.rs +++ b/crates/subjektiv/src/lib.rs @@ -1,7 +1,9 @@ #![doc = include_str!("../README.md")] pub mod api; +pub mod deletion; pub mod job; +pub use deletion::{DeletionSubjectJob, SubjectDeletion}; pub mod journal; mod legacy_migrations; pub mod permissions; @@ -51,7 +53,9 @@ pub enum SubjektivError { InvalidRecord(String), #[error("subject `{0}` was not found in this Workspace")] SubjectNotFound(String), - #[error("subject `{0}` is retired")] + #[error("Subject deletion request `{0}` conflicts with a saved deletion")] + SubjectDeletionConflict(String), + #[error("subject `{0}` is legacy-retired")] SubjectRetired(String), #[error("the selected Worker Profile does not enable subjektiv")] ProfileDisabled, @@ -146,7 +150,10 @@ impl<'de> Deserialize<'de> for SubjectRole { #[serde(rename_all = "snake_case")] pub enum SubjectState { Active, - Retired, + /// Saved pre-deletion lifecycle state. The old `retired` spelling remains readable + /// and serializable solely to preserve existing data; no retirement operation exists. + #[serde(rename = "retired")] + LegacyRetired, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -644,6 +651,11 @@ static MIGRATIONS: &[FeatureMigration] = &[ "bind Journal generation delivery to immutable Job input digest", journal::service::migrate_input_delivery, ), + FeatureMigration::new( + 14, + "add permanent Subject deletion fences and explicit erasure", + deletion::migrate, + ), ]; pub const REGISTRATION: FeatureRegistration = @@ -772,6 +784,9 @@ impl SubjektivStore { pub fn subject(&self, subject_id: &str) -> Result> { self.database.try_with_connection(|connection| { + if deletion::is_deleting(connection, subject_id)? { + return Ok(None); + } let raw = connection .query_row( "SELECT record_json FROM subjects WHERE subject_id = ?1", @@ -862,11 +877,13 @@ impl SubjektivStore { self.database.try_with_connection(|connection| { let sql = if after.is_some() { "SELECT record_json FROM subjects - WHERE created_at > ?1 OR (created_at = ?1 AND subject_id > ?2) + WHERE NOT EXISTS (SELECT 1 FROM subject_deletions d WHERE d.subject_id=subjects.subject_id) + AND (created_at > ?1 OR (created_at = ?1 AND subject_id > ?2)) ORDER BY created_at ASC, subject_id ASC LIMIT ?3" } else { "SELECT record_json FROM subjects + WHERE NOT EXISTS (SELECT 1 FROM subject_deletions d WHERE d.subject_id=subjects.subject_id) ORDER BY created_at ASC, subject_id ASC LIMIT ?1" }; @@ -895,27 +912,6 @@ impl SubjektivStore { }) } - /// Retires a Subject independently of stopping Workers. Memory/history remain unedited, undeleted, and readable. - /// Retirement is irreversible; new candidates and Memory changes require an active Subject. - pub fn retire_subject(&self, subject_id: &str) -> Result { - self.database.try_transaction(|transaction| { - let mut subject = require_subject(transaction, subject_id)?; - if subject.state == SubjectState::Retired { - return Ok(subject); - } - subject.state = SubjectState::Retired; - subject.updated_at = now(); - let raw = serde_json::to_string(&subject)?; - transaction.execute( - "UPDATE subjects - SET state = 'retired', record_json = ?2, updated_at = ?3 - WHERE subject_id = ?1", - params![subject_id, raw, subject.updated_at], - )?; - Ok(subject) - }) - } - /// Saves an immutable candidate without changing committed Memory. /// Replay with the same candidate identity and payload returns the original record; different content conflicts. /// Extraction failure after saving never revokes the candidate. The Host commits the extraction pointer separately. @@ -1003,6 +999,10 @@ impl SubjektivStore { ) -> Result> { validate_label("session id", session_id)?; self.database.try_with_connection(|connection| { + if let Some(subject) = deletion::deleted_session_subject(connection, session_id)? { + return Err(SubjektivError::SubjectNotFound(subject)); + } + let raw = connection .query_row( "SELECT record_json FROM subject_session_attributions @@ -1011,7 +1011,12 @@ impl SubjektivStore { |row| row.get::<_, String>(0), ) .optional()?; - raw.map(|raw| parse_session_attribution(&raw)).transpose() + raw.map(|raw| { + let record = parse_session_attribution(&raw)?; + deletion::require_not_deleting(connection, &record.subject_id)?; + Ok(record) + }) + .transpose() }) } @@ -1021,6 +1026,7 @@ impl SubjektivStore { ) -> Result> { validate_label("subject id", subject_id)?; self.database.try_with_connection(|connection| { + require_subject_in_connection(connection, subject_id)?; let mut statement = connection.prepare( "SELECT record_json FROM subject_session_attributions WHERE subject_id = ?1 @@ -1062,6 +1068,7 @@ impl SubjektivStore { } let query_limit = limit.saturating_add(1); self.database.try_with_connection(|connection| { + require_subject_in_connection(connection, subject_id)?; let raw = match (session_id, after) { (Some(session_id), _) => { let mut statement = connection.prepare( @@ -1128,6 +1135,7 @@ impl SubjektivStore { candidate_id: &str, ) -> Result> { self.database.try_with_connection(|connection| { + deletion::require_not_deleting(connection, subject_id)?; let raw = connection .query_row( "SELECT record_json FROM staging_records @@ -1292,6 +1300,7 @@ impl SubjektivStore { validate_nonempty("resolution reason", &request.reason)?; let request_raw = serde_json::to_string(&request)?; self.database.try_transaction(|transaction| { + deletion::require_not_deleting(transaction, subject_id)?; if let Some((stored_request, stored_receipt)) = transaction .query_row( "SELECT request_json, receipt_json @@ -1426,6 +1435,7 @@ impl SubjektivStore { candidate_id: &str, ) -> Result> { self.database.try_with_connection(|connection| { + deletion::require_not_deleting(connection, subject_id)?; let raw = connection .query_row( "SELECT resolution_json FROM staging_resolutions @@ -1440,6 +1450,7 @@ impl SubjektivStore { pub fn memory(&self, subject_id: &str, memory_id: &str) -> Result> { self.database.try_with_connection(|connection| { + deletion::require_not_deleting(connection, subject_id)?; let raw = connection .query_row( "SELECT record_json FROM memory_records @@ -1459,6 +1470,7 @@ impl SubjektivStore { change_id: String, ) -> Result> { self.database.try_with_connection(|connection| { + deletion::require_not_deleting(connection, subject_id)?; let raw = connection .query_row( "SELECT record_json FROM memory_changes @@ -1854,6 +1866,7 @@ impl SubjektivStore { snapshot_id: &str, ) -> Result> { self.database.try_with_connection(|connection| { + deletion::require_not_deleting(connection, subject_id)?; let raw = connection .query_row( "SELECT snapshot_json FROM surface_snapshots @@ -2427,6 +2440,11 @@ fn write_session_attribution( record: SubjectSessionAttribution, raw: String, ) -> Result { + deletion::require_live_worker(transaction, &record.runtime_id, &record.worker_id)?; + if let Some(subject) = deletion::deleted_session_subject(transaction, &record.session_id)? { + return Err(SubjektivError::SubjectNotFound(subject)); + } + let existing = transaction .query_row( "SELECT record_json FROM subject_session_attributions WHERE session_id = ?1", @@ -2621,6 +2639,7 @@ fn require_subject_in_connection( connection: &rusqlite::Connection, subject_id: &str, ) -> Result { + deletion::require_not_deleting(connection, subject_id)?; let raw = connection .query_row( "SELECT record_json FROM subjects WHERE subject_id = ?1", @@ -2637,7 +2656,7 @@ fn require_active_subject( subject_id: &str, ) -> Result { let subject = require_subject(transaction, subject_id)?; - if subject.state == SubjectState::Retired { + if subject.state == SubjectState::LegacyRetired { return Err(SubjektivError::SubjectRetired(subject_id.to_string())); } Ok(subject) @@ -3278,6 +3297,7 @@ fn require_job_generation( job_id: &str, attempt_id: &str, ) -> Result { + deletion::require_not_deleting(connection, subject_id)?; let change_id: Option = connection.query_row( "SELECT memory_fingerprint FROM surface_generation_runs WHERE subject_id = ?1 AND generation_id = ?2 AND job_id = ?3 AND attempt_id = ?4", params![subject_id, generation_id, job_id, attempt_id], |row| row.get(0), @@ -3370,6 +3390,21 @@ mod tests { use super::*; use feature_storage::FeatureStorage; + /// Builds saved pre-deletion lifecycle data; not a current product operation. + pub(crate) fn legacy_retired_fixture( + store: &SubjektivStore, + subject_id: &str, + ) -> Result { + store.database.try_transaction(|tx| { + let mut subject = require_subject(tx, subject_id)?; + subject.state = SubjectState::LegacyRetired; + subject.updated_at = now(); + tx.execute("UPDATE subjects SET state='retired',record_json=?2,updated_at=?3 WHERE subject_id=?1", + params![subject_id,serde_json::to_string(&subject)?,subject.updated_at])?; + Ok(subject) + }) + } + pub(super) fn role() -> SubjectRole { SubjectRole::new("workspace_companion").unwrap() } @@ -5047,8 +5082,8 @@ mod tests { Err(SubjektivError::InvalidStateTransition { .. }) )); - let retired = store.retire_subject(&subject.id).unwrap(); - assert_eq!(retired.state, SubjectState::Retired); + let retired = crate::tests::legacy_retired_fixture(&store, &subject.id).unwrap(); + assert_eq!(retired.state, SubjectState::LegacyRetired); assert!(matches!( store.create_memory(&subject.id, draft("after retirement", "invalid")), Err(SubjektivError::SubjectRetired(_)) diff --git a/crates/subjektiv/src/permissions.rs b/crates/subjektiv/src/permissions.rs index 06e9df5a3..5af302b5d 100644 --- a/crates/subjektiv/src/permissions.rs +++ b/crates/subjektiv/src/permissions.rs @@ -94,6 +94,10 @@ impl SubjektivStore { pub fn bind_worker_execution_grant(&self, grant: &WorkerExecutionGrant) -> Result<()> { validate_grant(self.scope_id(), grant)?; self.database.try_transaction(|tx| { + require_subject(tx, &grant.subject_id)?; + for (runtime, worker) in [(&grant.runtime_id, &grant.worker_id), (&grant.origin_runtime_id, &grant.origin_worker_id)] { + if tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)", params![runtime,worker], |r|r.get::<_,bool>(0))? { return Err(denied("Worker belongs to a deleted Subject")); } + } if let Some(existing) = grant_in_connection(tx, &grant.runtime_id, &grant.worker_id)? { return if existing == *grant { Ok(()) } else { Err(SubjektivError::WorkerExecutionGrantConflict(format!("{}/{}", grant.runtime_id, grant.worker_id))) @@ -184,9 +188,12 @@ fn grant_in_connection( runtime: &str, worker: &str, ) -> Result> { + // Absence must never become non-Subject fallback authority after purge. + deletion::require_live_worker(connection, runtime, worker)?; + let raw: Option = connection .query_row( - "SELECT record_json FROM worker_execution_grants WHERE runtime_id=?1 AND worker_id=?2", + "SELECT record_json FROM worker_execution_grants WHERE runtime_id=?1 AND worker_id=?2 AND NOT EXISTS(SELECT 1 FROM subject_deletions d WHERE d.subject_id=worker_execution_grants.subject_id)", params![runtime, worker], |row| row.get(0), ) diff --git a/crates/subjektiv/src/permissions/tests.rs b/crates/subjektiv/src/permissions/tests.rs index ab426bf2a..af859b04f 100644 --- a/crates/subjektiv/src/permissions/tests.rs +++ b/crates/subjektiv/src/permissions/tests.rs @@ -315,7 +315,7 @@ fn redelegation_requires_current_source_and_explicit_retained_parent_ceiling() { store.worker_execution_grant("runtime", "child").unwrap(), Some(child) ); - store.retire_subject(&subject).unwrap(); + crate::tests::legacy_retired_fixture(&store, &subject).unwrap(); stale.source_change_id = revoked.change_id; assert!(store.bind_worker_execution_grant(&stale).is_err()); } @@ -582,7 +582,7 @@ fn subject_grant_listing_is_scoped_retained_ordered_and_fails_on_overflow() { &update("revoke", Some(&source.change_id), Vec::new()), ) .unwrap(); - store.retire_subject(&subject).unwrap(); + crate::tests::legacy_retired_fixture(&store, &subject).unwrap(); let listed = store .list_subject_worker_execution_grants(&subject) .unwrap(); @@ -654,3 +654,51 @@ fn failed_schema_migration_rolls_back_new_tables_and_preserves_old_data() { Ok(()) }).unwrap(); } + +#[test] +fn deletion_fences_source_and_child_grant_replays_and_never_falls_back_after_purge() { + let temp = tempfile::tempdir().unwrap(); + let (_manager, _scope, store) = open_store(temp.path(), "workspace-a"); + let subject = store.create_subject(role()).unwrap().id; + let replacement = update("allow", None, vec![permission("file.read", ".", true)]); + let source = store + .set_subject_permissions(&subject, &replacement) + .unwrap(); + let child = grant(&subject, &source.change_id, "child"); + store.bind_worker_execution_grant(&child).unwrap(); + store.begin_subject_deletion(&subject, "delete").unwrap(); + assert!(store.subject_permissions(&subject).is_err()); + assert!( + store + .set_subject_permissions(&subject, &replacement) + .is_err() + ); + assert!(store.bind_worker_execution_grant(&child).is_err()); + assert!(store.worker_execution_grant("runtime", "child").is_err()); + assert!(store.worker_subject_deleted("runtime", "body").unwrap()); + assert!(store.worker_subject_deleted("runtime", "child").unwrap()); + assert_eq!(store.deletion_subject_workers(&subject).unwrap().len(), 2); + store.finish_subject_deletion(&subject, "delete").unwrap(); + assert!(store.worker_execution_grant("runtime", "child").is_err()); + assert!(store.worker_execution_grant("runtime", "body").is_err()); + assert!( + store + .worker_execution_grant("runtime", "unrelated") + .unwrap() + .is_none() + ); + let fresh = store.create_subject(role()).unwrap().id; + assert!(store.subject_permissions(&fresh).unwrap().is_none()); + assert!( + store + .record_session_attribution( + SubjectSessionAttribution::new(&fresh, "runtime", "child", "new-session").unwrap() + ) + .is_err() + ); + assert!( + store + .claim_journal_writer(&fresh, "runtime", "body") + .is_err() + ); +} diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index f4c7d85e9..72c22e46e 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -727,6 +727,23 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { Err("execution backend cannot confirm Subject create execution absence".into()) } + /// Read-only exact execution identity check before Runtime persists exclusion. + fn preflight_subject_delete_worker( + &self, + _request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + Err("execution backend cannot validate Subject Worker identity".into()) + } + + /// Runtime has durably fenced admission. Confirm shutdown and discard all + /// execution candidates under the backend Worker lock; uncertainty must fail. + fn subject_delete_worker( + &self, + _request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + Err("execution backend cannot confirm Subject Worker shutdown".into()) + } + /// Must reserve the actual Controller, not infer Idle from a catalog snapshot. fn subject_release( &self, @@ -1069,6 +1086,20 @@ impl WorkerExecutionBackendRef { self.backend.recover_subject_create(request) } + pub(crate) fn preflight_subject_delete_worker( + &self, + request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + self.backend.preflight_subject_delete_worker(request) + } + + pub(crate) fn subject_delete_worker( + &self, + request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + self.backend.subject_delete_worker(request) + } + pub(crate) fn subject_release( &self, request: &runtime_api::SubjectReleaseRequest, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index cbc549383..fe750e2ff 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -43,7 +43,8 @@ use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; #[path = "test_support.rs"] pub mod test_support; -pub(crate) const SCHEMA_VERSION: u32 = 14; +pub(crate) const SCHEMA_VERSION: u32 = 15; +const RESTORE_REPAIR_SCHEMA_VERSION: u32 = 14; const RESTORE_DECISION_SCHEMA_VERSION: u32 = 13; const RESTORE_ADMISSION_SCHEMA_VERSION: u32 = 12; // Frozen source schemas: do not move their meaning when adding a migration. @@ -560,6 +561,22 @@ impl FsRuntimeStore { self.root.join(RUNTIME_FILE) } + pub(crate) fn worker_originals_absent( + &self, + worker_id: WorkerId, + ) -> Result { + let path = self.worker_dir(&worker_id); + match fs::symlink_metadata(&path) { + Ok(_) => Ok(false), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + Err(source) => Err(RuntimeError::StoreIo { + operation: "confirm Subject Worker original absence", + path, + source, + }), + } + } + fn worker_dir(&self, worker_id: &WorkerId) -> PathBuf { self.root.join(WORKERS_DIR).join(worker_id.to_string()) } @@ -659,6 +676,8 @@ pub(crate) struct OpenedFsRuntimeStore { #[derive(Clone, Debug)] pub(crate) struct PersistedRuntimeState { + pub(crate) subject_delete_worker_fences: + BTreeMap, pub(crate) subject_create_recovery_fences: BTreeMap, pub(crate) subject_releases: Vec<( @@ -876,11 +895,12 @@ fn plan_runtime_store_migration( | SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION | RESTORE_DECISION_SCHEMA_VERSION + | RESTORE_REPAIR_SCHEMA_VERSION ) { return Err(runtime_store_corrupt( &runtime_path, format!( - "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, or {SCHEMA_VERSION}" + "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, {RESTORE_REPAIR_SCHEMA_VERSION}, or {SCHEMA_VERSION}" ), )); } @@ -1216,6 +1236,7 @@ fn migrate_worker_record( | SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION | RESTORE_DECISION_SCHEMA_VERSION + | RESTORE_REPAIR_SCHEMA_VERSION ) { // Admission identity survives migration. Older pending records did not // distinguish a committed decision from preparation or failed cleanup. @@ -1229,10 +1250,12 @@ fn migrate_worker_record( "Worker schema does not match Runtime migration source".into(), )); } - if source_schema_version != RESTORE_DECISION_SCHEMA_VERSION { + if source_schema_version < RESTORE_DECISION_SCHEMA_VERSION { migrate_restore_commit_fact(&mut identity_document, identity_path)?; } - migrate_restore_repair_fact(&mut identity_document, identity_path)?; + if source_schema_version < RESTORE_REPAIR_SCHEMA_VERSION { + migrate_restore_repair_fact(&mut identity_document, identity_path)?; + } identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION); let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document) .map_err(|error| runtime_store_corrupt(identity_path, error.to_string()))?; @@ -1691,6 +1714,9 @@ fn migrate_runtime_document( "schema_version".to_string(), serde_json::Value::from(SCHEMA_VERSION), ); + object + .entry("subject_delete_worker_fences") + .or_insert_with(|| serde_json::json!({})); // Intermediate schema-11 writers may already have saved irreversible fences. // Default only a missing field; malformed or existing fences must not be erased. object @@ -1698,7 +1724,10 @@ fn migrate_runtime_document( .or_insert_with(|| serde_json::json!({})); if !matches!( source_schema_version, - SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION | RESTORE_DECISION_SCHEMA_VERSION + SOURCE_SCHEMA_VERSION + | RESTORE_ADMISSION_SCHEMA_VERSION + | RESTORE_DECISION_SCHEMA_VERSION + | RESTORE_REPAIR_SCHEMA_VERSION ) { object.remove("workers"); // This legacy cache is not Runtime lifecycle or restore authority. Bundles @@ -2249,6 +2278,7 @@ fn migrate_runtime_store_in_place( SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION | RESTORE_DECISION_SCHEMA_VERSION + | RESTORE_REPAIR_SCHEMA_VERSION ) { atomic_write_json( @@ -2288,6 +2318,7 @@ fn migrate_runtime_store_in_place( #[derive(Clone, Debug, Serialize, Deserialize)] struct RuntimeSnapshot { + subject_delete_worker_fences: BTreeMap, #[serde(default)] subject_create_recovery_fences: BTreeMap, #[serde(default)] @@ -2326,6 +2357,7 @@ fn record_worker_load_diagnostic( impl RuntimeSnapshot { fn from_persisted(state: &PersistedRuntimeState) -> Self { Self { + subject_delete_worker_fences: state.subject_delete_worker_fences.clone(), subject_create_recovery_fences: state.subject_create_recovery_fences.clone(), subject_releases: state.subject_releases.clone(), schema_version: SCHEMA_VERSION, @@ -2350,6 +2382,33 @@ impl RuntimeSnapshot { ), }); } + for (worker_id, fence) in &self.subject_delete_worker_fences { + let request = &fence.request; + let response_matches = fence.response.as_ref().is_none_or(|response| { + response.worker_id == request.worker_id + && response.workspace_id == request.workspace_id + && response.create_fingerprint == request.create_fingerprint + && response.operation_id == request.operation_id + }); + if *worker_id != request.worker_id + || request.workspace_id.trim().is_empty() + || request.create_fingerprint.trim().is_empty() + || request.create_fingerprint.len() > 512 + || request.operation_id.trim().is_empty() + || request.operation_id.len() > 512 + || self + .workspace_owners + .get(&request.workspace_id) + .is_none_or(|owner| owner.trim().is_empty()) + || !response_matches + { + return Err(RuntimeError::StoreCorrupt { + operation: "read Subject delete Worker fence", + path: path.to_path_buf(), + message: "invalid deletion identity, receipt or Workspace owner".into(), + }); + } + } for (worker_id, fence) in &self.subject_create_recovery_fences { let request = &fence.request; let response_matches = fence.response.as_ref().is_none_or(|response| { @@ -2392,6 +2451,7 @@ impl RuntimeSnapshot { workers: BTreeMap, ) -> PersistedRuntimeState { PersistedRuntimeState { + subject_delete_worker_fences: self.subject_delete_worker_fences, subject_create_recovery_fences: self.subject_create_recovery_fences, subject_releases: self.subject_releases, display_name: self.display_name, @@ -3218,8 +3278,7 @@ mod tests { assert_eq!(fs::read(&worker_path).unwrap(), worker_before); // The direct reader still rejects a previous schema: migration is // explicit, not permissive deserialization of a newer format. - let snapshot: RuntimeSnapshot = serde_json::from_value(runtime.clone()).unwrap(); - assert!(snapshot.validate(&runtime_path).is_err()); + assert!(serde_json::from_value::(runtime.clone()).is_err()); let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap(); let state = opened.state.as_ref().unwrap(); @@ -3231,6 +3290,7 @@ mod tests { expected_worker ); runtime["schema_version"] = serde_json::json!(SCHEMA_VERSION); + runtime["subject_delete_worker_fences"] = serde_json::json!({}); if fence_state == "absent" { runtime["subject_create_recovery_fences"] = serde_json::json!({}); } @@ -4072,6 +4132,8 @@ mod tests { ); } + include!("subject_delete_store_tests.rs"); + fn canonical_worker_store() -> ( tempfile::TempDir, FsRuntimeStore, diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 60bf0bf5f..7fd7880aa 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -2102,7 +2102,11 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s { return Some("workers:input"); } - if path == "/v1/subject-create-recovery" && *method == Method::POST { + if matches!( + path, + "/v1/subject-create-recovery" | "/v1/subject-delete-worker" + ) && *method == Method::POST + { return Some("workers:delete"); } if path.ends_with("/stop") @@ -3332,6 +3336,18 @@ mod tests { struct AcceptingBackend; impl WorkerExecutionBackend for AcceptingBackend { + fn preflight_subject_delete_worker( + &self, + _request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + Ok(()) + } + fn subject_delete_worker( + &self, + _request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + Ok(()) + } fn recover_subject_create( &self, _request: &runtime_api::SubjectCreateRecoveryRequest, diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index ff8585e1a..3ac26499b 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { } } + async fn subject_delete_worker( + &self, + request: runtime_api::SubjectDeleteWorkerRequest, + ) -> Result { + let scope = material_scope(&self.state, &request.workspace_id)?; + self.state + .runtime + .subject_delete_worker_scoped(&scope, request) + .map_err(RuntimeHttpRestError::runtime) + .map_err(api_error) + } + async fn recover_subject_create( &self, request: runtime_api::SubjectCreateRecoveryRequest, diff --git a/crates/worker-runtime/src/http_server/tests/session_material_tests.rs b/crates/worker-runtime/src/http_server/tests/session_material_tests.rs index f47845f92..0a89c9c53 100644 --- a/crates/worker-runtime/src/http_server/tests/session_material_tests.rs +++ b/crates/worker-runtime/src/http_server/tests/session_material_tests.rs @@ -1222,6 +1222,54 @@ async fn subject_create_recovery_signed_http_requires_delete_and_matching_worksp } } +#[tokio::test] +async fn subject_delete_worker_signed_http_requires_delete_and_matching_workspace() { + let f = Fixture::new(); + let identity = RuntimeIdentityMaterial::generate("recovery-key").unwrap(); + let trust = workspace_trust(&identity); + let app = runtime_http_router_with_workspace_auth( + f.runtime.clone(), + None, + workspace_test_auth(vec![trust.clone()]), + ); + let path = "/v1/subject-delete-worker"; + let mut request = runtime_api::SubjectDeleteWorkerRequest { + workspace_id: f.scope.workspace_id.clone(), + worker_id: WorkerId::now_v7(), + create_fingerprint: "unpublished-create".into(), + operation_id: "recovery-1".into(), + }; + for operation in ["workers:read", "workers:stop", "workers:create"] { + let body = serde_json::to_vec(&request).unwrap(); + let mut claims = signed_claims(&trust, "POST", path, &body); + claims.operation = operation.into(); + let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + request.workspace_id = "other-workspace".into(); + let body = serde_json::to_vec(&request).unwrap(); + let claims = signed_claims(&trust, "POST", path, &body); + let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + request.workspace_id = f.scope.workspace_id.clone(); + for _ in 0..2 { + let body = serde_json::to_vec(&request).unwrap(); + let claims = signed_claims(&trust, "POST", path, &body); + assert_eq!(claims.operation, "workers:delete"); + let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await; + assert_eq!(response.status(), StatusCode::OK); + let bytes = axum::body::to_bytes(response.into_body(), 16384) + .await + .unwrap(); + let receipt: runtime_api::SubjectDeleteWorkerResponse = + serde_json::from_slice(&bytes).unwrap(); + assert_eq!(receipt.worker_id, request.worker_id); + assert_eq!(receipt.workspace_id, request.workspace_id); + assert_eq!(receipt.create_fingerprint, request.create_fingerprint); + assert_eq!(receipt.operation_id, request.operation_id); + } +} + #[tokio::test] async fn explicit_material_erasure_requires_delete_capability_and_matching_scope() { let f = Fixture::new(); diff --git a/crates/worker-runtime/src/retention/runtime_v5_tests.rs b/crates/worker-runtime/src/retention/runtime_v5_tests.rs index b14a77e80..064caecc9 100644 --- a/crates/worker-runtime/src/retention/runtime_v5_tests.rs +++ b/crates/worker-runtime/src/retention/runtime_v5_tests.rs @@ -97,7 +97,7 @@ fn runtime_fixture(root: &Path, schema: u32) -> (WorkerId, SessionId, SegmentId) "schema_version": schema, "display_name": "retained Runtime", "backend": "fs_store", "status": "running", "next_diagnostic_id": 1, "config_bundles": {}, "workspace_owners": {"workspace-a": "owner-a"}, - "diagnostics": [], "subject_releases": [], + "diagnostics": [], "subject_releases": [], "subject_delete_worker_fences": {}, "subject_create_recovery_fences": {fenced_worker.to_string(): { "request": {"workspace_id": "workspace-a", "worker_id": fenced_worker, "create_fingerprint": "retained-create", "operation_id": "retained-recovery"}, diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 57b70bc8e..71583069d 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1,3 +1,5 @@ +mod subject_delete; +pub(crate) use subject_delete::SubjectDeleteWorkerFence; #[cfg(feature = "restore-checkpoints")] mod restore_checkpoint; mod restore_continuation; @@ -5645,6 +5647,7 @@ struct RuntimeState { // Ephemeral write barrier, not authority: only canonical reread + sync can // release it. Prevents another mutation from persisting an older projection. unreconciled_worker_writes: std::collections::BTreeSet, + subject_delete_worker_fences: BTreeMap, subject_create_recovery_fences: BTreeMap, subject_release_operations: BTreeMap< (WorkerId, String), @@ -5686,6 +5689,7 @@ impl RuntimeState { next_diagnostic_id: 1, workers: BTreeMap::new(), unreconciled_worker_writes: Default::default(), + subject_delete_worker_fences: BTreeMap::new(), subject_create_recovery_fences: BTreeMap::new(), subject_release_operations: BTreeMap::new(), workdir_operations: BTreeMap::new(), @@ -5722,6 +5726,7 @@ impl RuntimeState { next_diagnostic_id: 1, workers: BTreeMap::new(), unreconciled_worker_writes: Default::default(), + subject_delete_worker_fences: BTreeMap::new(), subject_create_recovery_fences: BTreeMap::new(), subject_release_operations: BTreeMap::new(), workdir_operations: BTreeMap::new(), @@ -5773,6 +5778,7 @@ impl RuntimeState { next_diagnostic_id, workers, unreconciled_worker_writes: Default::default(), + subject_delete_worker_fences: persisted.subject_delete_worker_fences, subject_create_recovery_fences: persisted.subject_create_recovery_fences, subject_release_operations: persisted .subject_releases @@ -5804,6 +5810,7 @@ impl RuntimeState { #[cfg(feature = "fs-store")] fn persisted_state(&self) -> PersistedRuntimeState { PersistedRuntimeState { + subject_delete_worker_fences: self.subject_delete_worker_fences.clone(), subject_create_recovery_fences: self.subject_create_recovery_fences.clone(), subject_releases: self.subject_release_operations.values().cloned().collect(), display_name: self.display_name.clone(), @@ -6027,7 +6034,9 @@ impl RuntimeState { } fn ensure_subject_create_not_fenced(&self, worker_id: WorkerId) -> Result<(), RuntimeError> { - if self.subject_create_recovery_fences.contains_key(&worker_id) { + if self.subject_create_recovery_fences.contains_key(&worker_id) + || self.subject_delete_worker_fences.contains_key(&worker_id) + { return Err(RuntimeError::InvalidRequest( "Subject create identity is irreversibly fenced".into(), )); @@ -6040,6 +6049,14 @@ impl RuntimeState { worker: &WorkerRef, allow_disconnected: bool, ) -> Result<(), RuntimeError> { + if allow_disconnected + && self + .subject_delete_worker_fences + .get(&worker.worker_id) + .is_some_and(|fence| fence.response.is_some()) + { + return Ok(()); + } self.ensure_subject_create_not_fenced(worker.worker_id)?; use runtime_api::SubjectReleaseState as State; if self @@ -6110,6 +6127,13 @@ impl RuntimeState { } fn forget_workspace_owner_if_unused(&mut self, workspace_id: &str) -> bool { + if self + .subject_delete_worker_fences + .values() + .any(|fence| fence.request.workspace_id == workspace_id) + { + return false; + } if self .subject_create_recovery_fences .values() @@ -8767,6 +8791,8 @@ mod tests { #[derive(Default)] struct TestExecutionBackend { + deletion_confirmed: AtomicBool, + deletion_checks: std::sync::atomic::AtomicUsize, recovery_absent: AtomicBool, recovery_checks: std::sync::atomic::AtomicUsize, spawn_result: Mutex>, @@ -8848,6 +8874,23 @@ mod tests { } impl WorkerExecutionBackend for TestExecutionBackend { + fn preflight_subject_delete_worker( + &self, + _request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + Ok(()) + } + fn subject_delete_worker( + &self, + _request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + self.deletion_checks.fetch_add(1, Ordering::SeqCst); + if self.deletion_confirmed.load(Ordering::SeqCst) { + Ok(()) + } else { + Err("shutdown is unconfirmed".into()) + } + } fn recover_subject_create( &self, _request: &runtime_api::SubjectCreateRecoveryRequest, @@ -14074,7 +14117,7 @@ mod tests { }) .unwrap_err(); assert!(error.to_string().contains( - "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, or 14" + "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, 14, or 15" )); let _ = std::fs::remove_dir_all(root); @@ -14259,6 +14302,8 @@ mod tests { let _ = std::fs::remove_dir_all(root); } + include!("runtime/subject_delete_tests.rs"); + #[cfg(feature = "fs-store")] #[test] fn subject_create_recovery_is_durable_exact_and_rechecks_absence() { diff --git a/crates/worker-runtime/src/runtime/subject_delete.rs b/crates/worker-runtime/src/runtime/subject_delete.rs new file mode 100644 index 000000000..485faf258 --- /dev/null +++ b/crates/worker-runtime/src/runtime/subject_delete.rs @@ -0,0 +1,168 @@ +use super::*; + +/// Permanent exclusion survives WorkerRemoval. A receipt never substitutes for a +/// fresh execution reconciliation on exact retry, including after Runtime reopen. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SubjectDeleteWorkerFence { + pub(crate) request: runtime_api::SubjectDeleteWorkerRequest, + pub(crate) response: Option, +} + +impl Runtime { + pub fn subject_delete_worker_scoped( + &self, + scope: &RuntimeWorkspaceScope, + request: runtime_api::SubjectDeleteWorkerRequest, + ) -> Result { + if request.workspace_id != scope.workspace_id + || request.create_fingerprint.trim().is_empty() + || request.operation_id.trim().is_empty() + || request.create_fingerprint.len() > 512 + || request.operation_id.len() > 512 + { + return Err(RuntimeError::InvalidRequest( + "invalid Subject delete Worker identity".into(), + )); + } + let lease = self.worker_operation_lock(request.worker_id)?; + let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?; + let worker_ref = WorkerRef::new(request.worker_id); + let backend = { + let mut state = self.lock()?; + state.ensure_running()?; + if state.backend == RuntimeBackendKind::Memory { + return Err(RuntimeError::InvalidRequest( + "Subject delete requires durable storage".into(), + )); + } + state.ensure_workspace_owner(scope, false)?; + if state.workers.contains_key(&request.worker_id) { + state.refresh_uncertain_worker_write(&worker_ref)?; + let worker = state.worker(&worker_ref)?; + if worker.workspace_id.as_deref() != Some(request.workspace_id.as_str()) + || worker.request.as_ref().is_none_or(|create| { + create.create_fingerprint != request.create_fingerprint + }) + { + return Err(RuntimeError::InvalidRequest( + "Subject delete Worker identity mismatch".into(), + )); + } + } + if let Some(recovery) = state.subject_create_recovery_fences.get(&request.worker_id) + && (recovery.request.workspace_id != request.workspace_id + || recovery.request.create_fingerprint != request.create_fingerprint) + { + return Err(RuntimeError::InvalidRequest( + "Subject recovery identity mismatch".into(), + )); + } + if state.subject_delete_worker_fences.values().any(|fence| { + fence.request.workspace_id == request.workspace_id + && fence.request.operation_id == request.operation_id + && fence.request.worker_id != request.worker_id + }) { + return Err(RuntimeError::InvalidRequest( + "Subject delete operation reused for another Worker".into(), + )); + } + state.execution_backend.clone() + }; + // Unpublished execution identity is checked without holding the catalog + // mutex. The Worker operation lock excludes Runtime create/restore. + if let Some(backend) = backend.as_ref() { + backend + .preflight_subject_delete_worker(&request) + .map_err(RuntimeError::InvalidRequest)?; + } + { + let mut state = self.lock()?; + state.ensure_workspace_owner(scope, false)?; + if state.subject_delete_worker_fences.values().any(|fence| { + fence.request.workspace_id == request.workspace_id + && fence.request.operation_id == request.operation_id + && fence.request.worker_id != request.worker_id + }) { + return Err(RuntimeError::InvalidRequest( + "Subject delete operation reused for another Worker".into(), + )); + } + if let Some(saved) = state.subject_delete_worker_fences.get(&request.worker_id) { + if saved.request != request { + return Err(RuntimeError::InvalidRequest( + "different Subject delete Worker request retained".into(), + )); + } + } else { + state.ensure_workspace_owner(scope, true)?; + state.subject_delete_worker_fences.insert( + request.worker_id, + SubjectDeleteWorkerFence { + request: request.clone(), + response: None, + }, + ); + } + // Keep exclusion on persistence uncertainty; never start execution + // cleanup until this durable write has been confirmed. + state.persist_runtime_snapshot()?; + } + let backend = backend.ok_or_else(|| { + RuntimeError::InvalidRequest( + "Subject Worker shutdown cannot be confirmed without execution backend".into(), + ) + })?; + backend + .subject_delete_worker(&request) + .map_err(RuntimeError::InvalidRequest)?; + let mut state = self.lock()?; + if let Some(mut worker) = state.workers.get(&request.worker_id).cloned() { + let restore_receipt = + RuntimeWorkerRestoreResult::superseded_by_stop(worker.pending_restore); + worker.pending_stop = None; + worker.pending_restore = None; + if let Some(context) = worker.restore_candidate_context.take() { + context.discard_candidate(); + } + worker.restore_guard.rotate(); + worker.restore_guard.finish(restore_receipt); + worker.status = WorkerStatus::Stopped; + worker.worker_state = None; + worker.restore_intent = WorkerRestoreIntent::Explicit; + worker.internal_workers.clear(); + worker.execution_bound = false; + state.persist_worker_record(&worker)?; + state.workers.insert(request.worker_id, worker); + state.publish_worker_upsert(request.worker_id)?; + } + // This is storage-owner evidence under the same lock that permanently + // excludes create/restore, not an inference from catalog projection. + #[cfg(feature = "fs-store")] + let originals_absent = !state.workers.contains_key(&request.worker_id) + && state + .fs_store() + .ok_or_else(|| { + RuntimeError::InvalidRequest( + "Subject original absence requires fs storage".into(), + ) + })? + .worker_originals_absent(request.worker_id)?; + #[cfg(not(feature = "fs-store"))] + let originals_absent = false; + let response = runtime_api::SubjectDeleteWorkerResponse { + workspace_id: request.workspace_id.clone(), + worker_id: request.worker_id, + create_fingerprint: request.create_fingerprint.clone(), + operation_id: request.operation_id.clone(), + originals_absent, + }; + state + .subject_delete_worker_fences + .get_mut(&request.worker_id) + .expect("delete fence retained under Worker operation lock") + .response = Some(response.clone()); + state.persist_runtime_snapshot()?; + Ok(response) + } +} diff --git a/crates/worker-runtime/src/runtime/subject_delete_tests.rs b/crates/worker-runtime/src/runtime/subject_delete_tests.rs new file mode 100644 index 000000000..0a6feba55 --- /dev/null +++ b/crates/worker-runtime/src/runtime/subject_delete_tests.rs @@ -0,0 +1,274 @@ +// Runtime component tests use real filesystem persistence and replace only the +// external execution boundary. Actual Controller shutdown is tested by the adapter. +#[cfg(feature = "fs-store")] +#[test] +fn subject_delete_reserved_id_survives_unknown_reopen_and_rejects_identity_changes() { + let root = tempfile::tempdir().unwrap(); + let options = + crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime"); + let scoped = scope("workspace-a", "server-a"); + let create = scoped_task_request("delayed create", "workspace-a"); + let request = runtime_api::SubjectDeleteWorkerRequest { + workspace_id: scoped.workspace_id.clone(), + worker_id: create.worker_id, + create_fingerprint: create.create_fingerprint.clone(), + operation_id: "subject-delete-1".into(), + }; + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + assert!( + runtime + .subject_delete_worker_scoped(&scoped, request.clone()) + .is_err() + ); + assert!( + runtime.lock().unwrap().subject_delete_worker_fences[&create.worker_id] + .response + .is_none() + ); + assert!( + runtime + .create_worker_scoped(&scoped, create.clone()) + .is_err() + ); + assert_eq!(backend.deletion_checks.load(Ordering::SeqCst), 1); + drop(runtime); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + assert!( + runtime + .create_worker_scoped(&scoped, create.clone()) + .is_err() + ); + assert!( + runtime + .subject_delete_worker_scoped(&scoped, request.clone()) + .is_err() + ); + backend.deletion_confirmed.store(true, Ordering::SeqCst); + let receipt = runtime + .subject_delete_worker_scoped(&scoped, request.clone()) + .unwrap(); + let mut expected = serde_json::to_value(&request).unwrap(); + expected["originals_absent"] = true.into(); + assert_eq!(serde_json::to_value(&receipt).unwrap(), expected); + for index in 0..4 { + let mut changed = request.clone(); + match index { + 0 => changed.worker_id = WorkerId::now_v7(), + 1 => changed.operation_id.push_str("-changed"), + 2 => changed.create_fingerprint.push_str("-changed"), + _ => changed.workspace_id.push_str("-changed"), + } + assert!( + runtime + .subject_delete_worker_scoped(&scoped, changed) + .is_err() + ); + } + assert!( + runtime + .subject_delete_worker_scoped(&scope("workspace-a", "server-b"), request.clone()) + .is_err() + ); + assert!( + runtime + .subject_delete_worker_scoped(&scope("workspace-b", "server-a"), request.clone()) + .is_err() + ); + assert!( + !runtime + .lock() + .unwrap() + .forget_workspace_owner_if_unused("workspace-a") + ); + drop(runtime); + // Even a saved receipt must not manufacture fresh shutdown proof on reopen. + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap(); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + assert!( + runtime + .subject_delete_worker_scoped(&scoped, request.clone()) + .is_err() + ); + backend.deletion_confirmed.store(true, Ordering::SeqCst); + assert_eq!( + runtime + .subject_delete_worker_scoped(&scoped, request) + .unwrap(), + receipt + ); + assert!(runtime.create_worker_scoped(&scoped, create).is_err()); +} + +#[cfg(feature = "fs-store")] +#[test] +fn subject_delete_existing_stopped_or_active_worker_preserves_record_and_allows_canonical_removal() +{ + for stopped in [false, true] { + let root = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + backend.deletion_confirmed.store(true, Ordering::SeqCst); + let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path()) + .with_runtime_id("test-runtime"); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let scoped = scope("workspace-a", "server-a"); + let create = scoped_task_request("subject body", "workspace-a"); + let worker = runtime + .create_worker_scoped(&scoped, create.clone()) + .unwrap(); + if stopped { + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + } + let request = runtime_api::SubjectDeleteWorkerRequest { + workspace_id: scoped.workspace_id.clone(), + worker_id: create.worker_id, + create_fingerprint: create.create_fingerprint.clone(), + operation_id: "delete-body".into(), + }; + let mut mismatch = request.clone(); + mismatch.create_fingerprint.push_str("-wrong"); + assert!( + runtime + .subject_delete_worker_scoped(&scoped, mismatch) + .is_err() + ); + let mut mismatch = request.clone(); + mismatch.workspace_id = "workspace-b".into(); + assert!( + runtime + .subject_delete_worker_scoped(&scope("workspace-b", "server-b"), mismatch) + .is_err() + ); + assert!( + !runtime + .lock() + .unwrap() + .subject_delete_worker_fences + .contains_key(&create.worker_id) + ); + // A lost Worker settlement cannot be reported as completed deletion. + runtime + .test_runtime_store() + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert!( + runtime + .subject_delete_worker_scoped(&scoped, request.clone()) + .is_err() + ); + let receipt = runtime + .subject_delete_worker_scoped(&scoped, request.clone()) + .unwrap(); + assert!(!receipt.originals_absent, "a stopped catalog record still has original material"); + assert_eq!( + runtime.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + assert!( + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref) + ) + .is_err() + ); + assert!( + runtime + .create_worker_scoped(&scoped, create.clone()) + .is_err() + ); + let record = runtime + .test_runtime_store() + .read_worker_record(&worker.worker_ref) + .unwrap(); + assert_eq!(record.worker_id, worker.worker_ref.worker_id); + drop(runtime); + let runtime = + Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap(); + assert!( + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref) + ) + .is_err() + ); + assert_eq!( + runtime + .subject_delete_worker_scoped(&scoped, request) + .unwrap(), + receipt + ); + runtime + .delete_worker_scoped(&scoped, &worker.worker_ref) + .unwrap(); + assert!(runtime.create_worker_scoped(&scoped, create).is_err()); + assert!( + !runtime + .lock() + .unwrap() + .forget_workspace_owner_if_unused("workspace-a") + ); + } +} + +#[cfg(feature = "fs-store")] +#[test] +fn subject_delete_unavailable_backend_never_issues_receipt_but_preserves_exclusion() { + let root = tempfile::tempdir().unwrap(); + let options = + crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime"); + let scoped = scope("workspace-a", "server-a"); + let create = scoped_task_request("reserved without backend", "workspace-a"); + let request = runtime_api::SubjectDeleteWorkerRequest { + workspace_id: scoped.workspace_id.clone(), + worker_id: create.worker_id, + create_fingerprint: create.create_fingerprint.clone(), + operation_id: "delete-offline".into(), + }; + let runtime = Runtime::with_fs_store(options.clone()).unwrap(); + assert!( + runtime + .subject_delete_worker_scoped(&scoped, request.clone()) + .is_err() + ); + assert!( + runtime.lock().unwrap().subject_delete_worker_fences[&create.worker_id] + .response + .is_none() + ); + drop(runtime); + let backend = Arc::new(TestExecutionBackend::default()); + backend.deletion_confirmed.store(true, Ordering::SeqCst); + let runtime = Runtime::with_fs_store_and_execution_backend(options, backend).unwrap(); + assert!(runtime.create_worker_scoped(&scoped, create).is_err()); + runtime + .subject_delete_worker_scoped(&scoped, request) + .unwrap(); +} + +#[cfg(feature = "fs-store")] +#[test] +fn subject_delete_missing_catalog_does_not_claim_residual_originals_absent() { + let root = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime"); + let backend = Arc::new(TestExecutionBackend::default()); + backend.deletion_confirmed.store(true, Ordering::SeqCst); + let runtime = Runtime::with_fs_store_and_execution_backend(options, backend).unwrap(); + let scoped = scope("workspace-a", "server-a"); + let create = scoped_task_request("unpublished original", "workspace-a"); + let original = root.path().join("workers").join(create.worker_id.to_string()).join("session/original.jsonl"); + std::fs::create_dir_all(original.parent().unwrap()).unwrap(); + std::fs::write(&original, "preserve original text").unwrap(); + let receipt = runtime.subject_delete_worker_scoped(&scoped, runtime_api::SubjectDeleteWorkerRequest { + workspace_id: scoped.workspace_id.clone(), worker_id: create.worker_id, + create_fingerprint: create.create_fingerprint.clone(), operation_id: "delete-unpublished".into(), + }).unwrap(); + assert!(!receipt.originals_absent); + assert_eq!(std::fs::read_to_string(original).unwrap(), "preserve original text"); + assert!(runtime.create_worker_scoped(&scoped, create).is_err()); +} diff --git a/crates/worker-runtime/src/subject_delete_backend_tests.rs b/crates/worker-runtime/src/subject_delete_backend_tests.rs new file mode 100644 index 000000000..c41830d5b --- /dev/null +++ b/crates/worker-runtime/src/subject_delete_backend_tests.rs @@ -0,0 +1,285 @@ +// Integration of Runtime's durable fence with the real adapter and Controller; +// MockFactory replaces only model/provider/profile construction for a hanging run. +#[test] +#[cfg(feature = "fs-store")] +#[serial_test::serial(worker_allocation)] +fn subject_delete_running_controller_confirms_shutdown_and_preserves_history() { + let root = tempfile::tempdir().unwrap(); + let client = MockClient::sequential(vec![ + MockResponse::Complete(simple_text_events()), + MockResponse::Hang(vec![]), + ]); + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new(MockFactory { + client: client.clone(), + runtime_base: root.path().join("control"), + cwd: root.path().to_path_buf(), + store_dir: root.path().join("sessions"), + worker_metadata_dir: root.path().join("metadata"), + observed_cwds: Arc::default(), + observed_workspace_clients: Arc::default(), + }) + .unwrap(), + ); + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + crate::fs_store::FsRuntimeStoreOptions::new(root.path().join("catalog")) + .with_runtime_id("delete-test"), + backend.clone(), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "server-test"); + let mut create = create_request("running Subject body"); + create.workspace_api = Some(WorkspaceApiRef { + workspace_id: scope.workspace_id.clone(), + base_url: "https://workspace.invalid".into(), + }); + create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: scope.workspace_id.clone(), + language: "English".into(), + }); + let worker = runtime + .create_worker_scoped(&scope, create.clone()) + .unwrap(); + runtime + .send_input(&worker.worker_ref, WorkerInput::user("settled history")) + .unwrap(); + let execution = backend.workers.lock().unwrap()[&worker.worker_ref].clone(); + backend + .run_on_adapter_runtime({ + let handle = execution.handle.clone(); + let client = client.clone(); + async move { + tokio::time::timeout(Duration::from_secs(5), async { + while client.call_count.load(Ordering::SeqCst) < 1 + || handle.shared_state.catalog_status() != WorkerStatus::Idle + { + tokio::task::yield_now().await; + } + }) + .await + .map_err(|error| error.to_string())?; + Ok(()) + } + }) + .unwrap(); + let before = persisted_files(&root.path().join("sessions")); + assert!(!before.is_empty()); + runtime + .send_input(&worker.worker_ref, WorkerInput::user("run until stopped")) + .unwrap(); + backend + .run_on_adapter_runtime({ + let handle = execution.handle.clone(); + let client = client.clone(); + async move { + tokio::time::timeout(Duration::from_secs(5), async { + while client.call_count.load(Ordering::SeqCst) < 2 { + tokio::task::yield_now().await; + } + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Running); + }) + .await + .map_err(|error| error.to_string())?; + Ok(()) + } + }) + .unwrap(); + let request = runtime_api::SubjectDeleteWorkerRequest { + workspace_id: scope.workspace_id.clone(), + worker_id: create.worker_id, + create_fingerprint: create.create_fingerprint.clone(), + operation_id: "delete-running".into(), + }; + runtime + .subject_delete_worker_scoped(&scope, request.clone()) + .unwrap(); + assert!(execution.handle.protocol_is_closed()); + assert!(execution.tasks.tasks.lock().unwrap().is_empty()); + assert!(backend.workers.lock().unwrap().is_empty()); + assert!(backend.pending_restores.lock().unwrap().is_empty()); + assert_eq!( + runtime.worker_detail(&worker.worker_ref).unwrap().status, + crate::catalog::WorkerStatus::Stopped + ); + // Preserve the settled prefix. Ordinary Controller empty-turn rollback remains unchanged. + let after = persisted_files(&root.path().join("sessions")); + for (path, bytes) in before { + let entries = |bytes: &[u8]| { + bytes + .split(|b| *b == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_slice::(line).unwrap()) + .collect::>() + }; + assert!( + entries(&after[&path]).starts_with(&entries(&bytes)), + "{path:?} BEFORE={:?} AFTER={:?}", + entries(&bytes), + entries(&after[&path]) + ); + } + assert!(runtime.create_worker_scoped(&scope, create).is_err()); + assert!( + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref) + ) + .is_err() + ); + runtime + .subject_delete_worker_scoped(&scope, request) + .unwrap(); + assert_eq!(client.call_count.load(Ordering::SeqCst), 2); +} + +#[test] +#[serial_test::serial(worker_allocation)] +#[cfg(feature = "http-server")] +fn subject_delete_unregistered_profile_controller_retries_unknown_shutdown_without_erasing_session() +{ + let root = tempfile::tempdir().unwrap(); + let (backend, request) = profile_stop_fixture(root.path()); + let execution = backend.workers.lock().unwrap()[&request.worker_ref].clone(); + let deletion = runtime_api::SubjectDeleteWorkerRequest { + workspace_id: request + .workspace_scope + .as_ref() + .unwrap() + .workspace_id + .clone(), + worker_id: request.worker_ref.worker_id, + create_fingerprint: request.request.create_fingerprint.clone(), + operation_id: "delete-unregistered".into(), + }; + let mut bad = deletion.clone(); + bad.create_fingerprint.push_str("-wrong"); + assert!(backend.preflight_subject_delete_worker(&bad).is_err()); + assert!(backend.subject_delete_worker(&bad).is_err()); + let mut bad = deletion.clone(); + bad.workspace_id.push_str("-wrong"); + assert!(backend.subject_delete_worker(&bad).is_err()); + assert!( + backend.admissions.lock().unwrap()[&request.worker_ref] + .deletion + .is_none() + ); + let preparing = execution.clone(); + backend + .run_on_adapter_runtime(async move { + prepare_rejected_shutdown(&preparing, false, None).await; + Ok(()) + }) + .unwrap(); + let session_root = root + .path() + .join("runtime/workers") + .join(request.worker_ref.worker_id.to_string()) + .join("session"); + let before = persisted_files(&session_root); + assert!(!before.is_empty()); + assert!( + backend + .subject_delete_worker(&deletion) + .unwrap_err() + .contains("Conflict") + ); + assert!( + backend + .workers + .lock() + .unwrap() + .contains_key(&request.worker_ref) + ); + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::Rejected(_) + )); + backend.subject_delete_worker(&deletion).unwrap(); + backend.subject_delete_worker(&deletion).unwrap(); + assert!(execution.handle.protocol_is_closed()); + assert!(execution.tasks.tasks.lock().unwrap().is_empty()); + assert!(backend.workers.lock().unwrap().is_empty()); + assert_eq!(persisted_files(&session_root), before); + assert!(backend.preflight_restore(&request).is_err()); + assert!( + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .is_err() + ); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn subject_delete_pending_restore_fences_late_controller_and_never_restarts_it() { + let root = tempfile::tempdir().unwrap(); + let restores = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let handles = Arc::new(Mutex::new(Vec::new())); + let backend = WorkerRuntimeExecutionBackend::new(LateControllerFactory { + inner: MockFactory { + client: MockClient::new(vec![]), + runtime_base: root.path().join("control"), + cwd: root.path().to_path_buf(), + store_dir: root.path().join("sessions"), + worker_metadata_dir: root.path().join("metadata"), + observed_cwds: Arc::default(), + observed_workspace_clients: Arc::default(), + }, + restores: restores.clone(), + release: release.clone(), + actual_handles: handles.clone(), + }) + .unwrap() + .with_spawn_restore_timeout(Duration::from_millis(1)); + let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + request.workspace_scope = Some(crate::runtime::RuntimeWorkspaceScope::new( + "ws-test", + "server-test", + )); + request.request.workspace_api = Some(WorkspaceApiRef { + workspace_id: "ws-test".into(), + base_url: "https://workspace.invalid".into(), + }); + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let deletion = runtime_api::SubjectDeleteWorkerRequest { + workspace_id: "ws-test".into(), + worker_id: request.worker_ref.worker_id, + create_fingerprint: request.request.create_fingerprint.clone(), + operation_id: "delete-pending".into(), + }; + // Unfinished factory ownership is unknown, never shutdown success. + assert!( + backend + .subject_delete_worker(&deletion) + .unwrap_err() + .contains("pending") + ); + assert!( + backend + .pending_restores + .lock() + .unwrap() + .contains_key(&request.worker_ref) + ); + assert!( + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .is_err() + ); + release.notify_one(); + backend.subject_delete_worker(&deletion).unwrap(); + assert!(backend.pending_restores.lock().unwrap().is_empty()); + assert!(backend.workers.lock().unwrap().is_empty()); + assert_eq!(restores.load(Ordering::SeqCst), 1); + assert_eq!(handles.lock().unwrap().len(), 1); + assert!(handles.lock().unwrap()[0].protocol_is_closed()); + assert!(matches!( + backend.restore_worker(request), + WorkerExecutionSpawnResult::Rejected(_) + )); +} diff --git a/crates/worker-runtime/src/subject_delete_create_tests.rs b/crates/worker-runtime/src/subject_delete_create_tests.rs new file mode 100644 index 000000000..57642940a --- /dev/null +++ b/crates/worker-runtime/src/subject_delete_create_tests.rs @@ -0,0 +1,101 @@ +// Actual create factory future remains behind Runtime's Worker operation lock; +// deletion must wait for that owner, then permanently reject the delayed retry. +struct SubjectDeleteDeferredCreate { + entered: mpsc::Sender<()>, + release: Arc, +} +#[async_trait] +impl RuntimeWorkerFactory for SubjectDeleteDeferredCreate { + async fn prepare_restore( + &self, + _: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + panic!("fenced create must never restore") + } + async fn spawn_controller( + &self, + _: WorkerExecutionSpawnRequest, + ) -> Result { + self.entered.send(()).unwrap(); + self.release.notified().await; + Err("create failed before any Controller".into()) + } +} + +#[test] +fn subject_delete_serializes_inflight_create_and_rejects_all_late_admission() { + let root = tempfile::tempdir().unwrap(); + let (entered, entry) = mpsc::channel(); + let release = Arc::new(tokio::sync::Notify::new()); + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new(SubjectDeleteDeferredCreate { + entered, + release: release.clone(), + }) + .unwrap(), + ); + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("delete-test"), + backend.clone(), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "server-test"); + let mut create = create_request("inflight create"); + create.workspace_api = Some(WorkspaceApiRef { + workspace_id: scope.workspace_id.clone(), + base_url: "https://workspace.invalid".into(), + }); + create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: scope.workspace_id.clone(), + language: "English".into(), + }); + let creating = { + let runtime = runtime.clone(); + let scope = scope.clone(); + let create = create.clone(); + std::thread::spawn(move || runtime.create_worker_scoped(&scope, create)) + }; + entry.recv_timeout(Duration::from_secs(5)).unwrap(); + let request = runtime_api::SubjectDeleteWorkerRequest { + workspace_id: scope.workspace_id.clone(), + worker_id: create.worker_id, + create_fingerprint: create.create_fingerprint.clone(), + operation_id: "delete-creating".into(), + }; + let deleting = { + let runtime = runtime.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || runtime.subject_delete_worker_scoped(&scope, request)) + }; + release.notify_one(); + assert!(creating.join().unwrap().is_err()); + deleting.join().unwrap().unwrap(); + assert!( + runtime + .create_worker_scoped(&scope, create.clone()) + .is_err() + ); + assert!(backend.workers.lock().unwrap().is_empty()); + let worker_ref = WorkerRef::new(create.worker_id); + let spawn = WorkerExecutionSpawnRequest { + worker_ref: worker_ref.clone(), + request: create.clone(), + workspace_scope: Some(scope.clone()), + context: test_execution_context(worker_ref.clone()), + workdir_attachments: BTreeMap::new(), + config_bundle: None, + }; + assert!(matches!( + backend.spawn_worker(spawn), + WorkerExecutionSpawnResult::Rejected(_) + )); + let mut restore = direct_restore_request(worker_ref); + restore.request = create; + restore.workspace_scope = Some(scope); + assert!(matches!( + backend.restore_worker(restore), + WorkerExecutionSpawnResult::Rejected(_) + )); +} diff --git a/crates/worker-runtime/src/subject_delete_store_tests.rs b/crates/worker-runtime/src/subject_delete_store_tests.rs new file mode 100644 index 000000000..158b6dc03 --- /dev/null +++ b/crates/worker-runtime/src/subject_delete_store_tests.rs @@ -0,0 +1,104 @@ +#[test] +fn subject_delete_schema14_migration_adds_exclusion_store_without_changing_worker_or_receipts() { + for with_fence in [false, true] { + let (_temp, store, worker, path) = canonical_worker_store(); + let root = store.root.clone(); + let mut legacy = + serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap(); + legacy["schema_version"] = serde_json::json!(RESTORE_REPAIR_SCHEMA_VERSION); + fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap(); + let id = WorkerId::now_v7(); + let request = serde_json::json!({"workspace_id": "workspace-a", "worker_id": id, + "create_fingerprint": "fenced-create", "operation_id": "delete-1"}); + let mut response = request.clone(); + response["originals_absent"] = false.into(); + let fence = + serde_json::json!({ id.to_string(): {"request": request, "response": response} }); + let mut runtime = serde_json::json!({"schema_version": RESTORE_REPAIR_SCHEMA_VERSION, + "display_name": "retained", "backend": "fs_store", "status": "running", "next_diagnostic_id": 1, + "workspace_owners": {"workspace-a": "server-a"}, "diagnostics": [], + "config_bundles": {}, "subject_create_recovery_fences": {}, "subject_releases": []}); + if with_fence { + runtime["subject_delete_worker_fences"] = fence.clone(); + } + fs::write( + root.join(RUNTIME_FILE), + serde_json::to_vec(&runtime).unwrap(), + ) + .unwrap(); + drop(store); + let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("test-runtime"); + let before = fs::read(&path).unwrap(); + assert!( + FsRuntimeStore::migration_plan(&options) + .unwrap() + .migration_required + ); + assert_eq!(fs::read(&path).unwrap(), before); + let opened = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); + legacy["schema_version"] = serde_json::json!(SCHEMA_VERSION); + assert_eq!( + read_json::(&path, "read migrated Worker").unwrap(), + legacy + ); + let expected = if with_fence { + fence + } else { + serde_json::json!({}) + }; + assert_eq!( + serde_json::to_value(&opened.state.as_ref().unwrap().subject_delete_worker_fences) + .unwrap(), + expected + ); + opened + .store + .write_runtime_snapshot(opened.state.as_ref().unwrap()) + .unwrap(); + drop(opened); + let reopened = FsRuntimeStore::open_or_create(root, "test-runtime").unwrap(); + assert_eq!( + serde_json::to_value(&reopened.state.unwrap().subject_delete_worker_fences).unwrap(), + expected + ); + } +} + +#[test] +fn subject_delete_store_rejects_missing_exclusion_field_malformed_receipt_and_workspace_owner() { + for invalid in 0..4 { + let (_temp, store, _worker, _path) = canonical_worker_store(); + let root = store.root.clone(); + let id = WorkerId::now_v7(); + let request = serde_json::json!({"workspace_id": "workspace-a", "worker_id": id, + "create_fingerprint": "fenced-create", "operation_id": "delete-1"}); + let mut response = request.clone(); + response["originals_absent"] = false.into(); + let mut runtime = serde_json::json!({"schema_version": SCHEMA_VERSION, + "display_name": "retained", "backend": "fs_store", "status": "running", "next_diagnostic_id": 1, + "workspace_owners": {"workspace-a": "server-a"}, "diagnostics": [], + "subject_delete_worker_fences": { id.to_string(): {"request": request, "response": response} }}); + match invalid { + 0 => { + runtime + .as_object_mut() + .unwrap() + .remove("subject_delete_worker_fences"); + } + 1 => { + runtime["subject_delete_worker_fences"][id.to_string()]["response"]["create_fingerprint"] = + "different".into() + } + 2 => runtime["workspace_owners"] = serde_json::json!({}), + _ => { + runtime["subject_delete_worker_fences"][id.to_string()]["request"]["worker_id"] = + WorkerId::now_v7().to_string().into() + } + } + let bytes = serde_json::to_vec(&runtime).unwrap(); + fs::write(root.join(RUNTIME_FILE), &bytes).unwrap(); + drop(store); + assert!(FsRuntimeStore::open_or_create(root.clone(), "test-runtime").is_err()); + assert_eq!(fs::read(root.join(RUNTIME_FILE)).unwrap(), bytes); + } +} diff --git a/crates/worker-runtime/src/test_support.rs b/crates/worker-runtime/src/test_support.rs index 229815241..794578002 100644 --- a/crates/worker-runtime/src/test_support.rs +++ b/crates/worker-runtime/src/test_support.rs @@ -215,7 +215,8 @@ impl StoppedWorkerTemplate { snapshot .validate(&root.join(RUNTIME_FILE)) .map_err(|_| reject("unsupported Runtime schema"))?; - if !snapshot.subject_create_recovery_fences.is_empty() + if !snapshot.subject_delete_worker_fences.is_empty() + || !snapshot.subject_create_recovery_fences.is_empty() || !snapshot.subject_releases.is_empty() { return Err(reject("source has lifecycle operations")); @@ -427,6 +428,7 @@ impl StoppedWorkerTemplate { )?; self.validate_session(&session)?; store.write_runtime_snapshot(&PersistedRuntimeState { + subject_delete_worker_fences: BTreeMap::new(), subject_create_recovery_fences: BTreeMap::new(), subject_releases: Vec::new(), // Match normal store initialization: this is catalog availability, diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index c892788bf..8fcd8cfd5 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -3415,6 +3415,14 @@ struct SubjectControllerReservation { state: runtime_api::SubjectReleaseState, } +// Admission identity is retained even when execution publication or stop settlement +// failed. Runtime owns durable exclusion; this guards the adapter's delayed calls. +struct WorkerAdmission { + workspace_id: Option, + create_fingerprint: String, + deletion: Option, +} + /// Worker execution backend backed by real worker crate Workers. pub struct WorkerRuntimeExecutionBackend { #[cfg(test)] @@ -3438,6 +3446,7 @@ pub struct WorkerRuntimeExecutionBackend { >, >, worker_locks: Mutex>>>, + admissions: Mutex>, spawn_restore_timeout: Duration, subject_reservations: Mutex>, shutdown_admission_timeout: Duration, @@ -3464,12 +3473,92 @@ where pending_restores: Mutex::new(HashMap::new()), completed_restore_failures: Mutex::new(HashMap::new()), worker_locks: Mutex::new(HashMap::new()), + admissions: Mutex::new(HashMap::new()), spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT, subject_reservations: Mutex::new(HashMap::new()), shutdown_admission_timeout: Duration::from_secs(5), }) } + // Caller holds the adapter Worker lock. + fn admit_worker( + &self, + worker: &WorkerRef, + workspace: Option<&str>, + fingerprint: &str, + ) -> Result<(), String> { + let mut admissions = self + .admissions + .lock() + .map_err(|_| "worker admission lock poisoned")?; + if let Some(saved) = admissions.get(worker) { + if saved.deletion.is_some() { + return Err("Subject Worker permanently excluded".into()); + } + if saved.workspace_id.as_deref() != workspace || saved.create_fingerprint != fingerprint + { + return Err("Worker admission identity mismatch".into()); + } + } else { + admissions.insert( + worker.clone(), + WorkerAdmission { + workspace_id: workspace.map(str::to_string), + create_fingerprint: fingerprint.into(), + deletion: None, + }, + ); + } + Ok(()) + } + + fn validate_subject_delete_identity( + &self, + request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + let admissions = self + .admissions + .lock() + .map_err(|_| "worker admission lock poisoned")?; + if let Some(saved) = admissions.get(&WorkerRef::new(request.worker_id)) { + if saved.workspace_id.as_deref() != Some(request.workspace_id.as_str()) + || saved.create_fingerprint != request.create_fingerprint + || saved + .deletion + .as_ref() + .is_some_and(|saved| saved != request) + { + return Err("Subject delete execution identity mismatch".into()); + } + } + if let Some(pending) = self.pending_restore(&WorkerRef::new(request.worker_id))? { + if pending + .request + .workspace_scope + .as_ref() + .map(|scope| scope.workspace_id.as_str()) + != Some(request.workspace_id.as_str()) + || pending.request.request.create_fingerprint != request.create_fingerprint + { + return Err("Subject delete pending execution identity mismatch".into()); + } + } + Ok(()) + } + + fn ensure_not_subject_deleted(&self, worker: &WorkerRef) -> Result<(), String> { + if self + .admissions + .lock() + .map_err(|_| "worker admission lock poisoned")? + .get(worker) + .is_some_and(|admission| admission.deletion.is_some()) + { + return Err("Subject Worker permanently excluded".into()); + } + Ok(()) + } + pub fn with_backend_id(mut self, backend_id: impl Into) -> Self { self.backend_id = backend_id.into(); self @@ -4739,6 +4828,58 @@ where &self.backend_id } + fn preflight_subject_delete_worker( + &self, + request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + let worker = WorkerRef::new(request.worker_id); + let lock = self.worker_lock(&worker)?; + let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?; + self.validate_subject_delete_identity(request) + } + + fn subject_delete_worker( + &self, + request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> Result<(), String> { + let worker = WorkerRef::new(request.worker_id); + let lock = self.worker_lock(&worker)?; + let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?; + self.validate_subject_delete_identity(request)?; + self.admissions + .lock() + .map_err(|_| "worker admission lock poisoned")? + .insert( + worker.clone(), + WorkerAdmission { + workspace_id: Some(request.workspace_id.clone()), + create_fingerprint: request.create_fingerprint.clone(), + deletion: Some(request.clone()), + }, + ); + let result = self.stop_worker_under_lock(&worker); + if !result.is_accepted() { + return Err(result.message_or_default()); + } + // A live Controller and failed unpublished candidates can coexist. The + // ordinary stop's live branch joins the Controller; also reconcile any + // factory-owned cache/artifacts before issuing deletion proof. + let factory = self.factory.clone(); + let stopped = worker.clone(); + self.run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move { + factory.reconcile_stopped_worker(&stopped).await + })?; + self.subject_reservations + .lock() + .map_err(|_| "Subject reservations poisoned")? + .retain(|(saved, _), _| saved != &worker); + self.completed_restore_failures + .lock() + .map_err(|_| "completed restores poisoned")? + .remove(&worker); + Ok(()) + } + fn recover_subject_create( &self, request: &runtime_api::SubjectCreateRecoveryRequest, @@ -5178,6 +5319,19 @@ where )); } }; + if let Err(message) = self.admit_worker( + &request.worker_ref, + request + .workspace_scope + .as_ref() + .map(|scope| scope.workspace_id.as_str()), + &request.request.create_fingerprint, + ) { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Spawn, + message, + )); + } match self.pending_restore(&request.worker_ref) { Ok(None) => {} Ok(Some(_)) => { @@ -5405,6 +5559,10 @@ where )); } }; + self.ensure_not_subject_deleted(&request.worker_ref) + .map_err(|message| { + WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, message) + })?; if self .pending_restore(&request.worker_ref) .map_err(|message| { @@ -5499,6 +5657,19 @@ where )); } }; + if let Err(message) = self.admit_worker( + &request.worker_ref, + request + .workspace_scope + .as_ref() + .map(|scope| scope.workspace_id.as_str()), + &request.request.create_fingerprint, + ) { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + message, + )); + } if let Some(result) = self.completed_restore_failure(request.operation_id, &request.worker_ref) { @@ -6182,6 +6353,7 @@ where return Err(message); } }; + self.ensure_not_subject_deleted(worker_ref)?; if let Some(pending) = self.pending_restore(worker_ref)? { if pending.request.operation_id != operation_id { return Err("restored Worker operation identity does not match".into()); @@ -6820,6 +6992,10 @@ mod tests { }; use worker::PreparedWorker; + include!("subject_delete_backend_tests.rs"); + #[cfg(feature = "fs-store")] + include!("subject_delete_create_tests.rs"); + // Both candidate and failure suites need the same guarded, checkout-owned // temporary root, independently of Runtime's optional filesystem catalog. fn restore_failure_tempdir() -> tempfile::TempDir { diff --git a/crates/workspace-server/src/assignment_tracking_tests.rs b/crates/workspace-server/src/assignment_tracking_tests.rs index 3ccac8020..73b3445f0 100644 --- a/crates/workspace-server/src/assignment_tracking_tests.rs +++ b/crates/workspace-server/src/assignment_tracking_tests.rs @@ -478,7 +478,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip (101, "Durable explicit Subject body release".into()), (102, "Durable failed Subject creation recovery".into()), (103, "Passkey metadata and atomic addition results".into()), - (104, "Converge Passkey results and Job input retention".into()) + (104, "Converge Passkey results and Job input retention".into()), + (105, "Durable Subject deletion snapshots and Job admission fences".into()) ] ); for (table, rows) in retained.iter().zip(original.iter()) { diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 7f7ac29d2..8ba623282 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -1292,6 +1292,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { }) } + fn subject_delete_worker( + &self, + _request: runtime_api::SubjectDeleteWorkerRequest, + ) -> Result { + Err("Subject deletion fencing is unavailable on this Runtime".into()) + } + fn recover_subject_create( &self, _request: runtime_api::SubjectCreateRecoveryRequest, @@ -2269,6 +2276,21 @@ impl RuntimeRegistry { runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id) } + pub fn subject_delete_worker( + &self, + runtime_id: &str, + request: runtime_api::SubjectDeleteWorkerRequest, + ) -> Result { + validate_backend_identifier("runtime_id", runtime_id)?; + self.runtime(runtime_id)? + .subject_delete_worker(request) + .map_err(|message| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: runtime_id.into(), + code: "subject_delete_worker_unproven".into(), + message, + }) + } + pub fn recover_subject_create( &self, runtime_id: &str, @@ -3367,6 +3389,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { }) } + fn subject_delete_worker( + &self, + request: runtime_api::SubjectDeleteWorkerRequest, + ) -> Result { + if request.workspace_id != self.workspace_id { + return Err("Subject deletion Workspace mismatch".into()); + } + let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend"); + self.runtime + .subject_delete_worker_scoped(&scope, request) + .map_err(|e| e.to_string()) + } + fn recover_subject_create( &self, request: runtime_api::SubjectCreateRecoveryRequest, @@ -4255,7 +4290,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s } if path.contains("/retention/") || (path.starts_with("/v1/workers/") && method == "DELETE") - || (path == "/v1/subject-create-recovery" && method == "POST") + || (matches!( + path, + "/v1/subject-create-recovery" | "/v1/subject-delete-worker" + ) && method == "POST") { return "workers:delete"; } @@ -5656,6 +5694,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { }) } + fn subject_delete_worker( + &self, + request: runtime_api::SubjectDeleteWorkerRequest, + ) -> Result { + self.run_runtime_api( + self.request_timeout, + MAX_REMOTE_RUNTIME_RESPONSE_BYTES, + move |client| async move { client.subject_delete_worker(request).await }, + ) + .map_err(|diagnostic| diagnostic.message) + } + fn recover_subject_create( &self, request: runtime_api::SubjectCreateRecoveryRequest, @@ -8507,6 +8557,57 @@ mod tests { ); } + #[test] + fn remote_subject_delete_uses_generated_exact_identity_transport() { + assert_eq!( + workspace_runtime_operation("POST", "/v1/subject-delete-worker"), + "workers:delete" + ); + let request = runtime_api::SubjectDeleteWorkerRequest { + workspace_id: "workspace-test".into(), + worker_id: EmbeddedWorkerId::now_v7(), + create_fingerprint: "sha256:reserved-create".into(), + operation_id: "delete-worker-1".into(), + }; + let response = runtime_api::SubjectDeleteWorkerResponse { + originals_absent: true, + workspace_id: request.workspace_id.clone(), + worker_id: request.worker_id, + create_fingerprint: request.create_fingerprint.clone(), + operation_id: request.operation_id.clone(), + }; + // Only remote transport is substituted; actual Controller shutdown and + // signed admission are covered by worker-runtime's independent suites. + let (base_url, server) = serve_mock_http(vec![mock_response( + "POST", + "/v1/subject-delete-worker", + true, + 200, + serde_json::to_string(&response).unwrap(), + )]); + let registry = RuntimeRegistry::new(Vec::new()); + registry.register( + RemoteWorkerRuntime::new( + RemoteRuntimeConfig::new( + "remote:delete", + "Deletion transport fixture", + base_url, + Some("secret-token".into()), + ), + "workspace-test".into(), + "http://127.0.0.1:8787".into(), + ) + .unwrap(), + ); + assert_eq!( + registry + .subject_delete_worker("remote:delete", request) + .unwrap(), + response + ); + server.join().unwrap(); + } + #[test] fn remote_subject_create_recovery_uses_generated_management_transport() { assert_eq!( diff --git a/crates/workspace-server/src/implementation_result_tests.rs b/crates/workspace-server/src/implementation_result_tests.rs index 3f50234d5..baf355717 100644 --- a/crates/workspace-server/src/implementation_result_tests.rs +++ b/crates/workspace-server/src/implementation_result_tests.rs @@ -367,14 +367,54 @@ async fn implementation_result_rejects_forged_evidence_and_rolls_back_without_no .unwrap(); } -#[tokio::test] -async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() { - let (dir, store) = tracking_store_fixture().await; - tracking_store_assign(&store, "delegation", "assign"); - let worker = routing_worker_assign(&store, "implementation").worker; - // Reconstruct schema97 from a fresh current-schema fixture, including its historical capture. - store.with_conn(|conn| { - conn.execute_batch("DROP TRIGGER notification_mr_event; +// Preserve seeded assignment evidence while rebuilding the actual pre-business +// capture composition. Changing only a receipt leaves schema105 admission fences +// and erasure guards in an otherwise legacy fixture. +fn reconstruct_implementation_schema96(conn: &Connection) -> Result<()> { + assert_eq!(current_schema_version(conn)?, 105); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("retained96.db"); + prepare_retained_schema(&path, 96); + let retained = Connection::open(path)?; + for trigger in [ + "notification_sources_no_delete", + "notification_decisions_no_delete", + "notification_requests_no_delete", + "notification_acknowledgements_no_delete", + "notification_failures_no_delete", + "notification_job_targets_no_delete", + "notification_job_legacy_no_delete", + "notification_source_preparation_failures_no_delete", + ] { + let definition: String = retained.query_row( + "SELECT sql FROM sqlite_schema WHERE type='trigger' AND name=?1", + [trigger], + |row| row.get(0), + )?; + conn.execute_batch(&format!("DROP TRIGGER {trigger}; {definition};"))?; + } + for trigger in [ + "subject_deleted_job_insert", + "subject_deleted_job_result", + "subject_deleted_attempt_insert", + "subject_deleted_attempt_result", + "subject_deleted_worker_reservation", + "subject_deleted_notification_source", + "subject_deleted_notification_payload", + "subject_deleted_notification_send", + ] { + conn.execute_batch(&format!("DROP TRIGGER {trigger};"))?; + } + // Children before parents; dropping the tables also drops snapshot guards. + conn.execute_batch( + "DROP TABLE subject_deletion_allocations; + DROP TABLE subject_deletion_jobs; + DROP TABLE subject_deletion_workers; + DROP TABLE subject_deletions; + UPDATE __yoi_schema_migrations SET version=104 WHERE version=105;", + )?; + conn.execute_batch( + "DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture; @@ -396,7 +436,43 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no UPDATE __yoi_schema_migrations SET version=99 WHERE version=100; DROP TABLE hosting_administrators; DROP TABLE hosting_settings; UPDATE __yoi_schema_migrations SET version=98 WHERE version=99; - UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?; + UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;", + )?; + assert_eq!(current_schema_version(conn)?, 96); + assert_eq!(conn.query_row( + "SELECT count(*) FROM sqlite_schema WHERE name LIKE 'subject_delet%' OR sql LIKE '%subject_deletion%'", + [], |row| row.get::<_, i64>(0), + )?, 0); + for table in [ + "notification_sources", + "notification_decisions", + "notification_requests", + "notification_acknowledgements", + "notification_failures", + "notification_job_targets", + "notification_job_legacy_receipts", + "notification_source_preparation_failures", + "backend_jobs", + "backend_job_attempts", + "worker_create_reservations", + ] { + assert_eq!( + state_contract_schema_signature(conn, table), + state_contract_schema_signature(&retained, table), + "schema96 {table}" + ); + } + Ok(()) +} + +#[tokio::test] +async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() { + let (dir, store) = tracking_store_fixture().await; + tracking_store_assign(&store, "delegation", "assign"); + let worker = routing_worker_assign(&store, "implementation").worker; + // Reconstruct schema97 from a fresh current-schema fixture, including its historical capture. + store.with_conn(|conn| { + reconstruct_implementation_schema96(conn)?; migrate_business_notifications_v96_to_v97(conn)?; assert_eq!(current_schema_version(conn)?, 97); Ok(()) @@ -522,24 +598,7 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow store.with_conn(|conn| { conn.execute("INSERT INTO ticket_assignment_operation_fences VALUES('w','operation-known-implementation','delegation','r','worker','workspace-orchestrator','tracking-binding',?1)",[serde_json::to_string(&tracking_store_source()).unwrap()])?; // Reconstruct the prior composition before its new business capture existed. - conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture; - DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; - DROP TRIGGER passkey_metadata_insert; - DROP TABLE passkey_addition_receipts; - DROP TABLE passkey_metadata; - DROP TRIGGER backend_job_input_retirement; - DROP TRIGGER backend_job_identity_immutable; - ALTER TABLE backend_jobs DROP COLUMN input_disposition; - UPDATE __yoi_schema_migrations SET version=102 WHERE version=104; - DROP TABLE subject_create_recoveries; - UPDATE __yoi_schema_migrations SET version=101 WHERE version=102; - DROP TABLE subject_worker_releases; - UPDATE __yoi_schema_migrations SET version=100 WHERE version=101; - DROP TABLE user_activations; - UPDATE __yoi_schema_migrations SET version=99 WHERE version=100; - DROP TABLE hosting_administrators; DROP TABLE hosting_settings; - UPDATE __yoi_schema_migrations SET version=98 WHERE version=99; - UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?; + reconstruct_implementation_schema96(conn)?; conn.execute_batch("INSERT INTO merge_requests VALUES('w','historical-mr','main','open','refs/heads/work','refs/heads/main','3','3'); INSERT INTO merge_request_ticket_relations VALUES('w','historical-mr','tracking','implements','3'); INSERT INTO merge_request_thread_events VALUES('w','historical-mr','approved-event',1,'review','{\"subject_ref\":\"source-a\",\"decision\":\"approve\"}',NULL,'3'); @@ -691,24 +750,7 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() { ]; store.with_conn(|conn| { if historical { - conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture; - DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; - DROP TRIGGER passkey_metadata_insert; - DROP TABLE passkey_addition_receipts; - DROP TABLE passkey_metadata; - DROP TRIGGER backend_job_input_retirement; - DROP TRIGGER backend_job_identity_immutable; - ALTER TABLE backend_jobs DROP COLUMN input_disposition; - UPDATE __yoi_schema_migrations SET version=102 WHERE version=104; - DROP TABLE subject_create_recoveries; - UPDATE __yoi_schema_migrations SET version=101 WHERE version=102; - DROP TABLE subject_worker_releases; - UPDATE __yoi_schema_migrations SET version=100 WHERE version=101; - DROP TABLE user_activations; - UPDATE __yoi_schema_migrations SET version=99 WHERE version=100; - DROP TABLE hosting_administrators; DROP TABLE hosting_settings; - UPDATE __yoi_schema_migrations SET version=98 WHERE version=99; - UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?; + reconstruct_implementation_schema96(conn)?; } conn.execute_batch("INSERT INTO merge_requests VALUES('w','actor-mr','main','open','work','main','3','3'); INSERT INTO merge_request_ticket_relations VALUES('w','actor-mr','tracking','implements','3');")?; diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 30244ac7f..01ebf25b5 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -1757,3 +1757,5 @@ WHEN (OLD.runtime_id != NEW.runtime_id OR OLD.worker_id != NEW.worker_id OR OLD. WHERE workspace_id = OLD.workspace_id AND runtime_id = OLD.runtime_id AND worker_id = OLD.worker_id) BEGIN SELECT RAISE(ABORT, 'Worker has unfinished Ticket work'); END; + +-- Subject deletion schema is composed by store.rs. diff --git a/crates/workspace-server/src/notification_delivery.rs b/crates/workspace-server/src/notification_delivery.rs index 8b21fb2ef..1297001cd 100644 --- a/crates/workspace-server/src/notification_delivery.rs +++ b/crates/workspace-server/src/notification_delivery.rs @@ -444,6 +444,28 @@ fn job_result_facts( pub(super) fn deliver_job_result(api: &WorkspaceApi, acceptance: &BackendJobResultAcceptance) { use crate::store::JobNotificationFacts; + // A recovered/captured result is not delivery authority. Serialize even the + // Runtime send with deletion, so no prepared bytes escape after its fence. + let subject = subjektiv_deletion::job_subject(&acceptance.job.request); + let gate = match subject + .map(|subject| subjektiv_journal::lifecycle_lock(api, subject)) + .transpose() + { + Ok(gate) => gate, + Err(error) => { + tracing::warn!(%error, "Subject Job notification gate unavailable"); + return; + } + }; + let _guard = match gate.as_ref().map(|gate| gate.lock()).transpose() { + Ok(guard) => guard, + Err(_) => return, + }; + if let Some(subject) = subject + && subjektiv_deletion::require_available(api, subject).is_err() + { + return; + } let Some(_permit) = api.notification_delivery_lifecycle.begin() else { return; }; diff --git a/crates/workspace-server/src/notification_routing_tests.rs b/crates/workspace-server/src/notification_routing_tests.rs index db28581cb..3647ab75a 100644 --- a/crates/workspace-server/src/notification_routing_tests.rs +++ b/crates/workspace-server/src/notification_routing_tests.rs @@ -849,7 +849,16 @@ async fn notification_routing_fresh_and_upgraded_schema_and_guards_match() { let snapshot = |conn: &Connection| -> Vec<(String, String)> { conn.prepare("SELECT name,sql FROM sqlite_schema WHERE name LIKE 'notification_%' OR name LIKE 'ticket_assignment_content%' OR name IN ('ticket_assignment_title_input','ticket_assignment_initial_post_input') ORDER BY name").unwrap().query_map([],|r|Ok((r.get(0)?,r.get(1)?))).unwrap().collect::>().unwrap() }; - let upgraded = snapshot(&conn); + assert_eq!(current_schema_version(&conn).unwrap(), 98); + drop(conn); + // Compare the complete current composition, not upgraded98 against fresh105. + let upgraded_store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let upgraded = upgraded_store + .with_conn(|conn| { + assert_eq!(current_schema_version(conn)?, 105); + Ok(snapshot(conn)) + }) + .unwrap(); let (fresh_directory, fresh) = tracking_store_fixture().await; fresh .with_conn(|conn| { diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index b8469122e..b727b0eef 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -333,6 +333,15 @@ impl SqliteWorkspaceStore { return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()})); } }; + if crate::store::subject_deletion_worker_kind(&tx, &req.workspace_id, &req.worker)?.is_some() { + let prior = tx.prepare("SELECT plan_id FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('executing','failed')")? + .query_map(params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|r.get::<_,String>(0))? + .collect::>>()?; + for id in prior { + let prior=load_plan(&tx,&id)?.ok_or_else(||StoreError::Store("removal plan disappeared".into()))?; + require_current_subject_deletion_plan(&tx,&prior)?; + } + } let mut blockers=Vec::new(); if worker.retention_state=="pinned" { blockers.push(WorkerRemovalBlocker::Hold); } if let Some((assignment_id,ticket_id))=tx.query_row("SELECT a.assignment_id,a.ticket_id FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3",params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()? { @@ -363,6 +372,7 @@ impl SqliteWorkspaceStore { let mut plan=load_plan(&tx,plan_id)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:plan missing")))?; if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));} if plan.input_fingerprint!=fp{return Err(StoreError::InvalidInput(format!("fingerprint:{}",plan.operation_id)));} + require_current_subject_deletion_plan(&tx, &plan)?; if plan.state==WorkerRemovalPlanState::Succeeded{tx.commit()?;return Ok(plan);} if plan.state==WorkerRemovalPlanState::Blocked { return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap()))); @@ -420,6 +430,8 @@ impl SqliteWorkspaceStore { WorkerRetentionError::Invalid("Worker id must be a canonical UUIDv7".to_string()) })?; let removed_at = plan.created_at.clone(); + self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan)) + .map_err(map_error)?; let prior_failure_category = plan.failure_category.clone(); Ok(PreparedWorkerRemoval { runtime_request: WorkerRetentionExecutionRequest { @@ -459,8 +471,8 @@ impl SqliteWorkspaceStore { WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 ) ) - ORDER BY CASE state WHEN 'succeeded' THEN 0 ELSE 1 END, - created_at DESC LIMIT 1", + ORDER BY CASE WHEN EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=worker_removal_operations.workspace_id AND d.runtime_id=worker_removal_operations.runtime_id AND d.worker_id=worker_removal_operations.worker_id AND d.prior_removal_operation_id=worker_removal_operations.operation_id) THEN 0 WHEN state='succeeded' THEN 1 ELSE 2 END, + created_at DESC,operation_id DESC LIMIT 1", params![workspace_id, worker.runtime_id, worker.worker_id], |row| row.get::<_, String>(0), ) @@ -474,6 +486,8 @@ impl SqliteWorkspaceStore { let Some(plan) = plan else { return Ok(None); }; + self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan)) + .map_err(map_error)?; let prior_failure_category = plan.failure_category.clone(); let worker_id = plan .worker @@ -659,6 +673,7 @@ impl SqliteWorkspaceStore { let mut plan=load_plan_op(&tx,operation_id)?.ok_or_else(||StoreError::InvalidInput("operation missing".into()))?; if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));} if plan.input_fingerprint!=fp||result.input_fingerprint!=fp||result.operation_id!=operation_id{return Err(StoreError::InvalidInput(format!("fingerprint:{operation_id}")));} + require_current_subject_deletion_plan(&tx, &plan)?; if plan.state==WorkerRemovalPlanState::Succeeded{ // Repair projection fences left by the pre-atomic implementation and // return a replayable change for a broadcast missed after commit. @@ -1024,6 +1039,49 @@ fn load_removal_policy( let Some(mut policy) = load_policy(c, w)? else { return Ok(None); }; + // Snapshot authority outranks normal policy and Journal cleanup policy. Body + // and delegated Sessions must never inherit the ephemeral Job exception. + if let Some(is_job) = crate::store::subject_deletion_worker_kind(c, w, worker)? { + let update = WorkerRetentionPolicyUpdate { + policy_id: if is_job { + "subject-deletion-job-v1" + } else { + "subject-deletion-conversation-v1" + } + .into(), + session_disposition: if is_job { + SessionDisposition::Purge + } else { + SessionDisposition::Archive + }, + metadata_disposition: MetadataDisposition::Tombstone, + archive_retention: ArchiveRetention::Forever, + // Runtime calls diagnostic archiving Retain, not Archive. Preserve + // the Workspace's duration, or thirty days when it normally purges. + diagnostics_disposition: if is_job { + DiagnosticsDisposition::Purge + } else { + DiagnosticsDisposition::Retain + }, + diagnostics_retention_seconds: if is_job { + None + } else { + Some( + policy + .diagnostics_retention_seconds + .unwrap_or(30 * 24 * 60 * 60), + ) + }, + }; + policy.policy_id = update.policy_id.clone(); + policy.policy_digest = worker_retention_policy_digest(&update); + policy.session_disposition = update.session_disposition; + policy.metadata_disposition = update.metadata_disposition; + policy.archive_retention = update.archive_retention; + policy.diagnostics_disposition = update.diagnostics_disposition; + policy.diagnostics_retention_seconds = update.diagnostics_retention_seconds; + return Ok(Some(policy)); + } if erased_journal || crate::store::terminal_journal_job_worker(c, w, worker)? { let update = WorkerRetentionPolicyUpdate { policy_id: if erased_journal { @@ -1047,6 +1105,39 @@ fn load_removal_policy( } Ok(Some(policy)) } +fn require_current_subject_deletion_plan( + c: &Connection, + plan: &WorkerRemovalPlan, +) -> crate::Result<()> { + if plan.state == WorkerRemovalPlanState::Succeeded + && crate::store::subject_deletion_prior_removal_matches( + c, + &plan.workspace_id, + &plan.worker, + &plan.operation_id, + )? + { + // An exact successful receipt pinned before Subject deletion owns its + // independent original/archive lifecycle. Never retarget its policy. + return Ok(()); + } + if crate::store::subject_deletion_worker_kind(c, &plan.workspace_id, &plan.worker)?.is_some() { + let current = load_removal_policy(c, &plan.workspace_id, &plan.worker, false)? + .ok_or_else(|| StoreError::InvalidInput("Subject deletion policy missing".into()))?; + if plan.session_disposition != current.session_disposition + || plan.metadata_disposition != current.metadata_disposition + || plan.archive_retention != current.archive_retention + || plan.diagnostics_disposition != current.diagnostics_disposition + { + return Err(stale_error( + plan, + "Subject deletion retention differs from saved plan; reconcile canonical Runtime removal before retry", + )); + } + } + Ok(()) +} + fn load_policy(c: &Connection, w: &str) -> crate::Result> { c.query_row( "SELECT p.policy_id,p.policy_digest,r.session_disposition,r.metadata_disposition, @@ -2736,4 +2827,78 @@ mod tests { ); assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Failed); } + + #[test] + fn subject_deletion_snapshot_overrides_policy_without_changing_normal_removal() { + let store = setup(); + let ordinary = store.plan_worker_removal(&req(), &inv()).unwrap(); + store + .prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()]) + .unwrap(); + let body = store.plan_worker_removal(&req(), &inv()).unwrap(); + assert_eq!(body.session_disposition, SessionDisposition::Archive); + assert_eq!(body.archive_retention, ArchiveRetention::Forever); + assert_eq!(body.metadata_disposition, MetadataDisposition::Tombstone); + assert_eq!(body.diagnostics_disposition, DiagnosticsDisposition::Retain); + assert!(body.diagnostics_retention_seconds.is_some()); + assert_ne!(body.policy_id, ordinary.policy_id); + // A saved plan that would discard diagnostics cannot silently stand in + // for the Subject conversation archive plan. + assert!(matches!( + store.begin_worker_removal("w", &ordinary.plan_id, &ordinary.input_fingerprint), + Err(WorkerRetentionError::StalePlan { .. }) + )); + let ordinary_store = setup(); + assert_eq!( + ordinary_store + .plan_worker_removal(&req(), &inv()) + .unwrap() + .policy_id, + ordinary.policy_id + ); + } + + #[test] + fn subject_deletion_job_purge_rejects_recovery_of_old_executing_archive() { + let store = setup(); + let old = store.plan_worker_removal(&req(), &inv()).unwrap(); + store + .prepare_worker_removal_execution("w", &old.plan_id, &old.input_fingerprint) + .unwrap(); + store + .prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()]) + .unwrap(); + store + .with_conn(|c| { + c.execute( + "UPDATE subject_deletion_workers SET is_job=1 WHERE workspace_id='w'", + [], + )?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + store.recover_worker_removal_execution("w", &req().worker), + Err(WorkerRetentionError::StalePlan { .. }) + )); + assert!(matches!( + store.plan_worker_removal(&req(), &inv()), + Err(WorkerRetentionError::StalePlan { .. }) + )); + let fresh = store + .with_conn(|c| load_removal_policy(c, "w", &req().worker, false)) + .unwrap() + .unwrap(); + assert_eq!(fresh.session_disposition, SessionDisposition::Purge); + assert_eq!(fresh.diagnostics_disposition, DiagnosticsDisposition::Purge); + assert_eq!(fresh.metadata_disposition, MetadataDisposition::Tombstone); + assert_eq!( + store + .with_conn(|c| load_plan(c, &old.plan_id)) + .unwrap() + .unwrap() + .state, + WorkerRemovalPlanState::Executing + ); + } } diff --git a/crates/workspace-server/src/schema_convergence_tests.rs b/crates/workspace-server/src/schema_convergence_tests.rs index 8870441ca..a918b0950 100644 --- a/crates/workspace-server/src/schema_convergence_tests.rs +++ b/crates/workspace-server/src/schema_convergence_tests.rs @@ -91,10 +91,16 @@ mod schema_convergence { assert_eq!(plan.current_schema_version, 103); assert_eq!( plan.migrations, - vec![WorkspaceSchemaMigrationStep { - version: 104, - name: "Converge Passkey results and Job input retention".into(), - }] + vec![ + WorkspaceSchemaMigrationStep { + version: 104, + name: "Converge Passkey results and Job input retention".into(), + }, + WorkspaceSchemaMigrationStep { + version: 105, + name: "Durable Subject deletion snapshots and Job admission fences".into(), + }, + ] ); assert_eq!( schema_rows(&conn), @@ -109,6 +115,7 @@ mod schema_convergence { assert_eq!(feature_rows(conn, job_variant), rows); let mut expected = history.clone(); expected.push((104, "Converge Passkey results and Job input retention".into())); + expected.push((105, "Durable Subject deletion snapshots and Job admission fences".into())); assert_eq!(workspace_schema_migration_history(conn)?, expected); for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] { assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}"); @@ -138,7 +145,7 @@ mod schema_convergence { assert_eq!(feature_rows(conn, job_variant), rows); assert_eq!( workspace_schema_migration_history(conn)?.len(), - history.len() + 1 + history.len() + 2 ); Ok(()) }) @@ -176,7 +183,7 @@ mod schema_convergence { upgraded .with_conn(|conn| { assert_eq!(feature_rows(conn, job_variant), rows); - assert_eq!(current_schema_version(conn)?, 104); + assert_eq!(current_schema_version(conn)?, 105); Ok(()) }) .unwrap(); diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index cc505f91a..c08bbdfc7 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -28,6 +28,7 @@ use subjektiv::api::*; mod backend_job_profile; mod external_operation_audit; mod subject_authorization; +mod subjektiv_deletion; mod subjektiv_job; mod subjektiv_journal; mod subjektiv_journal_browser; @@ -4577,6 +4578,12 @@ impl WorkspaceApi { .run(move || recover_workdir_removals(&workdir_recovery_api)) .await?; recover_runtime_removals(&api).await; + let deletion_api = api.clone(); + let deletion_runtime = tokio::runtime::Handle::current(); + api.host_executors + .host()? + .run(move || deletion_runtime.block_on(subjektiv_deletion::recover(&deletion_api))) + .await?; api.host_executors .host()? .run(move || { @@ -4604,9 +4611,23 @@ impl WorkspaceApi { ) -> Result { self.resolve_backend_job_profile(request)?; let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - let reservation = + let reservation = { + let gate = subjektiv_deletion::job_subject(request) + .map(|subject| subjektiv_journal::lifecycle_lock(self, subject)) + .transpose()?; + let _guard = gate + .as_ref() + .map(|gate| { + gate.lock() + .map_err(|_| Error::Store("Subject lifecycle lock poisoned".into())) + }) + .transpose()?; + if let Some(subject) = subjektiv_deletion::job_subject(request) { + subjektiv_deletion::require_available(self, subject)?; + } self.store - .reserve_backend_job(&self.config.workspace_id, request, &now)?; + .reserve_backend_job(&self.config.workspace_id, request, &now)? + }; let result = self.dispatch_backend_job_reservation(reservation); if result.is_err() { self.drain_reserved_backend_jobs(); @@ -4621,9 +4642,27 @@ impl WorkspaceApi { /// immutable request. pub fn retry_backend_job(&self, job_id: &str) -> Result { let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); - let reservation = + let reservation = { + let current = self + .store + .get_backend_job(self.workspace_id(), job_id)? + .ok_or_else(|| Error::InvalidInput("unknown Backend Job".into()))?; + let gate = subjektiv_deletion::job_subject(¤t.request) + .map(|subject| subjektiv_journal::lifecycle_lock(self, subject)) + .transpose()?; + let _guard = gate + .as_ref() + .map(|gate| { + gate.lock() + .map_err(|_| Error::Store("Subject lifecycle lock poisoned".into())) + }) + .transpose()?; + if let Some(subject) = subjektiv_deletion::job_subject(¤t.request) { + subjektiv_deletion::require_available(self, subject)?; + } self.store - .reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?; + .reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)? + }; let result = self.dispatch_backend_job_reservation(reservation); if result.is_err() { self.drain_reserved_backend_jobs(); @@ -4637,13 +4676,8 @@ impl WorkspaceApi { ) -> Result { // Reread even a terminal/stale reservation before deciding whether it can // dispatch or replay: an explicit erasure may have retired its saved input. - let journal_gate = reservation - .job - .request - .grants - .subjektiv_journal - .as_ref() - .map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id)) + let journal_gate = subjektiv_deletion::job_subject(&reservation.job.request) + .map(|subject| subjektiv_journal::lifecycle_lock(self, subject)) .transpose()?; let _journal_guard = journal_gate .as_ref() @@ -4652,6 +4686,9 @@ impl WorkspaceApi { .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into())) }) .transpose()?; + if let Some(subject) = subjektiv_deletion::job_subject(&reservation.job.request) { + subjektiv_deletion::require_available(self, subject)?; + } let current_job = self .store .get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)? @@ -5571,6 +5608,27 @@ impl WorkspaceApi { source_worker: &RuntimeWorkerRef, submission: &BackendJobResultSubmission, ) -> Result { + let result_job = self + .store + .get_backend_job(self.workspace_id(), &submission.job_id)?; + let subject_gate = result_job + .as_ref() + .and_then(|job| subjektiv_deletion::job_subject(&job.request)) + .map(|subject| subjektiv_journal::lifecycle_lock(self, subject)) + .transpose()?; + let subject_guard = subject_gate + .as_ref() + .map(|gate| { + gate.lock() + .map_err(|_| Error::Store("Subject lifecycle lock poisoned".into())) + }) + .transpose()?; + if let Some(subject) = result_job + .as_ref() + .and_then(|job| subjektiv_deletion::job_subject(&job.request)) + { + subjektiv_deletion::require_available(self, subject)?; + } let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); let mut verified_submission = submission.clone(); if let Some(job) = self @@ -5696,6 +5754,7 @@ impl WorkspaceApi { &verified_submission, &accepted_at, )?; + drop(subject_guard); if let Some(job) = self .store .get_backend_job(&self.config.workspace_id, &submission.job_id)? @@ -5899,15 +5958,21 @@ impl WorkspaceApi { .map(|assignment| assignment.operation_id.clone()) }) .unwrap_or_else(|| format!("manual:{}", WorkerId::now_v7())); - let subject = singleton_key + let body_subject = singleton_key .as_deref() .and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX)); - if let Some(subject) = subject + if let Some(subject) = body_subject && let Err(error) = subjektiv_journal::recover_failed_subject_create(self, subject) { let diagnostics = cleanup_spawn_created_workdirs(self, &spawned_workdir_ids); return Err(ApiError::with_diagnostics(error, diagnostics)); } + let subject = body_subject.or_else(|| { + request + .resolved_execution_grant + .as_ref() + .map(|grant| grant.subject_id.as_str()) + }); let subject_gate = subject .map(|s| subjektiv_journal::lifecycle_lock(self, s)) .transpose()?; @@ -5919,6 +5984,9 @@ impl WorkspaceApi { }) .transpose()?; if let Some(subject) = subject { + subjektiv_deletion::require_available(self, subject)?; + } + if let Some(subject) = body_subject { subjektiv_journal::check_start_allowed(self, subject)?; } let reservation = match self.config_store.reserve_worker_create( @@ -5943,11 +6011,11 @@ impl WorkspaceApi { return Err(ApiError::with_diagnostics(error, diagnostics)); } }; - drop(subject_guard); let worker_id = reservation.worker_id; if let Some(grant) = request.resolved_execution_grant.as_ref() { subject_authorization::bind_reserved_grant(self, grant, &worker_id.to_string())?; } + drop(subject_guard); // A completed exact-source allocation is historical acceptance, not a new // launch. Source/grant authorization and the full fingerprint were checked // above. Do not reserve its already-linked attachments again, redispatch @@ -7458,7 +7526,7 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou .merge(server_api::server_api_axum::subjektiv_memory_changes( service.clone(), )) - .merge(server_api::server_api_axum::subjektiv_subject_retire( + .merge(server_api::server_api_axum::subjektiv_subject_delete( service.clone(), )) .merge(server_api::server_api_axum::subjektiv_stage_candidate( @@ -9724,22 +9792,25 @@ impl server_api::ServerApi for ServerApiContractService { .map_err(ApiError::into_repository_api_error) } - async fn subjektiv_subject_retire( + async fn subjektiv_subject_delete( &self, + context: server_api::ServerRequestContext, workspace_id: String, subject_id: String, - ) -> std::result::Result - { - scoped_retire_subjektiv_subject( - State(self.workspace_api()?.clone()), - AxumPath(ScopedSubjektivSubjectPath { - workspace_id, - subject_id, - }), - ) - .await - .map(|Json(response)| response) - .map_err(ApiError::into_repository_api_error) + request: server_api::SubjektivSubjectDeleteRequest, + ) -> std::result::Result< + server_api::SubjektivSubjectDeleteResponse, + server_api::RepositoryApiError, + > { + let api = self.workspace_api()?.clone(); + let actor = subject_authorization::administration_actor(&context) + .map_err(ApiError::into_repository_api_error)?; + require_workspace_owner(&api, &workspace_id, actor, "Subject deletion") + .await + .map_err(ApiError::into_repository_api_error)?; + subjektiv_deletion::delete(api, workspace_id, subject_id, request) + .await + .map_err(ApiError::into_repository_api_error) } async fn subjektiv_stage_candidate( @@ -22674,7 +22745,9 @@ fn subjektiv_subject_response( memory_fingerprint, state: match subject.state { crate::subjektiv::SubjectState::Active => server_api::SubjektivSubjectState::Active, - crate::subjektiv::SubjectState::Retired => server_api::SubjektivSubjectState::Retired, + crate::subjektiv::SubjectState::LegacyRetired => { + server_api::SubjektivSubjectState::LegacyRetired + } }, created_at: subject.created_at, updated_at: subject.updated_at, @@ -22962,24 +23035,6 @@ async fn scoped_list_subjektiv_memory_changes( .await } -async fn scoped_retire_subjektiv_subject( - State(api): State, - AxumPath(path): AxumPath, -) -> ApiResult> { - api.host_executors - .host()? - .run(move || { - validate_workspace_scope(&api, &path.workspace_id)?; - let subject = open_subjektiv_store(&api)? - .retire_subject(&path.subject_id) - .map_err(subjektiv_store_error)?; - // Retirement deliberately does not stop or remove the current Worker. The - // Worker lease and subject lifecycle are independent authorities. - Ok(Json(subjektiv_subject_response(&api, subject)?)) - }) - .await -} - async fn scoped_stage_subjektiv_candidate( State(api): State, AxumPath(path): AxumPath, @@ -24564,6 +24619,7 @@ fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error { Error::WorkspacePermissionDenied(error.to_string()) } crate::subjektiv::SubjektivError::SubjectPermissionsConflict(_) + | crate::subjektiv::SubjektivError::SubjectDeletionConflict(_) | crate::subjektiv::SubjektivError::PermissionOperationConflict(_) | crate::subjektiv::SubjektivError::WorkerExecutionGrantConflict(_) => { Error::RepositoryConflict(error.to_string()) @@ -31543,6 +31599,7 @@ async fn workspace_response( manage_secrets: is_owner, manage_runtimes: is_owner, delete_workspace: is_owner, + delete_subjects: is_owner, }, extension_points: WorkspaceExtensionPoints { store: "sqlite".to_string(), @@ -37777,6 +37834,7 @@ mod tests { mod restore_workdir_tests; mod restored_parent_control_tests; mod subject_authorization_tests; + mod subject_deletion_tests; mod subject_spawn_tests; mod subjektiv_jobs_tests; mod subjektiv_journal_tests; @@ -46241,7 +46299,13 @@ mod tests { let retired = store .create_subject(crate::subjektiv::SubjectRole::new("retired").unwrap()) .unwrap(); - store.retire_subject(&retired.id).unwrap(); + // Frozen lifecycle fixture: migration preserves old inactive Subjects; there is + // intentionally no public operation that creates this historical state anymore. + api.feature_storage.open(&api.subjektiv_registration).unwrap() + .try_transaction::<_, feature_storage::FeatureStorageError>(|tx| { + tx.execute("UPDATE subjects SET state='retired', record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&retired.id])?; + Ok(()) + }).unwrap(); let request = |subject_id: Option<&str>, profile: &str, singleton_key: Option<&str>| { CreateWorkspaceWorkerRequest { @@ -48843,6 +48907,7 @@ mod tests { >, materializer: worker_runtime::working_directory::RuntimeGitMaterializer, spawn_failure: std::sync::Mutex>, + subject_delete_failure: std::sync::Mutex>, recovery_failure: std::sync::Mutex>, recovery_requests: std::sync::Mutex>, recovery_hook: std::sync::Mutex>>, @@ -48887,6 +48952,7 @@ mod tests { std::env::temp_dir().join(unique), ), spawn_failure: std::sync::Mutex::new(None), + subject_delete_failure: std::sync::Mutex::new(None), recovery_failure: std::sync::Mutex::new(None), recovery_requests: std::sync::Mutex::new(Vec::new()), recovery_hook: std::sync::Mutex::new(None), @@ -49242,6 +49308,24 @@ mod tests { .map(|execution| execution.snapshot()) } + fn preflight_subject_delete_worker( + &self, + _request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> std::result::Result<(), String> { + Ok(()) + } + + fn subject_delete_worker( + &self, + request: &runtime_api::SubjectDeleteWorkerRequest, + ) -> std::result::Result<(), String> { + if let Some(message) = self.subject_delete_failure.lock().unwrap().clone() { + return Err(message); + } + self.stop_worker(&worker_runtime::identity::WorkerRef::new(request.worker_id)); + Ok(()) + } + fn stop_worker( &self, worker_ref: &worker_runtime::identity::WorkerRef, diff --git a/crates/workspace-server/src/server/subject_authorization.rs b/crates/workspace-server/src/server/subject_authorization.rs index 0181161eb..d28232a46 100644 --- a/crates/workspace-server/src/server/subject_authorization.rs +++ b/crates/workspace-server/src/server/subject_authorization.rs @@ -149,6 +149,13 @@ pub(super) fn worker_permissions( api: &WorkspaceApi, worker: &RuntimeWorkerRef, ) -> ApiResult>> { + if subjektiv_deletion::deleting_worker(api, worker)? + || open_subjektiv_store(api)? + .worker_subject_deleted(&worker.runtime_id, &worker.worker_id) + .map_err(subjektiv_store_error)? + { + return Err(denied("Worker belonged to an explicitly deleted Subject")); + } if let Some(grant) = live_child_grant(api, worker)? { return Ok(Some(grant.permissions)); } @@ -226,6 +233,8 @@ pub(super) fn prepare_delegation( singleton_key: Option<&str>, subject_connection: bool, ) -> ApiResult> { + // A deleted grant must never become an ordinary non-Subject delegation source. + worker_permissions(api, source)?; let child = live_child_grant(api, source)?; let body = if child.is_none() { current_body_subject(api, source)? diff --git a/crates/workspace-server/src/server/subjektiv_deletion.rs b/crates/workspace-server/src/server/subjektiv_deletion.rs new file mode 100644 index 000000000..1c380eee6 --- /dev/null +++ b/crates/workspace-server/src/server/subjektiv_deletion.rs @@ -0,0 +1,288 @@ +//! Explicit Subject destruction is independent of ordinary Worker retention. The domain +//! fence commits before external effects, remains after failures, and rejects cached grants. +//! Worker originals are archived; Job execution copies are purged. Existing archives, Workdirs, +//! Tickets and other Subjects are not erased. Archive contents can include previously injected +//! Memory: their independent retention/access policy still applies. This is logical content +//! deletion, not a promise to erase backups, external providers or already exported material. +//! +//! Child removal/create-fence receipts, not acceptance, Idle or missing projections, prove +//! quiescence. A retry carries the same scope/Subject/operation; no new child identity is issued. +//! The domain receipt is authoritative completion. Backend inventories retain only identifiers +//! needed for cleanup and fail-closed authorization, never a copy of Subject content. +use super::*; +use rusqlite::params; + +pub(super) fn job_subject(request: &BackendJobRequest) -> Option<&str> { + request + .grants + .subjektiv_journal + .as_ref() + .map(|g| g.subject_id.as_str()) + .or_else(|| { + request + .grants + .subjektiv_consolidation + .as_ref() + .map(|g| g.subject_id.as_str()) + }) +} + +pub(super) fn require_available(api: &WorkspaceApi, subject: &str) -> Result<()> { + let store = open_subjektiv_store(api).map_err(|error| error.error)?; + if store + .subject(subject) + .map_err(|e| Error::Store(e.to_string()))? + .is_none() + { + return Err(Error::SubjektivSubjectNotFound(subject.into())); + } + Ok(()) +} + +pub(super) fn pending(api: &WorkspaceApi, subject: &str) -> Result { + api.config_store.with_conn(|connection| { + Ok(connection.query_row( + "SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND subject_id=?2)", + params![api.workspace_id(), subject], |row| row.get(0), + )?) + }) +} + +/// This is internal cleanup evidence, never a user-supplied removal flag. It exists only +/// after the independently durable domain fence and exact-target inventory commit. +pub(super) fn deleting_worker(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Result { + api.config_store.with_conn(|connection| { + Ok(connection.query_row( + "SELECT EXISTS(SELECT 1 FROM subject_deletion_workers WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3)", + params![api.workspace_id(), worker.runtime_id, worker.worker_id], |row| row.get(0), + )?) + }) +} + +pub(super) async fn delete( + api: WorkspaceApi, + workspace: String, + subject: String, + request: server_api::SubjektivSubjectDeleteRequest, +) -> ApiResult { + let handle = tokio::runtime::Handle::current(); + api.host_executors + .host()? + .run(move || { + validate_workspace_scope(&api, &workspace)?; + handle.block_on(resume(&api, &subject, &request.operation_id)) + }) + .await +} + +pub(super) async fn recover(api: &WorkspaceApi) -> Result<()> { + let pending = open_subjektiv_store(api) + .map_err(|error| error.error)? + .pending_subject_deletions() + .map_err(|e| Error::Store(e.to_string()))?; + for deletion in pending { + if let Err(error) = resume(api, &deletion.subject_id, &deletion.operation_id).await { + tracing::warn!(%error.error, subject_id=%deletion.subject_id, + "Subject deletion restart recovery remains incomplete; same request can resume"); + } + } + Ok(()) +} + +async fn resume( + api: &WorkspaceApi, + subject: &str, + operation: &str, +) -> ApiResult { + let store = open_subjektiv_store(api)?; + let gate = subjektiv_journal::lifecycle_lock(api, subject)?; + let receipt = { + let _guard = gate + .lock() + .map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?; + let receipt = store + .begin_subject_deletion(subject, operation) + .map_err(subjektiv_store_error)?; + if !receipt.completed { + let delegated = store + .deletion_subject_workers(subject) + .map_err(subjektiv_store_error)? + .into_iter() + .map(|(runtime, worker)| RuntimeWorkerRef::new(runtime, worker)) + .collect::>(); + api.config_store.prepare_subject_deletion( + api.workspace_id(), + subject, + operation, + &delegated, + )?; + } + receipt + }; + let response = |completed| server_api::SubjektivSubjectDeleteResponse { + subject_id: subject.into(), + operation_id: operation.into(), + completed, + }; + if receipt.completed { + return Ok(response(true)); + } + // Never hold the Host callback gate across a Runtime call. A create/restore can be + // waiting for the callback that now has to reject this permanently fenced Subject. + let workers = + api.config_store + .list_subject_deletion_workers(api.workspace_id(), subject, operation)?; + for (worker, _is_job) in workers { + if let Err(error) = remove_worker(api, subject, operation, &worker).await { + tracing::warn!(%error.error, subject_id=subject, operation_id=operation, "Subject deletion remains incomplete; exact retry required"); + return Ok(response(false)); + } + } + { + let _guard = gate + .lock() + .map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?; + api.config_store + .erase_subject_deleted_jobs(api.workspace_id(), subject, operation)?; + // Finish Backend cleanup before the authoritative domain receipt. If domain + // purge fails, its separately committed fence and pending operation survive. + // No retry can report domain completion while occupancy/Job cleanup is unproven. + api.config_store + .finish_subject_deletion_record(api.workspace_id(), subject, operation)?; + store + .finish_subject_deletion(subject, operation) + .map_err(subjektiv_store_error)?; + } + Ok(response(true)) +} + +async fn remove_worker( + api: &WorkspaceApi, + subject: &str, + operation: &str, + worker: &RuntimeWorkerRef, +) -> ApiResult<()> { + let request = api.config_store.subject_deletion_worker_request( + api.workspace_id(), + subject, + operation, + worker, + )?; + let fence = api + .runtime + .subject_delete_worker(&worker.runtime_id, request.clone()) + .map_err(RuntimeRegistryError::into_error)?; + if fence.workspace_id != request.workspace_id + || fence.worker_id != request.worker_id + || fence.create_fingerprint != request.create_fingerprint + || fence.operation_id != request.operation_id + { + return Err(Error::RepositoryConflict( + "Subject Worker deletion fence receipt mismatch".into(), + ) + .into()); + } + // Replay the canonical child even when physical removal already happened: + // its retained receipt can finish the Backend catalog midpoint. The Runtime + // fence above also prevents an older create from arriving after that removal. + if api + .config_store + .recover_worker_removal_execution(api.workspace_id(), worker) + .map_err(|error| Error::Store(error.to_string()))? + .is_some() + { + return execute_removal(api, operation, worker).await; + } + match api.runtime.worker(worker) { + Ok(detail) => { + if !detail.state.eq_ignore_ascii_case("stopped") { + let stopped = api + .runtime + .stop_worker( + worker, + WorkerLifecycleRequest { + reason: Some(format!("Explicit Subject deletion {operation}")), + ticket_assignment: None, + }, + ) + .map_err(RuntimeRegistryError::into_error)?; + if stopped.state != InternalWorkerOperationState::Accepted { + return Err(Error::RepositoryConflict( + "Subject Worker stop was not accepted".into(), + ) + .into()); + } + } + let actual = api + .runtime + .worker(worker) + .map_err(RuntimeRegistryError::into_error)?; + if !actual.state.eq_ignore_ascii_case("stopped") { + return Err( + Error::RepositoryConflict("Subject Worker stop is unconfirmed".into()).into(), + ); + } + if api + .store + .get_worker_registry(api.workspace_id(), worker)? + .is_none() + { + // The Runtime fence authenticated this exact reserved fingerprint. A create + // can have reached Runtime before its Backend publication failed. Publish + // only its stopped record so the normal archive/Purge owner can remove it. + record_worker_summary( + api, + &actual, + &actual.display_name, + actual.profile.clone(), + WorkerRegistryDisplayNamePolicy::PreserveExisting, + )?; + } + } + Err(RuntimeRegistryError::UnknownWorker { .. }) => { + if api + .store + .get_worker_registry(api.workspace_id(), worker)? + .is_some() + { + return Err(Error::RepositoryConflict( + "Runtime object missing before canonical original archival".into(), + ) + .into()); + } + api.config_store.confirm_subject_deletion_absence( + api.workspace_id(), + subject, + operation, + worker, + &fence, + )?; + return Ok(()); + } + Err(error) => return Err(error.into_error().into()), + } + execute_removal(api, operation, worker).await +} + +async fn execute_removal( + api: &WorkspaceApi, + operation: &str, + worker: &RuntimeWorkerRef, +) -> ApiResult<()> { + let response = WorkerRemovalService::new(api) + .execute_target_removal_on_host( + api.runtime.as_ref(), + worker, + &format!("Explicit Subject deletion {operation}"), + None, + ) + .await + .map_err(Error::Store)?; + if response.status != StatusCode::OK.as_u16() { + return Err(Error::RepositoryConflict( + "Subject Worker archival/Purge remains unconfirmed".into(), + ) + .into()); + } + Ok(()) +} diff --git a/crates/workspace-server/src/server/subjektiv_journal.rs b/crates/workspace-server/src/server/subjektiv_journal.rs index fccee17a6..aba05bf94 100644 --- a/crates/workspace-server/src/server/subjektiv_journal.rs +++ b/crates/workspace-server/src/server/subjektiv_journal.rs @@ -446,6 +446,7 @@ pub(super) fn bind_host_session( /// Runs under the same Subject gate as final revoke/singleton release. A failed create or /// legacy removal may have released its Backend reservation but cannot hide a durable writer. pub(super) fn check_start_allowed(api: &WorkspaceApi, subject: &str) -> Result<()> { + subjektiv_deletion::require_available(api, subject)?; let store = store(api)?; let key = crate::subjektiv::subject_worker_singleton_key(subject) .map_err(|e| Error::InvalidInput(e.to_string()))?; @@ -1747,6 +1748,13 @@ pub(super) fn prepare_worker_removal( runtime: &RuntimeRegistry, target: &RuntimeWorkerRef, ) -> Result> { + if let Some(api) = service.journal_dispatch_api.upgrade() + && subjektiv_deletion::deleting_worker(&api, target)? + { + // Full Subject deletion has a permanent domain fence. It deliberately drops + // Journal continuity, but retains source originals through its archive policy. + return Ok(None); + } let Some(attr) = removal_attribution(service, target)? else { if service .store @@ -1809,6 +1817,13 @@ pub(super) fn erased_archive_removal_proof( target: &RuntimeWorkerRef, prepared: &crate::retention::PreparedWorkerRemoval, ) -> Result> { + if let Some(api) = service.journal_dispatch_api.upgrade() + && subjektiv_deletion::deleting_worker(&api, target)? + { + // Resume the exact canonical retention request after full Subject fencing. + // Subject deletion retains originals; it is not per-Session source erasure. + return Ok(None); + } let Some(attr) = removal_attribution(service, target)? else { return Ok(None); }; @@ -1852,6 +1867,13 @@ pub(super) fn validate_removal_retention( target: &RuntimeWorkerRef, prepared: &crate::retention::PreparedWorkerRemoval, ) -> Result<()> { + if let Some(api) = service.journal_dispatch_api.upgrade() + && subjektiv_deletion::deleting_worker(&api, target)? + { + // The plan-local retention policy and canonical Runtime receipt remain required. + // Neither unfinished Journals nor a stopped old Controller are deletion blockers. + return Ok(()); + } let Some(attr) = removal_attribution(service, target)? else { if service .store @@ -2066,6 +2088,9 @@ pub(super) fn session_erased(api: &WorkspaceApi, request: &BackendJobRequest) -> let Some(grant) = &request.grants.subjektiv_journal else { return Ok(false); }; + if subjektiv_deletion::pending(api, &grant.subject_id)? { + return Ok(true); + } let store = store(api)?; store .journal_session_erased(&host(api, &store, grant)?) diff --git a/crates/workspace-server/src/server/tests/subject_deletion_tests.rs b/crates/workspace-server/src/server/tests/subject_deletion_tests.rs new file mode 100644 index 000000000..298eea8a0 --- /dev/null +++ b/crates/workspace-server/src/server/tests/subject_deletion_tests.rs @@ -0,0 +1,729 @@ +//! Real router/authentication, feature store and Backend DB boundaries. Runtime +//! execution/unknown and restart fences are covered in worker-runtime and the +//! canonical removal suites, rather than claiming HTTP acceptance proves shutdown. +use super::*; + +async fn delete_request( + api: &WorkspaceApi, + subject: &str, + key: &str, + token: Option<&str>, + workspace: &str, +) -> (StatusCode, Value) { + let mut builder = Request::builder() + .method("DELETE") + .uri(format!("/api/w/{workspace}/subjektiv/subjects/{subject}")) + .header(CONTENT_TYPE, "application/json"); + if let Some(token) = token { + builder = builder.header("authorization", format!("Bearer {token}")); + } + let response = build_router(api.clone()) + .oneshot( + builder + .body(Body::from(json!({"operation_id":key}).to_string())) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), 2 * 1024 * 1024) + .await + .unwrap(); + let body = serde_json::from_slice(&bytes) + .unwrap_or_else(|_| json!({"text":String::from_utf8_lossy(&bytes)})); + (status, body) +} + +#[tokio::test] +async fn deletion_owner_scope_exact_retry_and_subject_isolation() { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject_with_behavior( + crate::subjektiv::SubjectRole::new("delete me").unwrap(), + "private behavior".into(), + ) + .unwrap(); + let other = store + .create_subject_with_behavior( + crate::subjektiv::SubjectRole::new("keep me").unwrap(), + "unchanged".into(), + ) + .unwrap(); + let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-owner").await; + let stranger = seed_test_api_token(api.store.as_ref(), "subject-delete-stranger"); + for (token, workspace, expected) in [ + (None, TEST_WORKSPACE_ID, StatusCode::UNAUTHORIZED), + ( + Some(stranger.as_str()), + TEST_WORKSPACE_ID, + StatusCode::FORBIDDEN, + ), + ( + Some(owner.as_str()), + "another-workspace", + StatusCode::NOT_FOUND, + ), + ] { + let (status, body) = + delete_request(&api, &subject.id, "same-operation", token, workspace).await; + assert_eq!(status, expected, "{body}"); + assert!(store.subject(&subject.id).unwrap().is_some()); + } + let (status, body) = delete_request( + &api, + &subject.id, + "same-operation", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["completed"], true); + assert!(store.subject(&subject.id).unwrap().is_none()); + assert_eq!( + store.subject(&other.id).unwrap().unwrap().behavior_md, + "unchanged" + ); + assert_eq!(store.list_subjects(100).unwrap().items.len(), 1); + let (status, replay) = delete_request( + &api, + &subject.id, + "same-operation", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{replay}"); + assert_eq!(replay, body); + let (status, _) = delete_request( + &api, + &subject.id, + "different-operation", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + let (status, _) = delete_request( + &api, + &other.id, + "same-operation", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + assert!(store.subject(&other.id).unwrap().is_some()); + let fresh = store + .create_subject(crate::subjektiv::SubjectRole::new("fresh").unwrap()) + .unwrap(); + assert_ne!(fresh.id, subject.id); + assert!(store.subject_permissions(&fresh.id).unwrap().is_none()); + assert_eq!( + store + .pending_staging_candidates(&fresh.id, 100) + .unwrap() + .len(), + 0 + ); +} + +#[tokio::test] +async fn legacy_inactive_subject_is_preserved_until_explicit_delete_and_retire_route_is_gone() { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject_with_behavior( + crate::subjektiv::SubjectRole::new("legacy fixture").unwrap(), + "retained original".into(), + ) + .unwrap(); + api.feature_storage.open(&api.subjektiv_registration).unwrap() + .try_transaction::<_, feature_storage::FeatureStorageError>(|tx| { + tx.execute("UPDATE subjects SET state='retired',record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&subject.id])?; + Ok(()) + }).unwrap(); + assert_eq!( + store.subject(&subject.id).unwrap().unwrap().state, + crate::subjektiv::SubjectState::LegacyRetired + ); + let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-legacy").await; + let request = Request::builder() + .method("POST") + .uri(format!( + "/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/retire", + subject.id + )) + .header("authorization", format!("Bearer {owner}")) + .body(Body::empty()) + .unwrap(); + let old = build_router(api.clone()).oneshot(request).await.unwrap(); + assert_eq!(old.status(), StatusCode::METHOD_NOT_ALLOWED); + assert_eq!( + store.subject(&subject.id).unwrap().unwrap().behavior_md, + "retained original" + ); + let (status, body) = delete_request( + &api, + &subject.id, + "explicit-legacy-delete", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["completed"], true); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn unconfirmed_worker_shutdown_is_not_completion_and_exact_retry_converges() { + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject_with_behavior( + crate::subjektiv::SubjectRole::new("delete connected body").unwrap(), + "private behavior".into(), + ) + .unwrap(); + let Json(created) = create_workspace_worker( + State(api.clone()), + HeaderMap::new(), + Json(CreateWorkspaceWorkerRequest { + runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), + display_name: "Subject deletion boundary".into(), + singleton_key: None, + profile: Some("builtin:companion".into()), + ticket_assignment: None, + initial_submit: vec![], + workdir_attachments: vec![], + feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest { + subjektiv: Some(server_api::SubjektivWorkerConnectionRequest { + subject_id: subject.id.clone(), + }), + }, + control_operation_id: None, + execution_grant: None, + }), + ) + .await + .unwrap(); + let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id); + seed_worker_session_for_cleanup(dir.path(), &worker); + let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unknown").await; + *execution.subject_delete_failure.lock().unwrap() = Some("shutdown completion unknown".into()); + let (status, body) = delete_request( + &api, + &subject.id, + "unknown-delete", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["completed"], false); + assert!(store.subject(&subject.id).unwrap().is_none()); + assert!( + !store + .subject_deletion(&subject.id) + .unwrap() + .unwrap() + .completed + ); + assert!(subject_authorization::worker_permissions(&api, &worker).is_err()); + assert!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id)) + .unwrap() + .is_some() + ); + *execution.subject_delete_failure.lock().unwrap() = None; + api.config_store.with_conn(|conn| { + conn.execute_batch("CREATE TRIGGER injected_worker_catalog_failure BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected catalog failure'); END;")?; + Ok(()) + }).unwrap(); + let (status, midpoint) = delete_request( + &api, + &subject.id, + "unknown-delete", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{midpoint}"); + assert_eq!(midpoint["completed"], false); + assert!(matches!( + api.runtime.worker(&worker), + Err(RuntimeRegistryError::UnknownWorker { .. }) + )); + assert!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &worker) + .unwrap() + .is_some() + ); + api.config_store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER injected_worker_catalog_failure;")?; + Ok(()) + }) + .unwrap(); + let (status, body) = delete_request( + &api, + &subject.id, + "unknown-delete", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["completed"], true, "{body}"); + assert!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id)) + .unwrap() + .is_none() + ); + let removal = api + .config_store + .recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker) + .unwrap() + .unwrap(); + assert_eq!( + removal.plan.state, + crate::retention::WorkerRemovalPlanState::Succeeded + ); + assert_eq!( + removal.runtime_request.session_disposition, + worker_runtime::retention::SessionDisposition::Archive + ); + assert!( + store + .subject_deletion(&subject.id) + .unwrap() + .unwrap() + .completed + ); +} + +#[tokio::test] +async fn captured_subject_job_result_cannot_deliver_after_domain_fence() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(crate::subjektiv::SubjectRole::new("notification fence").unwrap()) + .unwrap(); + let target = spawn_ticket_check_source(&api, "subject-notification-target"); + let worker = spawn_ticket_check_source(&api, "subject-notification-result"); + let request = BackendJobRequest { + job_id: "captured-subject-notification".into(), + purpose: "subject_notification_contract".into(), + input_ref: "subject:notification".into(), + input: json!({"private":"input"}), + instruction: "Read input".into(), + profile: "builtin:backend-job".into(), + grants: crate::backend_job::BackendJobGrants { + subjektiv_consolidation: Some(crate::backend_job::SubjektivConsolidationGrant { + subject_id: subject.id.clone(), + candidate_ids: vec!["candidate".into()], + }), + ..Default::default() + }, + serialization_key: None, + source_worker: Some(target.clone()), + notification_target: Some(target), + limits: Default::default(), + }; + // Enter at the committed result boundary: execution/result validation is + // outside this notification test, but routing and Runtime transport are real. + let now = Utc::now().to_rfc3339(); + let reserved = api + .store + .reserve_backend_job(TEST_WORKSPACE_ID, &request, &now) + .unwrap(); + let attempt = &reserved.attempt.attempt_id; + api.store + .claim_backend_job_attempt_dispatch(TEST_WORKSPACE_ID, &request.job_id, attempt, &now) + .unwrap(); + api.store + .bind_backend_job_attempt_worker( + TEST_WORKSPACE_ID, + &request.job_id, + attempt, + &worker, + Some("run"), + &now, + ) + .unwrap(); + let captured = api + .store + .accept_backend_job_result( + TEST_WORKSPACE_ID, + &worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: attempt.clone(), + input_digest: request.input_digest().unwrap(), + result: json!({"private":"result"}), + }, + &now, + ) + .unwrap(); + let delivery = crate::backend_job::delivery_id(&request.job_id, attempt); + assert!( + api.store + .job_notification_source(TEST_WORKSPACE_ID, &request.job_id, attempt) + .is_ok() + ); + execution.take_inputs(); + store + .begin_subject_deletion(&subject.id, "notification-delete") + .unwrap(); + notification_delivery::deliver_job_result(&api, &captured); + notification_delivery::recover_job_notifications(&api); + assert!( + execution.take_inputs().is_empty(), + "captured or recovered result must not escape the domain fence" + ); + assert!( + api.store + .job_notification_request(TEST_WORKSPACE_ID, &delivery) + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn domain_purge_failure_keeps_fence_and_backend_reopen_resumes_exact_operation() { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject_with_behavior( + crate::subjektiv::SubjectRole::new("retry purge").unwrap(), + "do not resurrect".into(), + ) + .unwrap(); + let other = store + .create_subject_with_behavior( + crate::subjektiv::SubjectRole::new("untouched").unwrap(), + "keep".into(), + ) + .unwrap(); + let database = api + .feature_storage + .open(&api.subjektiv_registration) + .unwrap(); + database.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| { + tx.execute_batch("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected domain purge failure'); END;")?; + Ok(()) + }).unwrap(); + let owner = + seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-purge-failure").await; + let (status, body) = delete_request( + &api, + &subject.id, + "purge-retry", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert!(status.is_server_error(), "{status}: {body}"); + assert!( + !store + .subject_deletion(&subject.id) + .unwrap() + .unwrap() + .completed + ); + assert!(store.subject(&subject.id).unwrap().is_none()); + assert_eq!( + store.subject(&other.id).unwrap().unwrap().behavior_md, + "keep" + ); + database + .try_transaction::<_, feature_storage::FeatureStorageError>(|tx| { + tx.execute_batch("DROP TRIGGER injected_subject_purge_failure;")?; + Ok(()) + }) + .unwrap(); + let mut reopened = api.clone(); + let config_store = + Arc::new(SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap()); + reopened.config_store = config_store.clone(); + reopened.store = config_store; + let recovery_api = reopened.clone(); + let handle = tokio::runtime::Handle::current(); + reopened + .host_executors + .host() + .unwrap() + .run(move || handle.block_on(subjektiv_deletion::recover(&recovery_api))) + .await + .unwrap(); + let recovered = open_subjektiv_store(&reopened) + .unwrap() + .subject_deletion(&subject.id) + .unwrap() + .unwrap(); + assert_eq!(recovered.operation_id, "purge-retry"); + assert!(recovered.completed); + let (status, body) = delete_request( + &reopened, + &subject.id, + "purge-retry", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["completed"], true); + assert_eq!( + store.subject(&other.id).unwrap().unwrap().behavior_md, + "keep" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn pending_consolidation_is_purged_and_captured_dispatch_and_late_result_are_rejected() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(crate::subjektiv::SubjectRole::new("delete running Job").unwrap()) + .unwrap(); + let candidate: memory::extract::StagingRecord = serde_json::from_value(json!({ + "schema_version": 2, "id": "delete-job-candidate", "extract_run_id": "extract", + "source": {"segment_id":"segment", "range":[1,2]}, "kind":"constraint", + "claim":"private candidate marker", "why_useful":"retained only until explicit erasure", + "evidence": [], "source_refs": [] + })) + .unwrap(); + store + .stage_candidate(crate::subjektiv::SubjectStagingRecord::attach( + &subject.id, + candidate, + )) + .unwrap(); + let started = start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true }, + ) + .await + .unwrap(); + assert_eq!(started.status, "started"); + let job = api + .store + .find_active_backend_job_for_resource( + TEST_WORKSPACE_ID, + &crate::backend_job::subjektiv_serialization_key(&subject.id), + ) + .unwrap() + .unwrap(); + let attempt = api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &job.request.job_id, + &crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt), + ) + .unwrap() + .unwrap(); + let worker = attempt.worker.clone().unwrap(); + seed_worker_session_for_cleanup(dir.path(), &worker); + let captured = BackendJobReservation { + job: job.clone(), + attempt: attempt.clone(), + replayed: true, + resource_reused: false, + }; + let owner = + seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-consolidation").await; + *execution.subject_delete_failure.lock().unwrap() = Some("completion unknown".into()); + let (status, body) = delete_request( + &api, + &subject.id, + "delete-job", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["completed"], false); + assert!( + api.dispatch_backend_job_reservation(captured.clone()) + .is_err() + ); + assert!( + api.accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_digest: job.input_digest.clone(), + result: json!({"late":true}), + } + ) + .is_err() + ); + *execution.subject_delete_failure.lock().unwrap() = None; + let (status, body) = delete_request( + &api, + &subject.id, + "delete-job", + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["completed"], true); + assert!( + api.store + .get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id) + .unwrap() + .is_none() + ); + assert!( + api.store + .get_backend_job_attempt(TEST_WORKSPACE_ID, &job.request.job_id, &attempt.attempt_id) + .unwrap() + .is_none() + ); + assert!(api.dispatch_backend_job_reservation(captured).is_err()); + assert!(api.dispatch_backend_job(&job.request).is_err()); + let removal = api + .config_store + .recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker) + .unwrap() + .unwrap(); + assert_eq!( + removal.plan.state, + crate::retention::WorkerRemovalPlanState::Succeeded + ); + assert_eq!( + removal.runtime_request.session_disposition, + worker_runtime::retention::SessionDisposition::Purge + ); + assert_eq!( + removal.runtime_request.diagnostics_disposition, + worker_runtime::retention::DiagnosticsDisposition::Purge + ); + assert!( + !dir.path() + .join(".test-embedded-runtime-store/workers") + .join(&worker.worker_id) + .exists() + ); +} + +#[tokio::test] +async fn unmaterialized_body_and_job_reservations_finish_only_with_runtime_storage_absence_proof() { + let dir = tempfile::tempdir().unwrap(); + let (api, _) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let owner = + seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unmaterialized").await; + for is_job in [false, true] { + let subject = store + .create_subject(crate::subjektiv::SubjectRole::new("reserved before deletion").unwrap()) + .unwrap(); + let singleton = (!is_job).then(|| format!("subjektiv:{}", subject.id)); + let allocation = if is_job { + let request = BackendJobRequest { + job_id: "unmaterialized-subject-job".into(), + purpose: "subject_reservation_contract".into(), + input_ref: "subject:reserved".into(), + input: json!({"private":"reserved input"}), + instruction: "read".into(), + profile: "builtin:backend-job".into(), + grants: crate::backend_job::BackendJobGrants { + subjektiv_consolidation: Some( + crate::backend_job::SubjektivConsolidationGrant { + subject_id: subject.id.clone(), + candidate_ids: vec!["candidate".into()], + }, + ), + ..Default::default() + }, + serialization_key: None, + source_worker: None, + notification_target: None, + limits: Default::default(), + }; + let reserved = api + .store + .reserve_backend_job(TEST_WORKSPACE_ID, &request, &Utc::now().to_rfc3339()) + .unwrap(); + crate::backend_job::allocation_key(&reserved.attempt.attempt_id) + } else { + "unmaterialized-subject-body".into() + }; + let settings = api + .config_store + .get_workspace_memory_settings(TEST_WORKSPACE_ID) + .unwrap(); + let reserved = api + .config_store + .reserve_worker_create( + TEST_WORKSPACE_ID, + EMBEDDED_WORKER_RUNTIME_ID, + &allocation, + "unmaterialized-fingerprint", + singleton.as_deref(), + &settings, + ) + .unwrap(); + let worker = + RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, reserved.worker_id.to_string()); + assert!(matches!( + api.runtime.worker(&worker), + Err(RuntimeRegistryError::UnknownWorker { .. }) + )); + let (status, body) = delete_request( + &api, + &subject.id, + &format!("delete-reserved-{is_job}"), + Some(&owner), + TEST_WORKSPACE_ID, + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["completed"], true); + assert!(subject_authorization::worker_permissions(&api, &worker).is_err()); + let proved = api.config_store.with_conn(|conn| Ok(conn.query_row("SELECT originals_absent FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4", rusqlite::params![TEST_WORKSPACE_ID,subject.id,worker.runtime_id,worker.worker_id], |row|row.get::<_,bool>(0))?)).unwrap(); + assert!(proved); + assert!( + api.config_store + .recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker) + .unwrap() + .is_none(), + "absence is not a fabricated Purge/archive receipt" + ); + assert!( + api.config_store + .reserve_worker_create( + TEST_WORKSPACE_ID, + EMBEDDED_WORKER_RUNTIME_ID, + &allocation, + "unmaterialized-fingerprint", + singleton.as_deref(), + &settings + ) + .is_err() + ); + if let Some(singleton) = singleton { + assert!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton) + .unwrap() + .is_none() + ); + } + } +} diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs index 364438db2..76865d783 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs @@ -1143,6 +1143,8 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ &Utc::now().to_rfc3339(), ) .unwrap(); + // Discard only the original delivery, before recovery can automatically retry. + f.execution.take_inputs(); f.api.recover_backend_job_worker_cleanups().unwrap(); wait_for_backend_job_worker_cleanup( &f.api, @@ -1151,23 +1153,64 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ BackendJobWorkerCleanupState::Completed, ) .await; - // Cleanup commits before domain reconciliation; wait for its authoritative outcome. - tokio::time::timeout(std::time::Duration::from_secs(10), async { + // Cleanup commits before domain reconciliation. Job Failed is only transient: + // reconcile_unknown fences this attempt, then automatically reserves/queues its + // retry. Observe the immutable attempt outcome and the delivered new generation, + // not the Job's transient state; a manual retry would race the automatic one. + let reconciliation = tokio::time::timeout(std::time::Duration::from_secs(10), async { loop { - let job = f - .api - .store - .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) - .unwrap() - .unwrap(); - if job.state == BackendJobState::Failed { - break; + let original = f.api.store.get_backend_job_attempt( + TEST_WORKSPACE_ID, + &f.request.job_id, + &dispatched.attempt.attempt_id, + ).unwrap().unwrap(); + if original.state == BackendJobAttemptState::Failed { + assert_eq!(original.failure_category.as_deref(), Some("journal_attempt_fenced")); + assert_eq!(original.worker_cleanup_state, Some(BackendJobWorkerCleanupState::Completed)); + assert!(original.worker_cleanup_completed_at.is_some()); + if let Some(generation) = f.store.active_journal_generation(&f.host, &f.id).unwrap() + && generation.job.attempt_id != original.attempt_id + && let Some(retry) = f.api.store.get_backend_job_attempt( + TEST_WORKSPACE_ID, + &f.request.job_id, + &generation.job.attempt_id, + ).unwrap() + && retry.state == BackendJobAttemptState::Dispatched + && let Some(resumed) = &retry.worker + // Worker binding precedes input delivery/attestation. ReadState + // succeeds only once the new generation's delivery is attested. + && subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok() + { + break (original, retry, generation); + } } - tokio::task::yield_now().await; + tokio::time::sleep(std::time::Duration::from_millis(10)).await; } }) - .await - .unwrap(); + .await; + assert!( + reconciliation.is_ok(), + "post-cleanup reconciliation timed out: job={:?}, original_attempt={:?}, active_generation={:?}", + f.api + .store + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap(), + f.api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &f.request.job_id, + &dispatched.attempt.attempt_id + ) + .unwrap(), + f.store.active_journal_generation(&f.host, &f.id).unwrap(), + ); + let (fenced, retry, generation) = reconciliation.unwrap(); + assert_eq!(fenced.worker, dispatched.attempt.worker); + assert_eq!(retry.attempt, dispatched.attempt.attempt + 1); + assert_eq!(generation.job.job_id, f.request.job_id); + assert_eq!(generation.job.attempt_id, retry.attempt_id); + assert_eq!(retry.input_digest, f.request.input_digest().unwrap()); f.assert_purged(worker); assert!(f.store.journal_retention_required(&f.host).unwrap()); assert_eq!( @@ -1187,25 +1230,25 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ 1, "source pin survives failed/partial Job cleanup" ); - f.execution.take_inputs(); - let retry = f + let job = f .api .store - .reserve_backend_job_retry( - TEST_WORKSPACE_ID, - &f.request.job_id, - &Utc::now().to_rfc3339(), - ) + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() .unwrap(); - assert_eq!(retry.job.request.input, f.request.input); - assert_eq!(retry.job.input_digest, f.request.input_digest().unwrap()); - let retry = f.api.dispatch_backend_job_reservation(retry).unwrap(); - let resumed = retry.attempt.worker.as_ref().unwrap(); + assert_eq!(job.state, BackendJobState::Pending); + assert_eq!(job.current_attempt, retry.attempt); + assert_eq!( + job.load_executable_request().unwrap().input, + f.request.input + ); + assert_eq!(job.input_digest, f.request.input_digest().unwrap()); + let resumed = retry.worker.as_ref().unwrap(); let delivered = f.execution.take_inputs(); assert_eq!(delivered.len(), 1); assert_eq!( delivered[0].1, - f.request.worker_input(&retry.attempt.attempt_id).unwrap() + f.request.worker_input(&retry.attempt_id).unwrap() ); assert_eq!( f.store diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 702dfb5ac..3fb7ea555 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -92,6 +92,13 @@ use crate::backend_job::{ use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; +#[path = "subject_deletion_store.rs"] +mod subject_deletion_store; +pub(crate) use subject_deletion_store::{ + prior_removal_matches as subject_deletion_prior_removal_matches, + worker_kind as subject_deletion_worker_kind, +}; + #[path = "activation.rs"] mod activation; pub use activation::ActivationCodeIssue; @@ -119,7 +126,7 @@ pub use ticket::{ }; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 104; +const LATEST_SCHEMA_VERSION: i64 = 105; const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants"; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; @@ -447,6 +454,11 @@ const MIGRATIONS: &[Migration] = &[ name: "Converge Passkey results and Job input retention", apply: migrate_schema_103_variants_to_v104, }, + Migration { + version: 105, + name: "Durable Subject deletion snapshots and Job admission fences", + apply: subject_deletion_store::migrate, + }, ]; // Both schema-103 identities shipped. Keep the alternate implementation and accept @@ -3000,6 +3012,7 @@ impl SqliteWorkspaceStore { validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?; self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + subject_deletion_store::require_worker_admission(&tx, workspace_id, allocation_key, singleton_key)?; let fixed_worker_id = prepare(&tx)?; let existing = tx .query_row( @@ -7455,6 +7468,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let deadline = backend_job_deadline(now, request.limits.timeout_seconds)?; self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + subject_deletion_store::require_job_admission(&tx, workspace_id, request)?; if let Some(job) = read_backend_job(&tx, workspace_id, &request.job_id)? { if job.intent_fingerprint != fingerprint { return Err(Error::InvalidInput(format!( @@ -7532,7 +7546,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore { return Ok(job); } if reason == JobInputRetirementReason::Completed { - job.load_executable_request()?; + subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; + job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))? .ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?; if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed @@ -7562,6 +7577,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let mut job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; job.load_executable_request()?; let previous_id = attempt_id(job_id, job.current_attempt); let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)? @@ -7701,6 +7717,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; @@ -7764,6 +7781,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; @@ -7827,6 +7845,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?; + subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| { @@ -7880,6 +7899,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{}`", submission.job_id)))?; let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?; + subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?; if submission.input_digest != job.input_digest || attempt.input_digest != submission.input_digest @@ -7908,6 +7928,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { tx.commit()?; return Ok(BackendJobResultAcceptance { job, attempt, replayed: true }); } + subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; job.load_executable_request()?; if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending { return Err(Error::InvalidInput( @@ -7972,6 +7993,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?; + subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?; let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?; // Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence. if &attempt != expected || job.current_attempt != attempt.attempt @@ -8277,6 +8299,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result<(BackendJobDeliveryRecord, bool)> { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let deleted:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_jobs WHERE workspace_id=?1 AND job_id=?2)",params![workspace_id,job_id],|r|r.get(0))?; + if deleted {return Err(Error::RepositoryConflict("subject_deleted: Job delivery fenced".into()));} if let Some(existing) = read_backend_job_delivery(&tx, workspace_id, delivery_id)? { if existing.job_id != job_id || existing.target != *target { return Err(Error::InvalidInput( @@ -15545,6 +15569,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { notification_routing::jobs::install(conn)?; notification_routing::install_business_events(conn)?; notification_routing::upgrade_result_reports(conn)?; + subject_deletion_store::install_notification_erasure(conn)?; ticket::verify_sqlite_ticket_schema(conn).map_err(|error| { Error::Store(format!( "fresh Workspace Ticket schema verification failed: {error}" @@ -15583,10 +15608,17 @@ fn create_workspace_schema_structure_from_sql( CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?; conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?; - conn.execute_batch(&schema_sql.replace( - "-- Ticket-owned assignment schema is composed by store.rs.", - ticket::TICKET_ASSIGNMENT_SCHEMA_SQL, - ))?; + conn.execute_batch( + &schema_sql + .replace( + "-- Ticket-owned assignment schema is composed by store.rs.", + ticket::TICKET_ASSIGNMENT_SCHEMA_SQL, + ) + .replace( + "-- Subject deletion schema is composed by store.rs.", + include_str!("subject_deletion_schema.sql"), + ), + )?; let composed_sql = r#" ALTER TABLE typed_tickets RENAME TO typed_tickets_legacy_v66; CREATE TABLE typed_tickets ( @@ -15809,6 +15841,9 @@ fn verify_workspace_resource_constraints(conn: &Connection) -> Result<()> { if current_schema_version(conn)? >= 96 { notification_routing::jobs::verify(conn)?; } + if current_schema_version(conn)? >= 105 { + subject_deletion_store::verify(conn)?; + } Ok(()) } @@ -16223,6 +16258,8 @@ fn current_worker_singleton_owner( CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \ AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \ AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \ + AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r JOIN subject_deletion_workers d ON d.workspace_id=r.workspace_id AND d.runtime_id=r.runtime_id AND d.worker_id=r.worker_id WHERE r.workspace_id=owner.workspace_id AND r.runtime_id=owner.runtime_id AND r.worker_id=owner.worker_id AND r.state='succeeded' AND owner.singleton_key='subjektiv:'||d.subject_id AND (d.prior_removal_operation_id=r.operation_id OR (d.prior_removal_operation_id IS NULL AND d.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.metadata_disposition='tombstone'))) \ + AND NOT EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=owner.workspace_id AND d.runtime_id=owner.runtime_id AND d.worker_id=owner.worker_id AND owner.singleton_key='subjektiv:'||d.subject_id AND d.originals_absent=1) \ THEN 'created' ELSE reservation.state END \ FROM worker_singleton_owners owner \ LEFT JOIN worker_create_reservations reservation \ @@ -19265,6 +19302,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()}, WorkspaceSchemaMigrationStep {version:103, name:"Passkey metadata and atomic addition results".into()}, WorkspaceSchemaMigrationStep {version:104, name:"Converge Passkey results and Job input retention".into()}, + WorkspaceSchemaMigrationStep {version:105, name:"Durable Subject deletion snapshots and Job admission fences".into()}, ] ); @@ -19379,6 +19417,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx (102, "Durable failed Subject creation recovery".to_string()), (103, "Passkey metadata and atomic addition results".to_string()), (104, "Converge Passkey results and Job input retention".to_string()), +(105, "Durable Subject deletion snapshots and Job admission fences".to_string()), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -22664,7 +22703,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx (101, "Durable explicit Subject body release".into()), (102, "Durable failed Subject creation recovery".into()), (103, "Passkey metadata and atomic addition results".into()), - (104, "Converge Passkey results and Job input retention".into())]); + (104, "Converge Passkey results and Job input retention".into()), + (105, "Durable Subject deletion snapshots and Job admission fences".into())]); for (table, rows) in unchanged.iter().zip(before.iter()) { assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}"); } @@ -26857,12 +26897,19 @@ INSERT INTO worker_registry ( .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) .unwrap(); store.with_conn(|conn| { + // This fixture deliberately reconstructs schema 102 from fresh. + // Remove the newer Subject fence composition, not just its receipt. + let triggers=conn.prepare("SELECT name,tbl_name FROM sqlite_schema WHERE type='trigger' AND tbl_name LIKE 'notification_%' AND name NOT LIKE 'subject_deleted_%' AND sql LIKE '%subject_deletion_jobs%'")? + .query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::>>()?; + for (name,table) in triggers {conn.execute_batch(&format!("DROP TRIGGER {name}; CREATE TRIGGER {name} BEFORE DELETE ON {table} BEGIN SELECT RAISE(ABORT,'Notification evidence is immutable'); END;"))?;} + for name in ["subject_deleted_job_insert","subject_deleted_job_result","subject_deleted_attempt_insert","subject_deleted_attempt_result","subject_deleted_worker_reservation","subject_deleted_notification_source","subject_deleted_notification_payload","subject_deleted_notification_send","subject_deletion_worker_prior_insert","subject_deletion_worker_identity_immutable"] {conn.execute_batch(&format!("DROP TRIGGER {name}"))?;} + conn.execute_batch("DROP TABLE subject_deletion_allocations; DROP TABLE subject_deletion_jobs; DROP TABLE subject_deletion_workers; DROP TABLE subject_deletions;")?; conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable; ALTER TABLE backend_jobs DROP COLUMN input_disposition; DROP TRIGGER passkey_metadata_insert; DROP TABLE passkey_addition_receipts; DROP TABLE passkey_metadata; - UPDATE __yoi_schema_migrations SET version=102 WHERE version=104; + UPDATE __yoi_schema_migrations SET version=102 WHERE version=105; CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(()) }).unwrap(); drop(store); diff --git a/crates/workspace-server/src/subject_deletion_schema.sql b/crates/workspace-server/src/subject_deletion_schema.sql new file mode 100644 index 000000000..610f29253 --- /dev/null +++ b/crates/workspace-server/src/subject_deletion_schema.sql @@ -0,0 +1,73 @@ +-- Content-free, monotonic main-DB fence. The Host installs the domain fence first +-- and holds its Subject gate across snapshot/admission. No Subject contents live here. +CREATE TABLE subject_deletions ( + workspace_id TEXT NOT NULL, + subject_id TEXT NOT NULL, + operation_id TEXT NOT NULL CHECK(length(operation_id)>0), + completed INTEGER NOT NULL DEFAULT 0 CHECK(completed IN (0,1)), + jobs_erased INTEGER NOT NULL DEFAULT 0 CHECK(jobs_erased IN (0,1)), + PRIMARY KEY(workspace_id,subject_id), + UNIQUE(workspace_id,operation_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE +); +CREATE TABLE subject_deletion_workers ( + workspace_id TEXT NOT NULL, + subject_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + worker_id TEXT NOT NULL, + is_job INTEGER NOT NULL CHECK(is_job IN (0,1)), + -- Exact canonical success already present at prepare, never a later receipt. + prior_removal_operation_id TEXT REFERENCES worker_removal_operations(operation_id), + -- Confirmed by the scoped Runtime storage owner, not catalog absence or Stop ACK. + originals_absent INTEGER NOT NULL DEFAULT 0 CHECK(originals_absent IN (0,1)), + PRIMARY KEY(workspace_id,subject_id,runtime_id,worker_id), + FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE +); +CREATE TRIGGER subject_deletion_worker_prior_insert BEFORE INSERT ON subject_deletion_workers +WHEN NEW.prior_removal_operation_id IS NOT NULL AND NOT EXISTS( + SELECT 1 FROM worker_removal_operations r WHERE r.operation_id=NEW.prior_removal_operation_id + AND r.workspace_id=NEW.workspace_id AND r.runtime_id=NEW.runtime_id + AND r.worker_id=NEW.worker_id AND r.state='succeeded') +BEGIN SELECT RAISE(ABORT,'Subject deletion prior removal must be an exact canonical success'); END; +CREATE TRIGGER subject_deletion_worker_identity_immutable BEFORE UPDATE OF + workspace_id,subject_id,runtime_id,worker_id,prior_removal_operation_id ON subject_deletion_workers +BEGIN SELECT RAISE(ABORT,'Subject deletion Worker snapshot identity is immutable'); END; +CREATE TRIGGER subject_deletion_original_absence_monotonic BEFORE UPDATE OF originals_absent ON subject_deletion_workers +WHEN OLD.originals_absent=1 AND NEW.originals_absent=0 +BEGIN SELECT RAISE(ABORT,'Subject deletion original absence proof is permanent'); END; +-- Job/attempt identities remain after payload erasure, preventing stale allocation +-- or caller-selected Job-id reuse from recreating the deleted Subject's inputs. +CREATE TABLE subject_deletion_jobs ( + workspace_id TEXT NOT NULL, + subject_id TEXT NOT NULL, + job_id TEXT NOT NULL, + PRIMARY KEY(workspace_id,job_id), + FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE +); +CREATE TABLE subject_deletion_allocations ( + workspace_id TEXT NOT NULL, + subject_id TEXT NOT NULL, + allocation_key TEXT NOT NULL, + PRIMARY KEY(workspace_id,allocation_key), + FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE +); +CREATE TRIGGER subject_deleted_job_insert BEFORE INSERT ON backend_jobs +WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND + (d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_journal.subject_id') OR + d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_consolidation.subject_id'))) + OR EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT,'subject_deleted: Job admission fenced'); END; +CREATE TRIGGER subject_deleted_job_result BEFORE UPDATE OF state,result_json ON backend_jobs +WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT,'subject_deleted: Job result fenced'); END; +CREATE TRIGGER subject_deleted_attempt_insert BEFORE INSERT ON backend_job_attempts +WHEN EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT,'subject_deleted: Job retry fenced'); END; +CREATE TRIGGER subject_deleted_attempt_result BEFORE UPDATE OF state,result_json,runtime_id,worker_id,runtime_run_id ON backend_job_attempts +WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT,'subject_deleted: Job attempt fenced'); END; +CREATE TRIGGER subject_deleted_worker_reservation BEFORE INSERT ON worker_create_reservations +WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND + NEW.singleton_key IN ('subjektiv:'||d.subject_id,'subjektiv-consolidation:'||d.subject_id)) + OR EXISTS(SELECT 1 FROM subject_deletion_allocations d WHERE d.workspace_id=NEW.workspace_id AND d.allocation_key=NEW.allocation_key) +BEGIN SELECT RAISE(ABORT,'subject_deleted: Worker allocation fenced'); END; diff --git a/crates/workspace-server/src/subject_deletion_store.rs b/crates/workspace-server/src/subject_deletion_store.rs new file mode 100644 index 000000000..61db2bfb2 --- /dev/null +++ b/crates/workspace-server/src/subject_deletion_store.rs @@ -0,0 +1,1178 @@ +//! Main-DB half of Subject deletion. Call prepare only after the domain's +//! content-free fence, while holding the Host Subject gate. Reuse the operation +//! after crashes; another operation is a conflict, never a fresh deletion. +//! +//! All historical Job attempts and dispatch allocations are snapshotted, not just +//! the current attempt. Pending/unknown work becomes failed/subject_deleted (not a +//! successful Job). Originals in body/delegated Sessions are archived forever; +//! Existing canonical Worker removals and archives are outside this deletion: +//! prepare pins their exact successful receipt, regardless of its disposition. +//! Newly removed Job Sessions/diagnostics need canonical Purge before input, +//! result, notification payloads and live Job/cache references can be erased. +//! This DB has no separate Job content cache. Runtime caches are the canonical +//! Worker removal's responsibility. Notification text already delivered into an +//! ordinary conversation follows that conversation's archive lifecycle. +use super::*; + +const MIGRATION_NAME: &str = "Durable Subject deletion snapshots and Job admission fences"; + +pub(super) fn migrate(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 104 { + return Err(Error::Store( + "Subject deletion migration requires schema 104".into(), + )); + } + let tx = conn.unchecked_transaction()?; + tx.execute_batch(include_str!("subject_deletion_schema.sql"))?; + install_notification_erasure(&tx)?; + tx.execute( + "INSERT INTO __yoi_schema_migrations(version,name) VALUES(105,?1)", + [MIGRATION_NAME], + )?; + verify(&tx)?; + tx.commit()?; + Ok(()) +} + +// Immutable notification evidence normally forbids deletion. The only exception +// is a saved Subject deletion snapshot; erasure below still requires physical +// Job Session Purge receipts. Updates remain forbidden, including payload edits. +pub(super) fn install_notification_erasure(conn: &Connection) -> Result<()> { + for (trigger, table, predicate) in [ + ( + "notification_sources_no_delete", + "notification_sources", + "s.sequence=OLD.sequence", + ), + ( + "notification_decisions_no_delete", + "notification_decisions", + "s.sequence=OLD.source_sequence", + ), + ( + "notification_requests_no_delete", + "notification_requests", + "s.workspace_id=OLD.workspace_id AND s.input_event_id=OLD.delivery_id", + ), + ( + "notification_acknowledgements_no_delete", + "notification_acknowledgements", + "s.workspace_id=OLD.workspace_id AND s.input_event_id=OLD.delivery_id", + ), + ( + "notification_failures_no_delete", + "notification_failures", + "s.workspace_id=OLD.workspace_id AND s.input_event_id=OLD.delivery_id", + ), + ( + "notification_job_targets_no_delete", + "notification_job_targets", + "s.workspace_id=OLD.workspace_id AND s.input_event_id=OLD.delivery_id", + ), + ( + "notification_job_legacy_no_delete", + "notification_job_legacy_receipts", + "s.workspace_id=OLD.workspace_id AND s.input_event_id=OLD.delivery_id", + ), + ( + "notification_source_preparation_failures_no_delete", + "notification_source_preparation_failures", + "s.sequence=OLD.source_sequence", + ), + ] { + conn.execute_batch(&format!("DROP TRIGGER {trigger}; + CREATE TRIGGER {trigger} BEFORE DELETE ON {table} + WHEN NOT EXISTS(SELECT 1 FROM notification_sources s JOIN subject_deletion_jobs j + ON j.workspace_id=s.workspace_id AND j.job_id=json_extract(s.facts_json,'$.backend_job.job_id') + WHERE s.role='backend_job' AND {predicate}) + BEGIN SELECT RAISE(ABORT,'Notification evidence is immutable outside Subject deletion'); END;"))?; + } + conn.execute_batch("CREATE TRIGGER subject_deleted_notification_source BEFORE INSERT ON notification_sources + WHEN NEW.role='backend_job' AND EXISTS(SELECT 1 FROM subject_deletion_jobs j WHERE j.workspace_id=NEW.workspace_id AND j.job_id=json_extract(NEW.facts_json,'$.backend_job.job_id')) + BEGIN SELECT RAISE(ABORT,'subject_deleted: Job notification source fenced'); END; + CREATE TRIGGER subject_deleted_notification_payload BEFORE INSERT ON notification_requests + WHEN EXISTS(SELECT 1 FROM notification_sources s JOIN subject_deletion_jobs j ON j.workspace_id=s.workspace_id AND j.job_id=json_extract(s.facts_json,'$.backend_job.job_id') WHERE s.role='backend_job' AND s.workspace_id=NEW.workspace_id AND s.input_event_id=NEW.delivery_id) + BEGIN SELECT RAISE(ABORT,'subject_deleted: Job notification payload fenced'); END; + CREATE TRIGGER subject_deleted_notification_send BEFORE INSERT ON notification_job_targets + WHEN EXISTS(SELECT 1 FROM notification_sources s JOIN subject_deletion_jobs j ON j.workspace_id=s.workspace_id AND j.job_id=json_extract(s.facts_json,'$.backend_job.job_id') WHERE s.role='backend_job' AND s.workspace_id=NEW.workspace_id AND s.input_event_id=NEW.delivery_id) + BEGIN SELECT RAISE(ABORT,'subject_deleted: Job notification send fenced'); END;")?; + Ok(()) +} + +pub(super) fn verify(conn: &Connection) -> Result<()> { + for name in [ + "subject_deletions", + "subject_deletion_workers", + "subject_deletion_jobs", + "subject_deletion_allocations", + "subject_deleted_job_insert", + "subject_deleted_job_result", + "subject_deleted_attempt_insert", + "subject_deleted_attempt_result", + "subject_deleted_worker_reservation", + "subject_deletion_worker_prior_insert", + "subject_deletion_worker_identity_immutable", + "subject_deletion_original_absence_monotonic", + "subject_deleted_notification_source", + "subject_deleted_notification_payload", + "subject_deleted_notification_send", + ] { + let present: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=?1)", + [name], + |r| r.get(0), + )?; + if !present { + return Err(Error::Store(format!( + "Subject deletion schema object {name} is missing" + ))); + } + } + if !column_exists( + conn, + "subject_deletion_workers", + "prior_removal_operation_id", + )? { + return Err(Error::Store( + "Subject deletion prior removal snapshot column is missing".into(), + )); + } + Ok(()) +} + +pub(super) fn require_worker_admission( + conn: &Connection, + workspace: &str, + allocation: &str, + singleton: Option<&str>, +) -> Result<()> { + let fenced: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND ?3 IN ('subjektiv:'||subject_id,'subjektiv-consolidation:'||subject_id)) OR EXISTS(SELECT 1 FROM subject_deletion_allocations WHERE workspace_id=?1 AND allocation_key=?2)",params![workspace,allocation,singleton],|r|r.get(0))?; + if fenced { + return Err(Error::RepositoryConflict( + "subject_deleted: Worker allocation fenced".into(), + )); + } + Ok(()) +} + +fn require_operation( + conn: &Connection, + workspace: &str, + subject: &str, + operation: &str, +) -> Result<()> { + let matches: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND subject_id=?2 AND operation_id=?3)", params![workspace,subject,operation], |r|r.get(0))?; + if !matches { + return Err(Error::RepositoryConflict( + "Subject deletion operation does not match saved fence".into(), + )); + } + Ok(()) +} + +pub(crate) fn worker_kind( + conn: &Connection, + workspace: &str, + worker: &RuntimeWorkerRef, +) -> Result> { + Ok(conn.query_row("SELECT max(is_job) FROM subject_deletion_workers WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3", params![workspace,worker.runtime_id,worker.worker_id], |r|r.get(0))?) +} + +pub(crate) fn prior_removal_matches( + conn: &Connection, + workspace: &str, + worker: &RuntimeWorkerRef, + operation: &str, +) -> Result { + Ok(conn.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_workers w JOIN worker_removal_operations r ON r.operation_id=w.prior_removal_operation_id AND r.workspace_id=w.workspace_id AND r.runtime_id=w.runtime_id AND r.worker_id=w.worker_id WHERE w.workspace_id=?1 AND w.runtime_id=?2 AND w.worker_id=?3 AND r.operation_id=?4 AND r.state='succeeded')",params![workspace,worker.runtime_id,worker.worker_id,operation],|r|r.get(0))?) +} + +/// Even an idempotent completed-result replay must pass this fence. Otherwise a +/// deleted Subject could reaccept old result bodies before the physical purge. +pub(super) fn require_job_admission( + conn: &Connection, + workspace: &str, + request: &BackendJobRequest, +) -> Result<()> { + let journal = request + .grants + .subjektiv_journal + .as_ref() + .map(|g| g.subject_id.as_str()); + let consolidation = request + .grants + .subjektiv_consolidation + .as_ref() + .map(|g| g.subject_id.as_str()); + let fenced: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND subject_id IN (?2,?3)) OR EXISTS(SELECT 1 FROM subject_deletion_jobs WHERE workspace_id=?1 AND job_id=?4)", params![workspace,journal,consolidation,request.job_id], |r|r.get(0))?; + if fenced { + return Err(Error::RepositoryConflict( + "subject_deleted: Backend Job admission/result fenced".into(), + )); + } + Ok(()) +} + +fn worker_fence_request( + conn: &Connection, + workspace: &str, + subject: &str, + operation: &str, + worker: &RuntimeWorkerRef, +) -> Result { + require_operation(conn, workspace, subject, operation)?; + let fingerprint: Option = conn.query_row( + "SELECT r.create_fingerprint FROM worker_create_reservations r JOIN subject_deletion_workers d USING(workspace_id,runtime_id,worker_id) WHERE d.workspace_id=?1 AND d.subject_id=?2 AND d.runtime_id=?3 AND d.worker_id=?4", + params![workspace, subject, worker.runtime_id, worker.worker_id], |row| row.get(0), + ).optional()?; + let digest = sha2::Sha256::digest( + serde_json::to_vec(&(subject, operation, worker)) + .map_err(|e| Error::Store(e.to_string()))?, + ); + let child_key = format!( + "subject-delete-worker:{}", + digest + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ); + Ok(runtime_api::SubjectDeleteWorkerRequest { + workspace_id: workspace.into(), + worker_id: worker_runtime::identity::WorkerId::parse(&worker.worker_id) + .ok_or_else(|| Error::InvalidInput("invalid Worker ID".into()))?, + create_fingerprint: fingerprint.ok_or_else(|| { + Error::RepositoryConflict( + "Subject Worker has no immutable creation identity in the deletion inventory" + .into(), + ) + })?, + operation_id: child_key, + }) +} + +impl SqliteWorkspaceStore { + pub(crate) fn subject_deletion_worker_request( + &self, + workspace: &str, + subject: &str, + operation: &str, + worker: &RuntimeWorkerRef, + ) -> Result { + self.with_conn(|conn| worker_fence_request(conn, workspace, subject, operation, worker)) + } + + /// Snapshot once, under the Host gate, after the domain deletion fence. + /// Delegated identities must come from domain grants, never request JSON. + pub fn prepare_subject_deletion( + &self, + workspace: &str, + subject: &str, + operation: &str, + delegated: &[RuntimeWorkerRef], + ) -> Result<()> { + if [workspace, subject, operation] + .iter() + .any(|s| s.is_empty() || s.trim() != *s || s.chars().any(char::is_control)) + { + return Err(Error::InvalidInput( + "Subject deletion identities must be non-empty and trimmed".into(), + )); + } + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let saved: Option = tx.query_row("SELECT operation_id FROM subject_deletions WHERE workspace_id=?1 AND subject_id=?2",params![workspace,subject],|r|r.get(0)).optional()?; + if let Some(saved) = saved { + if saved != operation { return Err(Error::RepositoryConflict("Subject deletion operation changed".into())); } + tx.commit()?; + return Ok(()); + } + let reused:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND operation_id=?2)",params![workspace,operation],|r|r.get(0))?; + if reused {return Err(Error::RepositoryConflict("Subject deletion operation reused for another Subject".into()));} + tx.execute("INSERT INTO subject_deletions(workspace_id,subject_id,operation_id) VALUES(?1,?2,?3)",params![workspace,subject,operation])?; + tx.execute("INSERT INTO subject_deletion_jobs(workspace_id,subject_id,job_id) + SELECT workspace_id,?2,job_id FROM backend_jobs WHERE workspace_id=?1 AND + (json_extract(request_json,'$.grants.subjektiv_journal.subject_id')=?2 OR + json_extract(request_json,'$.grants.subjektiv_consolidation.subject_id')=?2)",params![workspace,subject])?; + let mut workers = tx.prepare("SELECT runtime_id,worker_id,singleton_key='subjektiv-consolidation:'||?2 FROM worker_singleton_owners WHERE workspace_id=?1 AND singleton_key IN ('subjektiv:'||?2,'subjektiv-consolidation:'||?2) + UNION SELECT runtime_id,worker_id,singleton_key='subjektiv-consolidation:'||?2 FROM worker_create_reservations WHERE workspace_id=?1 AND singleton_key IN ('subjektiv:'||?2,'subjektiv-consolidation:'||?2) AND state!='removed'")? + .query_map(params![workspace,subject],|r|Ok((RuntimeWorkerRef::new(r.get::<_,String>(0)?,r.get::<_,String>(1)?),r.get::<_,bool>(2)?)))? + .collect::>>()?; + workers.extend(delegated.iter().cloned().map(|w|(w,false))); + let attempts = tx.prepare("SELECT a.attempt_id,a.runtime_id,a.worker_id FROM backend_job_attempts a JOIN subject_deletion_jobs j USING(workspace_id,job_id) WHERE j.workspace_id=?1 AND j.subject_id=?2")? + .query_map(params![workspace,subject],|r|Ok((r.get::<_,String>(0)?,r.get::<_,Option>(1)?,r.get::<_,Option>(2)?)))? + .collect::>>()?; + for (attempt, runtime, worker) in attempts { + if let (Some(runtime),Some(worker)) = (runtime,worker) { workers.push((RuntimeWorkerRef::new(runtime,worker),true)); } + let allocation = crate::backend_job::allocation_key(&attempt); + tx.execute("INSERT INTO subject_deletion_allocations(workspace_id,subject_id,allocation_key) VALUES(?1,?2,?3)",params![workspace,subject,allocation])?; + if let Some(worker) = tx.query_row("SELECT runtime_id,worker_id FROM worker_create_reservations WHERE workspace_id=?1 AND allocation_key=?2",params![workspace,allocation],|r|Ok(RuntimeWorkerRef::new(r.get::<_,String>(0)?,r.get::<_,String>(1)?))).optional()? { + workers.push((worker,true)); + } + } + for (worker,is_job) in workers { + let other: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM worker_registry WHERE runtime_id=?1 AND worker_id=?2 AND workspace_id!=?3 UNION ALL SELECT 1 FROM worker_create_reservations WHERE runtime_id=?1 AND worker_id=?2 AND workspace_id!=?3)",params![worker.runtime_id,worker.worker_id,workspace],|r|r.get(0))?; + if other { return Err(Error::RepositoryConflict("Subject deletion Worker belongs to another Workspace".into())); } + let prior:Option=tx.query_row("SELECT operation_id FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='succeeded' ORDER BY created_at DESC,operation_id DESC LIMIT 1",params![workspace,worker.runtime_id,worker.worker_id],|r|r.get(0)).optional()?; + tx.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id) VALUES(?1,?2,?3,?4,?5,?6) + ON CONFLICT(workspace_id,subject_id,runtime_id,worker_id) DO UPDATE SET is_job=max(is_job,excluded.is_job)",params![workspace,subject,worker.runtime_id,worker.worker_id,is_job,prior])?; + } + let now = chrono::Utc::now().to_rfc3339(); + tx.execute("UPDATE backend_job_attempts SET state='failed',failure_category='subject_deleted',failure_detail=NULL,completed_at=?3,updated_at=?3, + worker_cleanup_state=CASE WHEN worker_id IS NULL THEN 'completed' ELSE 'pending' END, + worker_cleanup_updated_at=?3,worker_cleanup_completed_at=CASE WHEN worker_id IS NULL THEN ?3 ELSE NULL END + WHERE workspace_id=?1 AND job_id IN (SELECT job_id FROM subject_deletion_jobs WHERE workspace_id=?1 AND subject_id=?2) AND state IN ('reserved','dispatching','dispatched','unknown')",params![workspace,subject,now])?; + tx.execute("UPDATE backend_jobs SET state='failed',failure_category='subject_deleted',failure_detail=NULL,completed_at=?3,updated_at=?3 + WHERE workspace_id=?1 AND job_id IN (SELECT job_id FROM subject_deletion_jobs WHERE workspace_id=?1 AND subject_id=?2) AND state IN ('pending','unknown')",params![workspace,subject,now])?; + tx.commit()?; + Ok(()) + }) + } + + pub fn list_subject_deletion_workers( + &self, + workspace: &str, + subject: &str, + operation: &str, + ) -> Result> { + self.with_conn(|conn| { + require_operation(conn,workspace,subject,operation)?; + Ok(conn.prepare("SELECT runtime_id,worker_id,is_job FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 ORDER BY runtime_id,worker_id")? + .query_map(params![workspace,subject],|r|Ok((RuntimeWorkerRef::new(r.get::<_,String>(0)?,r.get::<_,String>(1)?),r.get(2)?)))? + .collect::>>()?) + }) + } + + /// Stop/catalog absence is not erasure. An exact scoped Runtime receipt may + /// prove no live Worker/Session/diagnostic aggregate exists under its permanent + /// creation fence; retain that proof and release the reservation atomically. Otherwise require + /// a canonical original-retention receipt; never manufacture a Purge receipt. + pub(crate) fn confirm_subject_deletion_absence( + &self, + workspace: &str, + subject: &str, + operation: &str, + worker: &RuntimeWorkerRef, + fence: &runtime_api::SubjectDeleteWorkerResponse, + ) -> Result<()> { + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let confirm = |conn: &Connection| -> Result<()> { + let request = worker_fence_request(conn, workspace, subject, operation, worker)?; + if fence.workspace_id != request.workspace_id || fence.worker_id != request.worker_id + || fence.create_fingerprint != request.create_fingerprint || fence.operation_id != request.operation_id { + return Err(Error::RepositoryConflict("Subject deletion absence fence identity mismatch".into())); + } + if fence.originals_absent { + let registered: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3)", params![workspace,worker.runtime_id,worker.worker_id], |r|r.get(0))?; + if registered { return Err(Error::RepositoryConflict("Registered Worker requires canonical original retention".into())); } + conn.execute("UPDATE subject_deletion_workers SET originals_absent=1 WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4", params![workspace,subject,worker.runtime_id,worker.worker_id])?; + conn.execute("UPDATE worker_create_reservations SET state='removed' WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND create_fingerprint=?4", params![workspace,worker.runtime_id,worker.worker_id,request.create_fingerprint])?; + return Ok(()); + } + let is_job: Option = conn.query_row("SELECT is_job FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4",params![workspace,subject,worker.runtime_id,worker.worker_id],|r|r.get(0)).optional()?; + let Some(is_job)=is_job else { return Err(Error::RepositoryConflict("Worker is not in the Subject deletion snapshot".into())); }; + let prior:Option=conn.query_row("SELECT prior_removal_operation_id FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4",params![workspace,subject,worker.runtime_id,worker.worker_id],|r|r.get(0))?; + if let Some(prior)=prior { + if prior_removal_matches(conn,workspace,worker,&prior)? {return Ok(());} + return Err(Error::RepositoryConflict("Saved prior Worker removal receipt no longer matches canonical success".into())); + } + let proof: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='succeeded' AND ((?4=1 AND session_disposition='purge' AND diagnostics_disposition='purge') OR (?4=0 AND session_disposition='archive' AND archive_retention_kind='forever'))) + OR (?4=0 AND EXISTS(SELECT 1 FROM subject_create_recoveries WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND create_fingerprint=?5 AND completed=1))",params![workspace,worker.runtime_id,worker.worker_id,is_job,fence.create_fingerprint],|r|r.get(0))?; + if !proof { return Err(Error::RepositoryConflict("Runtime exclusion/UnknownWorker does not prove original absence; canonical create recovery or Worker removal receipt is required".into())); } + Ok(()) + }; + confirm(&tx)?; + tx.commit()?; + Ok(()) + }) + } + + /// Never infer physical erasure from absence, failed/unknown execution, or a + /// cleanup-state flag. Newly removed Job Workers require canonical Purge; + /// only exact prior successful removals retain their pre-deletion disposition. + pub fn erase_subject_deleted_jobs( + &self, + workspace: &str, + subject: &str, + operation: &str, + ) -> Result<()> { + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + require_operation(&tx,workspace,subject,operation)?; + require_worker_receipts(&tx,workspace,subject,true)?; + // Remove child evidence before parents; deletion exceptions remain + // scoped to the content-free saved Job identities throughout. + let source = "SELECT s.sequence FROM notification_sources s JOIN subject_deletion_jobs j ON j.workspace_id=s.workspace_id AND j.job_id=json_extract(s.facts_json,'$.backend_job.job_id') WHERE s.role='backend_job' AND j.workspace_id=?1 AND j.subject_id=?2"; + let delivery = format!("SELECT input_event_id FROM notification_sources WHERE sequence IN ({source})"); + for table in ["notification_acknowledgements","notification_job_targets","notification_job_legacy_receipts","notification_failures","notification_requests"] { + tx.execute(&format!("DELETE FROM {table} WHERE workspace_id=?1 AND delivery_id IN ({delivery})"),params![workspace,subject])?; + } + for table in ["notification_source_preparation_failures","notification_source_offers","notification_decisions"] { + tx.execute(&format!("DELETE FROM {table} WHERE source_sequence IN ({source})"),params![workspace,subject])?; + } + tx.execute(&format!("DELETE FROM notification_sources WHERE sequence IN ({source})"),params![workspace,subject])?; + tx.execute("DELETE FROM backend_jobs WHERE workspace_id=?1 AND job_id IN (SELECT job_id FROM subject_deletion_jobs WHERE workspace_id=?1 AND subject_id=?2)",params![workspace,subject])?; + tx.execute("UPDATE subject_deletions SET jobs_erased=1 WHERE workspace_id=?1 AND subject_id=?2",params![workspace,subject])?; + tx.commit()?; + Ok(()) + }) + } + + /// Confirm all Backend cleanup before the caller commits authoritative domain + /// completion. This main-DB receipt is not domain completion; a crash after + /// this commit resumes the same operation and still finishes the domain. + /// Retains content-free identities for recovery and late admission rejection. + pub fn finish_subject_deletion_record( + &self, + workspace: &str, + subject: &str, + operation: &str, + ) -> Result<()> { + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + require_operation(&tx, workspace, subject, operation)?; + require_worker_receipts(&tx, workspace, subject, false)?; + let erased: bool = tx.query_row( + "SELECT jobs_erased FROM subject_deletions WHERE workspace_id=?1 AND subject_id=?2", + params![workspace, subject], + |r| r.get(0), + )?; + if !erased { + return Err(Error::RepositoryConflict( + "Subject deletion Job erasure is incomplete".into(), + )); + } + // Canonical removal sets reservation state=removed, but Subject + // occupancy requires an exact prior success, new canonical removal, + // or explicit release/create recovery proof. + for key in [ + format!("subjektiv:{subject}"), + format!("subjektiv-consolidation:{subject}"), + ] { + if let Some((_, state)) = current_worker_singleton_owner(&tx, workspace, &key)? + && state.as_deref() != Some("removed") + { + return Err(Error::RepositoryConflict( + "Subject singleton occupancy requires canonical removal/create recovery" + .into(), + )); + } + } + tx.execute( + "UPDATE subject_deletions SET completed=1 WHERE workspace_id=?1 AND subject_id=?2", + params![workspace, subject], + )?; + tx.commit()?; + Ok(()) + }) + } +} + +fn require_worker_receipts( + conn: &Connection, + workspace: &str, + subject: &str, + jobs_only: bool, +) -> Result<()> { + let missing: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_workers w WHERE w.workspace_id=?1 AND w.subject_id=?2 AND (?3=0 OR w.is_job=1) AND w.originals_absent=0 + AND NOT ( + (w.prior_removal_operation_id IS NOT NULL AND EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.operation_id=w.prior_removal_operation_id AND r.workspace_id=w.workspace_id AND r.runtime_id=w.runtime_id AND r.worker_id=w.worker_id AND r.state='succeeded')) + OR (w.prior_removal_operation_id IS NULL AND ( + EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.workspace_id=w.workspace_id AND r.runtime_id=w.runtime_id AND r.worker_id=w.worker_id AND r.state='succeeded' + AND r.metadata_disposition='tombstone' AND ((w.is_job=1 AND r.session_disposition='purge' AND r.diagnostics_disposition='purge') OR + (w.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.diagnostics_disposition='retain'))) + OR (w.is_job=0 AND EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=w.workspace_id AND r.runtime_id=w.runtime_id AND r.worker_id=w.worker_id AND r.completed=1)) + )) + ))",params![workspace,subject,jobs_only],|r|r.get(0))?; + if missing { + return Err(Error::RepositoryConflict("Subject deletion requires canonical Worker removal/create recovery receipts; unknown or stale outcomes cannot complete deletion".into())); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backend_job::{BackendJobGrants, SubjektivConsolidationGrant}; + const NOW: &str = "2026-10-28T00:00:00Z"; + + fn seed(store: &SqliteWorkspaceStore) { + store.with_conn(|c| { + c.execute("INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) VALUES('owner','user','owner','Owner',?1,?1)",[NOW])?; + for w in ["w","other"] { + c.execute("INSERT INTO workspaces(workspace_id,display_name,state,created_at,updated_at,owner_account_id) VALUES(?1,?1,'active',?2,?2,'owner')",params![w,NOW])?; + } + Ok(()) + }).unwrap(); + } + fn setup() -> SqliteWorkspaceStore { + let store = SqliteWorkspaceStore::in_memory_fixture().unwrap(); + seed(&store); + store + } + fn request(id: &str, subject: &str) -> BackendJobRequest { + BackendJobRequest { + job_id: id.into(), + purpose: "subject_deletion_test".into(), + input_ref: format!("subject:{subject}"), + input: serde_json::json!({"private":"Subject source"}), + instruction: "Read the private input".into(), + profile: "builtin:backend-job".into(), + grants: BackendJobGrants { + subjektiv_consolidation: Some(SubjektivConsolidationGrant { + subject_id: subject.into(), + candidate_ids: vec!["candidate".into()], + }), + ..Default::default() + }, + serialization_key: None, + source_worker: None, + notification_target: None, + limits: Default::default(), + } + } + fn worker(n: u64) -> RuntimeWorkerRef { + RuntimeWorkerRef::new("r", WorkerId::from_legacy_u64(n).to_string()) + } + fn registry(store: &SqliteWorkspaceStore, worker: &RuntimeWorkerRef) { + store.with_conn(|c| { + c.execute("INSERT INTO worker_registry(workspace_id,runtime_id,worker_id,display_name,retention_state,created_at,updated_at) VALUES('w',?1,?2,'worker','normal',?3,?3)",params![worker.runtime_id,worker.worker_id,NOW])?; + Ok(()) + }).unwrap(); + } + fn reservation( + store: &SqliteWorkspaceStore, + worker: &RuntimeWorkerRef, + allocation: &str, + singleton: Option<&str>, + ) { + store.with_conn(|c| { + c.execute("INSERT INTO worker_create_reservations(workspace_id,runtime_id,worker_id,allocation_key,create_fingerprint,state,created_at,updated_at,singleton_key,singleton_generation) VALUES('w',?1,?2,?3,'fp','reserved',?4,?4,?5,?6)",params![worker.runtime_id,worker.worker_id,allocation,NOW,singleton,singleton.map(|_|1)])?; + if let Some(key) = singleton { + c.execute("INSERT INTO worker_singleton_owners(workspace_id,singleton_key,runtime_id,worker_id,generation,created_at,updated_at) VALUES('w',?1,?2,?3,1,?4,?4)",params![key,worker.runtime_id,worker.worker_id,NOW])?; + } + Ok(()) + }).unwrap(); + } + // Isolated store receipt fixture: production only creates this row through + // canonical removal's exact Runtime response checks and atomic catalog commit. + fn receipt( + store: &SqliteWorkspaceStore, + worker: &RuntimeWorkerRef, + state: &str, + session: &str, + diagnostics: &str, + ) { + store.with_conn(|c| { + c.execute("INSERT INTO worker_removal_operations(operation_id,plan_id,input_fingerprint,workspace_id,runtime_id,worker_id,worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition,archive_retention_kind,diagnostics_disposition,blockers_json,state,reason,created_at,updated_at) VALUES(?1,?1,'fp','w',?2,?3,?4,'test','digest',?5,'tombstone','forever',?6,'[]',?7,'test',?4,?4)",params![format!("receipt-{}-{state}",worker.worker_id),worker.runtime_id,worker.worker_id,NOW,session,diagnostics,state])?; + Ok(()) + }).unwrap(); + } + + #[test] + fn snapshot_all_attempts_dispatch_allocations_and_unknowns_is_scoped_and_idempotent() { + let s = setup(); + let req = request("job", "subject"); + let first = s.reserve_backend_job("w", &req, NOW).unwrap(); + let first_worker = worker(1); + registry(&s, &first_worker); + s.claim_backend_job_attempt_dispatch("w", "job", &first.attempt.attempt_id, NOW) + .unwrap(); + s.bind_backend_job_attempt_worker( + "w", + "job", + &first.attempt.attempt_id, + &first_worker, + Some("run"), + NOW, + ) + .unwrap(); + s.finish_backend_job_attempt( + "w", + "job", + &first.attempt.attempt_id, + BackendJobAttemptState::Failed, + "failed", + "failure", + NOW, + ) + .unwrap(); + s.with_conn(|c| {c.execute("UPDATE backend_job_attempts SET worker_cleanup_state='completed' WHERE workspace_id='w'",[])?;Ok(())}).unwrap(); + let second = s.reserve_backend_job_retry("w", "job", NOW).unwrap(); + s.claim_backend_job_attempt_dispatch("w", "job", &second.attempt.attempt_id, NOW) + .unwrap(); + let allocating = worker(2); + reservation( + &s, + &allocating, + &crate::backend_job::allocation_key(&second.attempt.attempt_id), + None, + ); + s.finish_backend_job_attempt( + "w", + "job", + &second.attempt.attempt_id, + BackendJobAttemptState::Unknown, + "unknown", + "unknown outcome", + NOW, + ) + .unwrap(); + let body = worker(3); + reservation(&s, &body, "body", Some("subjektiv:subject")); + let legacy = worker(4); + reservation( + &s, + &legacy, + "legacy", + Some("subjektiv-consolidation:subject"), + ); + let delegate = worker(5); + let other_job = s.reserve_backend_job("other", &req, NOW).unwrap(); + let other_subject = s + .reserve_backend_job("w", &request("unrelated", "another"), NOW) + .unwrap(); + s.prepare_subject_deletion("w", "subject", "delete", std::slice::from_ref(&delegate)) + .unwrap(); + assert_eq!( + s.list_subject_deletion_workers("w", "subject", "delete") + .unwrap(), + vec![ + (first_worker, true), + (allocating, true), + (body, false), + (legacy, true), + (delegate, false) + ] + ); + let saved = s.get_backend_job("w", "job").unwrap().unwrap(); + assert_eq!(saved.state, BackendJobState::Failed); + assert_eq!(saved.failure_category.as_deref(), Some("subject_deleted")); + assert_eq!( + s.get_backend_job("other", "job").unwrap().unwrap(), + other_job.job + ); + assert_eq!( + s.get_backend_job("w", "unrelated").unwrap().unwrap(), + other_subject.job + ); + let second = s + .get_backend_job_attempt("w", "job", &second.attempt.attempt_id) + .unwrap() + .unwrap(); + assert_eq!(second.state, BackendJobAttemptState::Failed); + assert_eq!(second.failure_category.as_deref(), Some("subject_deleted")); + s.prepare_subject_deletion("w", "subject", "delete", &[]) + .unwrap(); + assert_eq!( + s.list_subject_deletion_workers("w", "subject", "delete") + .unwrap() + .len(), + 5 + ); + assert!( + s.prepare_subject_deletion("w", "subject", "different", &[]) + .is_err() + ); + assert!( + s.prepare_subject_deletion("w", "another", "delete", &[]) + .is_err() + ); + assert!( + s.list_subject_deletion_workers("other", "subject", "delete") + .is_err() + ); + assert!( + s.reserve_backend_job_retry("w", "job", NOW) + .unwrap_err() + .to_string() + .contains("subject_deleted") + ); + assert!( + s.reserve_backend_job("w", &request("late", "subject"), NOW) + .is_err() + ); + s.with_conn(|c| { + assert!(require_worker_admission(c, "w", "body", Some("subjektiv:subject")).is_err()); + assert!( + require_worker_admission( + c, + "w", + &crate::backend_job::allocation_key(&second.attempt_id), + None + ) + .is_err() + ); + assert!( + require_worker_admission(c, "other", "body", Some("subjektiv:subject")).is_ok() + ); + Ok(()) + }) + .unwrap(); + assert!( + s.erase_subject_deleted_jobs("w", "subject", "delete") + .is_err() + ); + assert!( + s.finish_subject_deletion_record("w", "subject", "delete") + .is_err() + ); + } + + #[test] + fn completed_result_replay_is_fenced_and_payloads_wait_for_physical_purge() { + let s = setup(); + let target = worker(10); + registry(&s, &target); + let mut req = request("job", "subject"); + req.notification_target = Some(target.clone()); + let r = s.reserve_backend_job("w", &req, NOW).unwrap(); + s.claim_backend_job_attempt_dispatch("w", "job", &r.attempt.attempt_id, NOW) + .unwrap(); + let w = worker(11); + registry(&s, &w); + s.bind_backend_job_attempt_worker("w", "job", &r.attempt.attempt_id, &w, Some("run"), NOW) + .unwrap(); + let submission = BackendJobResultSubmission { + job_id: "job".into(), + attempt_id: r.attempt.attempt_id.clone(), + input_digest: req.input_digest().unwrap(), + result: serde_json::json!({"secret":"result body"}), + }; + s.accept_backend_job_result("w", &w, &submission, NOW) + .unwrap(); + let source = s + .job_notification_source("w", "job", &r.attempt.attempt_id) + .unwrap(); + s.route_job_notification( + "w", + &source, + &JobNotificationFacts::Result("secret notice body".into()), + ) + .unwrap(); + assert!( + s.job_notification_request("w", &source.delivery_id) + .unwrap() + .is_some() + ); + s.prepare_subject_deletion("w", "subject", "delete", &[]) + .unwrap(); + assert!( + s.route_job_notification( + "w", + &source, + &JobNotificationFacts::Result("late secret".into()) + ) + .is_err() + ); + assert!( + s.accept_backend_job_result("w", &w, &submission, NOW) + .unwrap_err() + .to_string() + .contains("subject_deleted") + ); + assert!( + s.erase_subject_deleted_jobs("w", "subject", "delete") + .is_err() + ); + assert!( + s.get_backend_job("w", "job") + .unwrap() + .unwrap() + .result + .is_some() + ); + receipt(&s, &w, "failed", "purge", "purge"); + assert!( + s.erase_subject_deleted_jobs("w", "subject", "delete") + .is_err() + ); + receipt(&s, &w, "succeeded", "archive", "retain"); + assert!( + s.erase_subject_deleted_jobs("w", "subject", "delete") + .is_err() + ); + s.with_conn(|c| {c.execute("UPDATE worker_removal_operations SET session_disposition='purge',diagnostics_disposition='purge' WHERE state='succeeded'",[])?;Ok(())}).unwrap(); + s.erase_subject_deleted_jobs("w", "subject", "delete") + .unwrap(); + assert!(s.get_backend_job("w", "job").unwrap().is_none()); + assert!( + s.get_backend_job_attempt("w", "job", &r.attempt.attempt_id) + .unwrap() + .is_none() + ); + s.with_conn(|c| { + for table in [ + "notification_sources", + "notification_decisions", + "notification_requests", + "notification_job_targets", + ] { + assert_eq!( + c.query_row( + &format!("SELECT count(*) FROM {table} WHERE workspace_id='w'"), + [], + |r| r.get::<_, i64>(0) + )?, + 0 + ); + } + assert_eq!( + c.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| r + .get::<_, i64>( + 0 + ))?, + 0 + ); + Ok(()) + }) + .unwrap(); + s.erase_subject_deleted_jobs("w", "subject", "delete") + .unwrap(); + s.finish_subject_deletion_record("w", "subject", "delete") + .unwrap(); + s.finish_subject_deletion_record("w", "subject", "delete") + .unwrap(); + assert!( + s.reserve_backend_job("w", &request("job", "different-subject"), NOW) + .is_err() + ); + assert!( + s.finish_subject_deletion_record("w", "subject", "wrong-operation") + .is_err() + ); + } + + #[test] + fn migration_is_atomic_matches_fresh_and_survives_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("main.db"); + crate::store::tests::prepare_retained_schema(&path, 104); + let c = Connection::open(&path).unwrap(); + configure_sqlite(&c).unwrap(); + c.execute_batch("CREATE TRIGGER interrupt_subject_deletion BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=105 BEGIN SELECT RAISE(ABORT,'interrupted'); END;").unwrap(); + assert!(migrate(&c).is_err()); + assert_eq!(current_schema_version(&c).unwrap(), 104); + assert!(!table_exists(&c, "subject_deletions").unwrap()); + c.execute_batch("DROP TRIGGER interrupt_subject_deletion") + .unwrap(); + migrate(&c).unwrap(); + assert!(migrate(&c).is_err()); + let schema = |c: &Connection| -> Result> { + Ok(c.prepare("SELECT name,sql FROM sqlite_schema WHERE name LIKE 'subject_delet%' OR name LIKE 'subject_deleted%' OR name IN ('notification_sources_no_delete','notification_requests_no_delete','notification_job_targets_no_delete') ORDER BY name")? + .query_map([],|r|Ok((r.get(0)?,r.get::<_,String>(1)?.split_whitespace().collect::>().join(" "))))? + .collect::>>()?) + }; + let fresh = SqliteWorkspaceStore::in_memory_fixture().unwrap(); + assert_eq!(schema(&c).unwrap(), fresh.with_conn(schema).unwrap()); + drop(c); + let s = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + seed(&s); + let old_worker = worker(61); + receipt(&s, &old_worker, "succeeded", "purge", "purge"); + s.prepare_subject_deletion("w", "subject", "delete", std::slice::from_ref(&old_worker)) + .unwrap(); + drop(s); + let s = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + assert_eq!( + saved_prior(&s, "subject", &old_worker), + Some(format!("receipt-{}-succeeded", old_worker.worker_id)) + ); + s.prepare_subject_deletion("w", "subject", "delete", &[]) + .unwrap(); + s.erase_subject_deleted_jobs("w", "subject", "delete") + .unwrap(); + s.finish_subject_deletion_record("w", "subject", "delete") + .unwrap(); + } + + #[test] + fn concurrent_job_reservation_is_either_snapshotted_or_rejected() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("main.db"); + let s = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + seed(&s); + let other = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let gate = std::sync::Arc::new(std::sync::Barrier::new(2)); + let other_gate = gate.clone(); + let reserve = std::thread::spawn(move || { + other_gate.wait(); + other.reserve_backend_job("w", &request("race", "subject"), NOW) + }); + gate.wait(); + s.prepare_subject_deletion("w", "subject", "delete", &[]) + .unwrap(); + let reserved = reserve.join().unwrap().is_ok(); + s.with_conn(|c| { + let included:bool=c.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_jobs WHERE workspace_id='w' AND job_id='race')",[],|r|r.get(0))?; + assert_eq!(included,reserved);Ok(()) + }).unwrap(); + if reserved { + assert_eq!( + s.get_backend_job("w", "race").unwrap().unwrap().state, + BackendJobState::Failed + ); + } + assert!( + s.reserve_backend_job("w", &request("later", "subject"), NOW) + .is_err() + ); + } + + #[test] + fn singleton_release_waits_for_canonical_removal_and_absence_is_not_success() { + let s = setup(); + let body = worker(21); + reservation(&s, &body, "body", Some("subjektiv:subject")); + s.prepare_subject_deletion("w", "subject", "delete", &[]) + .unwrap(); + let fence = runtime_api::SubjectDeleteWorkerResponse { + workspace_id: "w".into(), + worker_id: WorkerId::from_legacy_u64(21), + create_fingerprint: "fp".into(), + operation_id: s + .subject_deletion_worker_request("w", "subject", "delete", &body) + .unwrap() + .operation_id, + originals_absent: false, + }; + assert!( + s.confirm_subject_deletion_absence("w", "subject", "delete", &body, &fence) + .is_err() + ); + assert!( + s.current_worker_singleton_owner("w", "subjektiv:subject") + .unwrap() + .is_some() + ); + receipt(&s, &body, "failed", "archive", "retain"); + assert!( + s.confirm_subject_deletion_absence("w", "subject", "delete", &body, &fence) + .is_err() + ); + s.erase_subject_deleted_jobs("w", "subject", "delete") + .unwrap(); + assert!( + s.finish_subject_deletion_record("w", "subject", "delete") + .is_err() + ); + receipt(&s, &body, "succeeded", "archive", "retain"); + assert!( + s.finish_subject_deletion_record("w", "subject", "delete") + .is_err(), + "reservation is not yet terminal" + ); + s.with_conn(|c| { + c.execute( + "UPDATE worker_create_reservations SET state='removed' WHERE workspace_id='w'", + [], + )?; + Ok(()) + }) + .unwrap(); + assert!( + s.current_worker_singleton_owner("w", "subjektiv:subject") + .unwrap() + .is_none() + ); + s.confirm_subject_deletion_absence("w", "subject", "delete", &body, &fence) + .unwrap(); + s.finish_subject_deletion_record("w", "subject", "delete") + .unwrap(); + } + + fn saved_prior( + s: &SqliteWorkspaceStore, + subject: &str, + w: &RuntimeWorkerRef, + ) -> Option { + s.with_conn(|c|Ok(c.query_row("SELECT prior_removal_operation_id FROM subject_deletion_workers WHERE workspace_id='w' AND subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![subject,w.runtime_id,w.worker_id],|r|r.get(0))?)).unwrap() + } + + #[test] + fn prior_archived_job_and_purged_body_preserve_original_receipts_and_other_subject() { + let s = setup(); + let req = request("archived-job", "subject"); + let job = s.reserve_backend_job("w", &req, NOW).unwrap(); + let jw = worker(31); + registry(&s, &jw); + s.claim_backend_job_attempt_dispatch("w", &req.job_id, &job.attempt.attempt_id, NOW) + .unwrap(); + s.bind_backend_job_attempt_worker( + "w", + &req.job_id, + &job.attempt.attempt_id, + &jw, + Some("run"), + NOW, + ) + .unwrap(); + s.accept_backend_job_result( + "w", + &jw, + &BackendJobResultSubmission { + job_id: req.job_id.clone(), + attempt_id: job.attempt.attempt_id.clone(), + input_digest: req.input_digest().unwrap(), + result: serde_json::json!({"secret":"old result"}), + }, + NOW, + ) + .unwrap(); + receipt(&s, &jw, "succeeded", "archive", "retain"); + let job_receipt = format!("receipt-{}-succeeded", jw.worker_id); + let body = worker(32); + reservation(&s, &body, "body", Some("subjektiv:subject")); + receipt(&s, &body, "succeeded", "purge", "purge"); + let body_receipt = format!("receipt-{}-succeeded", body.worker_id); + // A different Subject's already-removed body remains occupied until its + // own deletion/release. Deleting a delegating Subject must not release it. + let other_body = worker(33); + reservation(&s, &other_body, "other-body", Some("subjektiv:another")); + receipt(&s, &other_body, "succeeded", "purge", "purge"); + let other_job = s + .reserve_backend_job("w", &request("other-job", "another"), NOW) + .unwrap(); + s.with_conn(|c| { + c.execute("DELETE FROM worker_registry WHERE workspace_id='w' AND worker_id=?1",[&jw.worker_id])?; + c.execute("UPDATE worker_create_reservations SET state='removed' WHERE workspace_id='w'",[])?; + c.execute("INSERT INTO worker_session_archives(archive_id,workspace_id,runtime_id,worker_id,session_id,checksum_sha256,content_bytes,policy_id,policy_digest,operation_id,committed_at,expires_at) VALUES('old-archive','w',?1,?2,'session','checksum',42,'test','digest',?3,?4,?5)",params![jw.runtime_id,jw.worker_id,job_receipt,NOW,"2027-01-01T00:00:00Z"])?; + // Prior metadata/retention selections need not match the new policy. + c.execute("UPDATE worker_removal_operations SET metadata_disposition='purge',archive_retention_kind='for_seconds',archive_retention_seconds=300 WHERE operation_id=?1",[&job_receipt])?; + Ok(()) + }).unwrap(); + let archive_before = s.worker_session_archive("w", &jw, "session").unwrap(); + assert!(archive_before.is_some()); + s.prepare_subject_deletion("w", "subject", "delete", std::slice::from_ref(&other_body)) + .unwrap(); + assert_eq!(saved_prior(&s, "subject", &jw), Some(job_receipt.clone())); + assert_eq!( + saved_prior(&s, "subject", &body), + Some(body_receipt.clone()) + ); + assert_eq!( + s.recover_worker_removal_execution("w", &jw) + .unwrap() + .unwrap() + .plan + .operation_id, + job_receipt + ); + assert_eq!( + s.recover_worker_removal_execution("w", &body) + .unwrap() + .unwrap() + .plan + .operation_id, + body_receipt + ); + s.erase_subject_deleted_jobs("w", "subject", "delete") + .unwrap(); + // Main finish intentionally precedes authoritative domain completion. + s.finish_subject_deletion_record("w", "subject", "delete") + .unwrap(); + assert_eq!( + s.worker_session_archive("w", &jw, "session").unwrap(), + archive_before + ); + assert!(s.get_backend_job("w", "archived-job").unwrap().is_none()); + assert_eq!( + s.get_backend_job("w", "other-job").unwrap().unwrap(), + other_job.job + ); + assert!( + s.current_worker_singleton_owner("w", "subjektiv:subject") + .unwrap() + .is_none() + ); + assert!( + s.current_worker_singleton_owner("w", "subjektiv:another") + .unwrap() + .is_some() + ); + s.with_conn(|c| { + let old:(String,String,String)=c.query_row("SELECT state,session_disposition,metadata_disposition FROM worker_removal_operations WHERE operation_id=?1",[&job_receipt],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?; + assert_eq!(old,("succeeded".into(),"archive".into(),"purge".into())); + Ok(()) + }).unwrap(); + } + + #[test] + fn late_success_and_unknown_removal_cannot_become_a_prior_receipt() { + let s = setup(); + let job_worker = worker(41); + let body = worker(42); + reservation( + &s, + &job_worker, + "job-owner", + Some("subjektiv-consolidation:subject"), + ); + receipt(&s, &job_worker, "executing", "archive", "retain"); + receipt(&s, &body, "failed", "purge", "purge"); + s.prepare_subject_deletion("w", "subject", "delete", std::slice::from_ref(&body)) + .unwrap(); + assert_eq!(saved_prior(&s, "subject", &job_worker), None); + assert_eq!(saved_prior(&s, "subject", &body), None); + receipt(&s, &job_worker, "succeeded", "archive", "retain"); + receipt(&s, &body, "succeeded", "purge", "purge"); + s.prepare_subject_deletion("w", "subject", "delete", &[]) + .unwrap(); + assert_eq!( + saved_prior(&s, "subject", &job_worker), + None, + "replay must not bless a later success" + ); + assert!( + s.erase_subject_deleted_jobs("w", "subject", "delete") + .is_err(), + "new Job Archive is not Purge" + ); + s.with_conn(|c| { + assert!(c.execute("UPDATE subject_deletion_workers SET prior_removal_operation_id=?1 WHERE workspace_id='w' AND worker_id=?2",params![format!("receipt-{}-succeeded",job_worker.worker_id),job_worker.worker_id]).is_err()); + c.execute("UPDATE worker_removal_operations SET session_disposition='purge',diagnostics_disposition='purge' WHERE worker_id=?1 AND state='succeeded'",[&job_worker.worker_id])?; + c.execute("UPDATE worker_create_reservations SET state='removed' WHERE workspace_id='w'",[])?; + Ok(()) + }).unwrap(); + s.erase_subject_deleted_jobs("w", "subject", "delete") + .unwrap(); + assert!( + s.finish_subject_deletion_record("w", "subject", "delete") + .is_err(), + "new body Purge is not Archive Forever" + ); + } + + #[test] + fn saved_prior_receipt_must_remain_the_exact_success_not_a_replacement() { + let s = setup(); + let w = worker(51); + receipt(&s, &w, "succeeded", "archive", "retain"); + s.prepare_subject_deletion("w", "subject", "delete", std::slice::from_ref(&w)) + .unwrap(); + let old = saved_prior(&s, "subject", &w).unwrap(); + s.with_conn(|c| { + // Simulate inconsistent retained evidence. A different valid new + // receipt must not silently replace the immutable snapshotted one. + c.execute("UPDATE worker_removal_operations SET state='failed' WHERE operation_id=?1",[&old])?; + c.execute("INSERT INTO worker_removal_operations SELECT 'replacement','replacement',input_fingerprint,workspace_id,runtime_id,worker_id,worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition,archive_retention_kind,archive_retention_seconds,diagnostics_disposition,diagnostics_retention_seconds,NULL,blockers_json,'succeeded',reason,failure_category,created_at,updated_at FROM worker_removal_operations WHERE operation_id=?1",[&old])?; + Ok(()) + }).unwrap(); + s.erase_subject_deleted_jobs("w", "subject", "delete") + .unwrap(); + assert!( + s.finish_subject_deletion_record("w", "subject", "delete") + .is_err() + ); + s.with_conn(|c| { + c.execute( + "UPDATE worker_removal_operations SET state='succeeded' WHERE operation_id=?1", + [&old], + )?; + Ok(()) + }) + .unwrap(); + s.finish_subject_deletion_record("w", "subject", "delete") + .unwrap(); + } +} diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index 802e4e52a..3de7c89cb 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -94,6 +94,12 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ "workdir_registry", "workdir_removal_operations", "worker_control_grants", + // Snapshot children precede their fence parent and canonical Worker removal + // receipts (subject_deletion_workers.prior_removal_operation_id). + "subject_deletion_allocations", + "subject_deletion_jobs", + "subject_deletion_workers", + "subject_deletions", "subject_worker_releases", "subject_create_recoveries", "worker_create_reservations", @@ -1276,6 +1282,134 @@ mod tests { } } + #[test] + fn workspace_purge_removes_subject_deletion_snapshots_and_preserves_other_workspace() { + let (store, owner, workspace_id) = setup(); + let tables = [ + "subject_deletion_allocations", + "subject_deletion_jobs", + "subject_deletion_workers", + "subject_deletions", + "worker_removal_operations", + ]; + // Check the explicit FK ordering as well as execution (the purge defers FKs). + let position = |table| { + WORKSPACE_DELETION_PURGE_TABLES + .iter() + .position(|name| *name == table) + .unwrap() + }; + for child in &tables[..3] { + assert!(position(*child) < position("subject_deletions")); + } + assert!(position("subject_deletion_workers") < position("worker_removal_operations")); + for workspace in ["workspace-a", "workspace-b"] { + store.with_conn(|conn| { + conn.execute("INSERT INTO worker_removal_operations( + operation_id,plan_id,input_fingerprint,workspace_id,runtime_id,worker_id, + worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition, + archive_retention_kind,diagnostics_disposition,blockers_json,state,reason,created_at,updated_at) + VALUES(?1,?1,'fingerprint',?1,'runtime','worker','1','policy','digest', + 'purge','purge','forever','purge','[]','succeeded','removed','1','1')", [workspace])?; + conn.execute("INSERT INTO subject_deletions(workspace_id,subject_id,operation_id,completed,jobs_erased) + VALUES(?1,'subject','subject-delete',1,1)", [workspace])?; + conn.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id,originals_absent) + VALUES(?1,'subject','runtime','worker',1,?1,1)", [workspace])?; + conn.execute("INSERT INTO subject_deletion_jobs VALUES(?1,'subject','job')", [workspace])?; + conn.execute("INSERT INTO subject_deletion_allocations VALUES(?1,'subject','allocation')", [workspace])?; + Ok(()) + }).unwrap(); + } + let rows = |workspace: &str| { + store + .with_conn(|conn| { + tables + .iter() + .map(|table| { + let mut statement = conn + .prepare(&format!("SELECT * FROM {table} WHERE workspace_id=?1"))?; + let count = statement.column_count(); + Ok(statement + .query_map([workspace], |row| { + (0..count) + .map(|index| row.get::<_, rusqlite::types::Value>(index)) + .collect::>>() + })? + .collect::>>()?) + }) + .collect::>>() + }) + .unwrap() + }; + let before = rows("workspace-a"); + let other = rows("workspace-b"); + assert!(before.iter().all(|table| table.len() == 1)); + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .unwrap(); + let operation = store + .reserve_workspace_deletion( + &owner, + &workspace_id, + &WorkspaceDeletionRequest { + operation_id: "purge-subject-snapshots".into(), + expected_workspace_updated_at: preflight.expected_workspace_updated_at, + confirmation: "Alpha".into(), + }, + ) + .unwrap() + .operation; + store + .with_conn(|conn| { + conn.execute_batch( + "CREATE TRIGGER reject_subject_snapshot_purge BEFORE DELETE ON workspaces + BEGIN SELECT RAISE(ABORT,'injected purge failure'); END;", + )?; + Ok(()) + }) + .unwrap(); + assert!( + store + .finalize_workspace_deletion(&operation.operation_id) + .is_err() + ); + assert_eq!( + rows("workspace-a"), + before, + "rollback restores snapshots and canonical receipt" + ); + assert_eq!(rows("workspace-b"), other); + store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER reject_subject_snapshot_purge")?; + Ok(()) + }) + .unwrap(); + let completed = store + .finalize_workspace_deletion(&operation.operation_id) + .unwrap(); + assert_eq!(completed.state, WorkspaceDeletionState::Succeeded); + assert!(rows("workspace-a").iter().all(|table| table.is_empty())); + assert_eq!(rows("workspace-b"), other); + assert_eq!( + store + .finalize_workspace_deletion(&operation.operation_id) + .unwrap(), + completed + ); + store + .with_conn(|conn| { + assert_eq!( + conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| { + row.get::<_, i64>(0) + })?, + 0 + ); + Ok(()) + }) + .unwrap(); + } + #[test] fn deletion_is_idempotent_and_removes_workspace_scoped_rows() { let (store, owner, workspace_id) = setup(); diff --git a/openapi/server-api.json b/openapi/server-api.json index ac5c7b8ea..72e76e24c 100644 --- a/openapi/server-api.json +++ b/openapi/server-api.json @@ -11127,6 +11127,40 @@ ], "type": "object" }, + "SubjektivSubjectDeleteRequest": { + "additionalProperties": false, + "properties": { + "operation_id": { + "description": "Stable identity retained for every retry, including unknown outcomes.", + "type": "string" + } + }, + "required": [ + "operation_id" + ], + "type": "object" + }, + "SubjektivSubjectDeleteResponse": { + "additionalProperties": false, + "properties": { + "completed": { + "description": "False confirms the deletion fence, not external stop or erasure completion.", + "type": "boolean" + }, + "operation_id": { + "type": "string" + }, + "subject_id": { + "type": "string" + } + }, + "required": [ + "subject_id", + "operation_id", + "completed" + ], + "type": "object" + }, "SubjektivSubjectListQuery": { "additionalProperties": false, "properties": { @@ -11228,11 +11262,19 @@ "type": "object" }, "SubjektivSubjectState": { - "enum": [ - "active", - "retired" - ], - "type": "string" + "oneOf": [ + { + "enum": [ + "active" + ], + "type": "string" + }, + { + "const": "retired", + "description": "Saved compatibility only; not a current lifecycle operation.", + "type": "string" + } + ] }, "SubjektivWorkerConnectionRequest": { "additionalProperties": false, @@ -17129,6 +17171,10 @@ "additionalProperties": false, "description": "Backend-authoritative permissions for the current Workspace actor.", "properties": { + "delete_subjects": { + "description": "Explicit Subject deletion authority for the current authenticated actor.", + "type": "boolean" + }, "delete_workspace": { "type": "boolean" }, @@ -17146,7 +17192,8 @@ "manage_repositories", "manage_secrets", "manage_runtimes", - "delete_workspace" + "delete_workspace", + "delete_subjects" ], "type": "object" }, @@ -18302,7 +18349,7 @@ "info": { "title": "Yoi Server API", "version": "0.1.0", - "x-yoi-source-digest": "sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4" + "x-yoi-source-digest": "sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e" }, "openapi": "3.1.0", "paths": { @@ -35014,6 +35061,121 @@ } }, "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}": { + "delete": { + "operationId": "subjektiv_subject_delete", + "parameters": [ + { + "explode": false, + "in": "path", + "name": "workspace_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/string" + }, + "style": "simple" + }, + { + "explode": false, + "in": "path", + "name": "subject_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/string" + }, + "style": "simple" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SubjektivSubjectDeleteRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SubjektivSubjectDeleteResponse" + } + } + }, + "description": "Successful response" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + } + }, + "security": [ + { + "bearerAuth": [] + }, + { + "browserSession": [] + } + ] + }, "get": { "operationId": "subjektiv_subject_get", "parameters": [ @@ -35790,113 +35952,6 @@ ] } }, - "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire": { - "post": { - "operationId": "subjektiv_subject_retire", - "parameters": [ - { - "explode": false, - "in": "path", - "name": "workspace_id", - "required": true, - "schema": { - "$ref": "#/components/schemas/string" - }, - "style": "simple" - }, - { - "explode": false, - "in": "path", - "name": "subject_id", - "required": true, - "schema": { - "$ref": "#/components/schemas/string" - }, - "style": "simple" - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SubjektivSubjectResponse" - } - } - }, - "description": "Successful response" - }, - "400": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RepositoryApiError" - } - } - }, - "description": "Error response" - }, - "401": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RepositoryApiError" - } - } - }, - "description": "Error response" - }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RepositoryApiError" - } - } - }, - "description": "Error response" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RepositoryApiError" - } - } - }, - "description": "Error response" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RepositoryApiError" - } - } - }, - "description": "Error response" - }, - "500": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RepositoryApiError" - } - } - }, - "description": "Error response" - } - }, - "security": [ - { - "bearerAuth": [] - }, - { - "browserSession": [] - } - ] - } - }, "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/surface": { "get": { "operationId": "subjektiv_resident_surface", diff --git a/tools/state-contract-name-exceptions.json b/tools/state-contract-name-exceptions.json index bebae656f..4a32b44bc 100644 --- a/tools/state-contract-name-exceptions.json +++ b/tools/state-contract-name-exceptions.json @@ -32,6 +32,14 @@ } ] }, + { + "path": "crates/subjektiv/src/deletion/tests.rs", + "kind": "saved_data", + "reason": "schema 6の固定保存Subject行を投入し、最新migrationが旧retiredとarchive原文を黙って消去・再活性化せず、明示削除だけで消去する検証に必要な旧SQL列名。現行writerや競合制御には使わない。", + "lines": [ + { "sha256": "defca5951b3d48785514b25912044fa3fc252659311b09a31fa2e64af2a19049", "count": 1 } + ] + }, { "path": "crates/subjektiv/src/journal/memory_links/tests.rs", "kind": "saved_data", diff --git a/tools/web-ux/scenarios/subject-delete-dark.json b/tools/web-ux/scenarios/subject-delete-dark.json new file mode 100644 index 000000000..12e9f5583 --- /dev/null +++ b/tools/web-ux/scenarios/subject-delete-dark.json @@ -0,0 +1,162 @@ +{ + "schemaVersion": 1, + "id": "subject-delete-dark", + "title": "Subject deletion confirmation and retry", + "baseUrl": "http://127.0.0.1:15287", + "colorScheme": "dark", + "reducedMotion": "reduce", + "personas": [ + { + "id": "fixture", + "label": "Authenticated owner (synthetic HTTP)", + "auth": { + "kind": "anonymous" + } + } + ], + "viewports": [ + { + "label": "desktop", + "width": 1440, + "height": 900 + }, + { + "label": "narrow", + "width": 768, + "height": 900 + }, + { + "label": "mobile", + "width": 390, + "height": 844 + } + ], + "routes": [ + { + "id": "subject", + "label": "Subject detail", + "path": "/w/memory-review/memory/pending-subject", + "goal": "Review the Subject before irreversible deletion.", + "dataState": "Active Subject, no current Worker; owner permissions from Backend projection.", + "ready": { + "kind": "selector", + "selector": ".subject-overview" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Subject detail" + }, + { + "id": "confirmation", + "label": "Irreversible deletion scope and confirmation", + "interaction": [ + { + "action": "click", + "selector": ".subject-delete-open" + } + ], + "ready": { + "kind": "selector", + "selector": "[data-delete-state=confirming]" + } + }, + { + "id": "pending", + "label": "Fence saved; erasure unconfirmed", + "interaction": [ + { + "action": "click", + "selector": ".delete-confirm input" + }, + { + "action": "click", + "selector": ".subject-deletion button[type=submit]" + } + ], + "ready": { + "kind": "selector", + "selector": "[data-delete-state=pending]" + } + }, + { + "id": "completed", + "label": "Deletion confirmed", + "interaction": [ + { + "action": "click", + "selector": ".subject-delete-retry" + } + ], + "ready": { + "kind": "selector", + "selector": "[data-delete-state=completed]" + } + } + ] + }, + { + "id": "legacy", + "label": "Legacy ended Subject", + "path": "/w/memory-review/memory/legacy-subject", + "goal": "Delete a legacy ended Subject without reactivating it.", + "dataState": "Saved retired wire state, owner.", + "ready": { + "kind": "selector", + "selector": ".subject-delete-open" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Legacy ended Subject" + }, + { + "id": "confirmation", + "label": "Delete legacy Subject", + "interaction": [ + { + "action": "click", + "selector": ".subject-delete-open" + } + ], + "ready": { + "kind": "selector", + "selector": "[data-delete-state=confirming]" + } + } + ] + }, + { + "id": "non-owner", + "label": "Nonowner Subject view", + "path": "/w/memory-review/memory/legacy-subject?persona=reader", + "goal": "Read Subject without presenting unauthorized deletion.", + "dataState": "Backend delete_subjects=false; no deletion controls.", + "ready": { + "kind": "selector", + "selector": ".subject-overview" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Nonowner details" + } + ] + } + ], + "processes": [ + { + "id": "subject-delete-fixture", + "command": "deno", + "cwd": "..", + "args": [ + "run", + "-A", + "../../web/workspace/test/subject-delete-browser/fixture-server.ts", + "15287", + "../../web/workspace/build" + ], + "readyUrl": "http://127.0.0.1:15287/health" + } + ] +} diff --git a/tools/web-ux/scenarios/subject-delete-light.json b/tools/web-ux/scenarios/subject-delete-light.json new file mode 100644 index 000000000..b40e43656 --- /dev/null +++ b/tools/web-ux/scenarios/subject-delete-light.json @@ -0,0 +1,162 @@ +{ + "schemaVersion": 1, + "id": "subject-delete-light", + "title": "Subject deletion confirmation and retry", + "baseUrl": "http://127.0.0.1:15287", + "colorScheme": "light", + "reducedMotion": "reduce", + "personas": [ + { + "id": "fixture", + "label": "Authenticated owner (synthetic HTTP)", + "auth": { + "kind": "anonymous" + } + } + ], + "viewports": [ + { + "label": "desktop", + "width": 1440, + "height": 900 + }, + { + "label": "narrow", + "width": 768, + "height": 900 + }, + { + "label": "mobile", + "width": 390, + "height": 844 + } + ], + "routes": [ + { + "id": "subject", + "label": "Subject detail", + "path": "/w/memory-review/memory/pending-subject", + "goal": "Review the Subject before irreversible deletion.", + "dataState": "Active Subject, no current Worker; owner permissions from Backend projection.", + "ready": { + "kind": "selector", + "selector": ".subject-overview" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Subject detail" + }, + { + "id": "confirmation", + "label": "Irreversible deletion scope and confirmation", + "interaction": [ + { + "action": "click", + "selector": ".subject-delete-open" + } + ], + "ready": { + "kind": "selector", + "selector": "[data-delete-state=confirming]" + } + }, + { + "id": "pending", + "label": "Fence saved; erasure unconfirmed", + "interaction": [ + { + "action": "click", + "selector": ".delete-confirm input" + }, + { + "action": "click", + "selector": ".subject-deletion button[type=submit]" + } + ], + "ready": { + "kind": "selector", + "selector": "[data-delete-state=pending]" + } + }, + { + "id": "completed", + "label": "Deletion confirmed", + "interaction": [ + { + "action": "click", + "selector": ".subject-delete-retry" + } + ], + "ready": { + "kind": "selector", + "selector": "[data-delete-state=completed]" + } + } + ] + }, + { + "id": "legacy", + "label": "Legacy ended Subject", + "path": "/w/memory-review/memory/legacy-subject", + "goal": "Delete a legacy ended Subject without reactivating it.", + "dataState": "Saved retired wire state, owner.", + "ready": { + "kind": "selector", + "selector": ".subject-delete-open" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Legacy ended Subject" + }, + { + "id": "confirmation", + "label": "Delete legacy Subject", + "interaction": [ + { + "action": "click", + "selector": ".subject-delete-open" + } + ], + "ready": { + "kind": "selector", + "selector": "[data-delete-state=confirming]" + } + } + ] + }, + { + "id": "non-owner", + "label": "Nonowner Subject view", + "path": "/w/memory-review/memory/legacy-subject?persona=reader", + "goal": "Read Subject without presenting unauthorized deletion.", + "dataState": "Backend delete_subjects=false; no deletion controls.", + "ready": { + "kind": "selector", + "selector": ".subject-overview" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Nonowner details" + } + ] + } + ], + "processes": [ + { + "id": "subject-delete-fixture", + "command": "deno", + "cwd": "..", + "args": [ + "run", + "-A", + "../../web/workspace/test/subject-delete-browser/fixture-server.ts", + "15287", + "../../web/workspace/build" + ], + "readyUrl": "http://127.0.0.1:15287/health" + } + ] +} diff --git a/web/workspace/deno.json b/web/workspace/deno.json index 7384240af..96f1cc0e5 100644 --- a/web/workspace/deno.json +++ b/web/workspace/deno.json @@ -6,7 +6,7 @@ "dev": "deno run -A npm:vite@8.3.2 dev", "dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787", "check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json", - "test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts", + "test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts tests/subject-delete.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts", "test:journal:backend-contract": "cd ../.. && mkdir -p target && cargo test -p yoi-workspace-server --lib journal_browser_web_contract -- --nocapture > target/journal-browser-contract.log && cd web/workspace && deno run --allow-read=../../target/journal-browser-contract.log test/journal-backend-contract.ts ../../target/journal-browser-contract.log", "test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts", "test:browser:activation": "mkdir -p ../../target/web-ux/tmp && TMPDIR=../../target/web-ux/tmp deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/activation-browser/activation-shell.browser.test.ts", diff --git a/web/workspace/deno.lock b/web/workspace/deno.lock index c8d81e1f8..dace4ea66 100644 --- a/web/workspace/deno.lock +++ b/web/workspace/deno.lock @@ -3,6 +3,7 @@ "specifiers": { "jsr:@std/assert@*": "1.0.19", "jsr:@std/internal@^1.0.12": "1.0.14", + "jsr:@std/path@1.1.4": "1.1.4", "npm:@codemirror/autocomplete@6.20.3": "6.20.3", "npm:@codemirror/commands@6.11.1": "6.11.1", "npm:@codemirror/language@6.12.4": "6.12.4", @@ -42,6 +43,12 @@ }, "@std/internal@1.0.14": { "integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7" + }, + "@std/path@1.1.4": { + "integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5", + "dependencies": [ + "jsr:@std/internal" + ] } }, "npm": { diff --git a/web/workspace/src/lib/generated/companion-api.ts b/web/workspace/src/lib/generated/companion-api.ts index c886812bb..275abef1b 100644 --- a/web/workspace/src/lib/generated/companion-api.ts +++ b/web/workspace/src/lib/generated/companion-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-companion-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4) +// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e) // Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/companion-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check diff --git a/web/workspace/src/lib/generated/legacy-server-api.ts b/web/workspace/src/lib/generated/legacy-server-api.ts index ba2d538e1..85f166635 100644 --- a/web/workspace/src/lib/generated/legacy-server-api.ts +++ b/web/workspace/src/lib/generated/legacy-server-api.ts @@ -52,6 +52,10 @@ export type WorkspacePermissionSummary = { manage_secrets: boolean; manage_runtimes: boolean; delete_workspace: boolean; + /** + * Explicit Subject deletion authority for the current authenticated actor. + */ + delete_subjects: boolean; }; export type WorkspaceDeletionState = diff --git a/web/workspace/src/lib/generated/memory-api.ts b/web/workspace/src/lib/generated/memory-api.ts index d2ed15536..ab6547ec4 100644 --- a/web/workspace/src/lib/generated/memory-api.ts +++ b/web/workspace/src/lib/generated/memory-api.ts @@ -118,6 +118,14 @@ export type SubjektivSubjectCreateRequest = { behavior_md?: string; }; +export type SubjektivSubjectDeleteRequest = { operation_id: string }; + +export type SubjektivSubjectDeleteResponse = { + subject_id: string; + operation_id: string; + completed: boolean; +}; + export type SubjektivSubjectBehaviorUpdateRequest = { expected_behavior_md: string; behavior_md: string; diff --git a/web/workspace/src/lib/generated/repository-api.ts b/web/workspace/src/lib/generated/repository-api.ts index 8cafc53b8..c23b3aa97 100644 --- a/web/workspace/src/lib/generated/repository-api.ts +++ b/web/workspace/src/lib/generated/repository-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-repository-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4) +// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e) // Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output // Output: web/workspace/src/lib/generated/repository-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check diff --git a/web/workspace/src/lib/generated/runtime-api.ts b/web/workspace/src/lib/generated/runtime-api.ts index c12b6c31e..e5e081829 100644 --- a/web/workspace/src/lib/generated/runtime-api.ts +++ b/web/workspace/src/lib/generated/runtime-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-runtime-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4) +// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e) // Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/runtime-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check diff --git a/web/workspace/src/lib/generated/ticket-api.ts b/web/workspace/src/lib/generated/ticket-api.ts index b653b6d8a..114891abc 100644 --- a/web/workspace/src/lib/generated/ticket-api.ts +++ b/web/workspace/src/lib/generated/ticket-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4) +// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e) // Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/ticket-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check diff --git a/web/workspace/src/lib/generated/worker-launch-api.ts b/web/workspace/src/lib/generated/worker-launch-api.ts index 19536c02c..69ba96cc0 100644 --- a/web/workspace/src/lib/generated/worker-launch-api.ts +++ b/web/workspace/src/lib/generated/worker-launch-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-worker-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4) +// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e) // Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/worker-launch-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check diff --git a/web/workspace/src/lib/workspace/api/workspace-model.ts b/web/workspace/src/lib/workspace/api/workspace-model.ts index c776a15f1..1170b520d 100644 --- a/web/workspace/src/lib/workspace/api/workspace-model.ts +++ b/web/workspace/src/lib/workspace/api/workspace-model.ts @@ -482,6 +482,7 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary { "manage_secrets", "manage_runtimes", "delete_workspace", + "delete_subjects", ], path, ); @@ -492,6 +493,7 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary { ), manage_secrets: boolean(item.manage_secrets, `${path}.manage_secrets`), manage_runtimes: boolean(item.manage_runtimes, `${path}.manage_runtimes`), + delete_subjects: boolean(item.delete_subjects, `${path}.delete_subjects`), delete_workspace: boolean( item.delete_workspace, `${path}.delete_workspace`, diff --git a/web/workspace/src/lib/workspace/memory/SubjectDelete.svelte b/web/workspace/src/lib/workspace/memory/SubjectDelete.svelte new file mode 100644 index 000000000..b6a35b541 --- /dev/null +++ b/web/workspace/src/lib/workspace/memory/SubjectDelete.svelte @@ -0,0 +1,134 @@ + + +{#if subject || attempt || storageError} +
+
+

Delete Subject

+ {#if phase === 'idle' && !storageError} + + {/if} +
+ {#if phase === 'confirming' || attempt} +

Target: {subject?.role ?? subjectId} · {subjectId}

+ {/if} + {#if phase === 'confirming'} +
{ event.preventDefault(); void submit(); }}> +

This is irreversible. This Subject, its behavior, Memory, change history, staging, Surface, Journal, permissions, delegations, Writer/Session associations, and Subject Job inputs/results will be deleted. Connected and delegated Workers and Jobs must stop before completion can be confirmed.

+

Original Worker/Session text, existing archives and Workdirs, Tickets, and other separately retained resources are not deleted. Minimal deletion retry/audit metadata is retained. Other Subjects and Workspaces are not affected.

+ +
+ + +
+
+ {:else if phase === 'running'} +

Deleting… Waiting for stop and erasure confirmation. Do not treat this as completed.

+ {:else if phase === 'pending'} +

Deletion started; completion unconfirmed. The deletion fence is saved and the Subject is hidden from reads/lists. External stopping or erasure is not yet confirmed. Retry the same request to continue.

+ {:else if phase === 'unknown'} +

Deletion outcome unknown. The request may already have taken effect. The saved target and operation ID are retained; retry the same request, not a new deletion.

+ {:else if phase === 'failed'} +

Deletion request rejected. Completion is not confirmed. Resolve the reported condition, then retry the same request.

+ {:else if phase === 'completed'} +

Subject deletion completed. Backend confirmed stopping and deletion of Subject-owned data. Separately retained Worker/Session text, archives, Workdirs, and Tickets remain.

+ Back to subjects → + {/if} + {#if error}{/if} + {#if attempt && phase !== 'completed'} +
Deletion request identity{attempt.operationId}

This tab retains the target and same request ID across reloads, even when GET can no longer find the Subject.

+ {#if phase === 'pending' || phase === 'unknown' || phase === 'failed'} + + {/if} + {/if} +
+{/if} + + diff --git a/web/workspace/src/lib/workspace/memory/api.ts b/web/workspace/src/lib/workspace/memory/api.ts index fbda4f362..8bb095fd1 100644 --- a/web/workspace/src/lib/workspace/memory/api.ts +++ b/web/workspace/src/lib/workspace/memory/api.ts @@ -29,6 +29,8 @@ import type { SubjektivResidentSurfaceAvailability, SubjektivResidentSurfaceResponse, SubjektivResidentSurfaceSnapshot, + SubjektivSubjectDeleteRequest, + SubjektivSubjectDeleteResponse, SubjektivSubjectBehaviorUpdateRequest, SubjektivSubjectCreateRequest, SubjektivSubjectListResponse, @@ -158,6 +160,70 @@ function validateSubjectBehavior(behaviorMd: string): void { } } +export class SubjektivSubjectDeleteError extends Error { + constructor( + readonly kind: "rejected" | "unknown_outcome", + message: string, + readonly status: number | null = null, + ) { + super(message); + this.name = "SubjektivSubjectDeleteError"; + } +} + +export function parseSubjektivSubjectDeleteResponse( + value: unknown, + subjectId: string, + operationId: string, +): SubjektivSubjectDeleteResponse { + const record = strictRecord(value, ["subject_id", "operation_id", "completed"], "Subject deletion"); + const response = { + subject_id: requiredIdentifier(record, "subject_id"), + operation_id: requiredIdentifier(record, "operation_id"), + completed: requiredBoolean(record, "completed"), + }; + if (response.subject_id !== subjectId || response.operation_id !== operationId) { + invalid("Subject deletion response does not match the request"); + } + return response; +} + +/** HTTP success can confirm only the deletion fence. Every retry retains this request. */ +export async function deleteSubjektivSubject( + fetchFn: typeof fetch, + workspaceId: string, + subjectId: string, + request: SubjektivSubjectDeleteRequest, +): Promise { + identifier(request.operation_id, "operation_id"); + let response: Response; + try { + response = await fetchFn(workspaceApiPath(workspaceId, `/subjektiv/subjects/${encodeURIComponent(subjectId)}`), { + method: "DELETE", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ operation_id: request.operation_id }), + }); + } catch { + throw new SubjektivSubjectDeleteError("unknown_outcome", "The deletion outcome is unknown. Retry the same request."); + } + if (!response.ok) { + throw new SubjektivSubjectDeleteError( + response.status >= 500 ? "unknown_outcome" : "rejected", + await subjektivSubjectCreateErrorMessage(response), + response.status, + ); + } + try { + return parseSubjektivSubjectDeleteResponse( + await readBoundedJson(response, MEMORY_API_LIMITS.maxResponseBytes), + subjectId, + request.operation_id, + ); + } catch { + throw new SubjektivSubjectDeleteError("unknown_outcome", "The deletion response could not be confirmed. Retry the same request."); + } +} + export async function createSubjektivSubject( fetchFn: typeof fetch, workspaceId: string, diff --git a/web/workspace/src/lib/workspace/memory/subject-delete.ts b/web/workspace/src/lib/workspace/memory/subject-delete.ts new file mode 100644 index 000000000..6e1be6786 --- /dev/null +++ b/web/workspace/src/lib/workspace/memory/subject-delete.ts @@ -0,0 +1,51 @@ +// 削除開始後はGETから対象が消えるため、同じtabでの再読込みにも依頼IDを保持する。 +// Subject本文は保存しない。これは削除結果の正本ではなく、安全な再送用の最小入力。 +export type SubjectDeleteAttempt = { + subjectId: string; + operationId: string; +}; +export function subjectDeleteStorageKey( + workspaceId: string, + subjectId: string, +): string { + return `yoi.subject-delete:${JSON.stringify([workspaceId, subjectId])}`; +} +export function readSubjectDeleteAttempt( + storage: Pick, + workspaceId: string, + subjectId: string, +): SubjectDeleteAttempt | null { + const raw = storage.getItem(subjectDeleteStorageKey(workspaceId, subjectId)); + if (raw === null) return null; + const value = JSON.parse(raw); + if ( + !value || value.subjectId !== subjectId || + typeof value.operationId !== "string" || + value.operationId.length === 0 || + Object.keys(value).some((key) => + key !== "subjectId" && key !== "operationId" + ) + ) { + throw new Error( + "The saved deletion request cannot be read. Do not start another deletion request.", + ); + } + return value; +} +export function saveSubjectDeleteAttempt( + storage: Pick, + workspaceId: string, + attempt: SubjectDeleteAttempt, +): void { + storage.setItem( + subjectDeleteStorageKey(workspaceId, attempt.subjectId), + JSON.stringify(attempt), + ); +} +export function clearSubjectDeleteAttempt( + storage: Pick, + workspaceId: string, + subjectId: string, +): void { + storage.removeItem(subjectDeleteStorageKey(workspaceId, subjectId)); +} diff --git a/web/workspace/src/routes/w/[workspaceId]/memory/PageContent.svelte b/web/workspace/src/routes/w/[workspaceId]/memory/PageContent.svelte index da555f8a5..dd672b917 100644 --- a/web/workspace/src/routes/w/[workspaceId]/memory/PageContent.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/memory/PageContent.svelte @@ -38,7 +38,7 @@ } function subjectStateLabel(state: SubjektivSubjectResponse['state']): string { - return state === 'active' ? 'Available' : 'Retired'; + return state === 'active' ? 'Available' : 'Legacy ended Subject'; } async function openCreateForm(): Promise { diff --git a/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/PageContent.svelte b/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/PageContent.svelte index 03c8f5d76..3b3f58bcf 100644 --- a/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/PageContent.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/PageContent.svelte @@ -6,6 +6,7 @@ // 実行中Workerの履歴へ反映済みであることの証明には使わない。診断値は詳細へ分離する。 import { untrack } from 'svelte'; import DocumentMarkdown from '#lib/workspace/markdown/DocumentMarkdown.svelte'; + import SubjectDelete from '#lib/workspace/memory/SubjectDelete.svelte'; import { formatDate, workspaceRoute } from '#lib/workspace/api/http.ts'; import { SubjektivSubjectCreateError, @@ -26,6 +27,7 @@ let behaviorSaving = $state(false); let behaviorError = $state(null); let behaviorSaveConfirmed = $state(false); + let deletionStarted = $state(false); let routeGeneration = 0; let subjectWorkspaceId = untrack(() => data.workspaceId); @@ -49,6 +51,7 @@ behaviorSaving = false; behaviorError = null; behaviorSaveConfirmed = false; + deletionStarted = false; }); }); @@ -149,7 +152,7 @@ } function subjectStateLabel(state: 'active' | 'retired'): string { - return state === 'active' ? 'Available' : 'Retired'; + return state === 'active' ? 'Available' : 'Legacy ended Subject'; } @@ -167,9 +170,16 @@

{subject?.role ?? data.subjectId}

{data.subjectId} - Behavior editable + {#if !deletionStarted}Behavior editable{/if} + {#if data.workspace.permissions.delete_subjects} + {#key JSON.stringify([data.workspaceId, data.subjectId])} + { deletionStarted = true; }} /> + {/key} + {/if} + + {#if !deletionStarted} {#if subject} @@ -388,6 +398,7 @@

Current Memory data is unavailable.

{/if} + {/if}