feat: replace Subject retirement with durable explicit deletion

This commit is contained in:
2026-10-28 11:18:43 +09:00
parent 618880d38d
commit a06f6bfb3d
71 changed files with 7448 additions and 315 deletions
+58
View File
@@ -1193,8 +1193,40 @@ pub struct SubjectCreateRecoveryResponse {
pub operation_id: String,
}
/// Host requests this only after durably fencing the Subject in Backend authority.
/// Permanently exclude the Worker from creation and restoration, and confirm actual
/// execution shutdown. This does not delete Worker or Session records.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SubjectDeleteWorkerRequest {
pub workspace_id: String,
pub worker_id: WorkerId,
pub create_fingerprint: String,
pub operation_id: String,
}
/// Durable admission exclusion plus confirmed execution shutdown/absence. Errors,
/// timeouts and lost responses are unknown outcomes; retry the same full request.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SubjectDeleteWorkerResponse {
pub workspace_id: String,
pub worker_id: WorkerId,
pub create_fingerprint: String,
pub operation_id: String,
/// The canonical live Worker/Session/diagnostic aggregate is physically absent
/// under the permanent creation fence. False is not a deletion receipt.
pub originals_absent: bool,
}
#[api(reqwest, axum)]
pub trait RuntimeApi {
#[post("/v1/subject-delete-worker", status = 200, error_status = 400)]
async fn subject_delete_worker(
&self,
#[body] request: SubjectDeleteWorkerRequest,
) -> Result<SubjectDeleteWorkerResponse, RuntimeApiError>;
#[post("/v1/subject-create-recovery", status = 200, error_status = 400)]
async fn recover_subject_create(
&self,
@@ -1755,6 +1787,19 @@ mod tests {
) -> Result<SessionMaterialPinResponse, RuntimeApiError> {
Err(test_error(501))
}
async fn subject_delete_worker(
&self,
request: SubjectDeleteWorkerRequest,
) -> Result<SubjectDeleteWorkerResponse, RuntimeApiError> {
Ok(SubjectDeleteWorkerResponse {
originals_absent: true,
workspace_id: request.workspace_id,
worker_id: request.worker_id,
create_fingerprint: request.create_fingerprint,
operation_id: request.operation_id,
})
}
async fn recover_subject_create(
&self,
request: SubjectCreateRecoveryRequest,
@@ -1934,6 +1979,19 @@ mod tests {
assert_eq!(response.create_fingerprint, recovery.create_fingerprint);
assert_eq!(response.operation_id, recovery.operation_id);
let deletion = SubjectDeleteWorkerRequest {
workspace_id: "workspace-a".into(),
worker_id: WorkerId::now_v7(),
create_fingerprint: "deleted-create".into(),
operation_id: "delete-1".into(),
};
let receipt = client
.subject_delete_worker(deletion.clone())
.await
.unwrap();
let mut expected = serde_json::to_value(deletion).unwrap();
expected["originals_absent"] = true.into();
assert_eq!(serde_json::to_value(receipt).unwrap(), expected);
let summary = client.runtime_summary().await.unwrap();
assert_eq!(summary.runtime.display_name.as_deref(), Some("round-trip"));
let workers = client
+35 -6
View File
@@ -338,6 +338,8 @@ impl_openapi_schema!(
MemoryConsolidationResponse,
SubjektivSubjectCreateRequest,
SubjektivSubjectBehaviorUpdateRequest,
SubjektivSubjectDeleteRequest,
SubjektivSubjectDeleteResponse,
SubjektivSubjectListQuery,
SubjektivSubjectListResponse,
SubjektivSubjectResponse,
@@ -1527,19 +1529,21 @@ pub trait ServerApi {
#[path] memory_id: String,
#[query] query: SubjektivMemoryChangesQuery,
) -> Result<SubjektivMemoryListChangesResponse, RepositoryApiError>;
#[post(
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire",
#[delete(
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}",
status = 200,
error_status = 400,
additional_error_statuses = [401, 403, 404, 409, 500],
bearer_auth = true,
browser_auth = true,
)]
async fn subjektiv_subject_retire(
async fn subjektiv_subject_delete(
&self,
#[extension] context: ServerRequestContext,
#[path] workspace_id: String,
#[path] subject_id: String,
) -> Result<SubjektivSubjectResponse, RepositoryApiError>;
#[body] request: SubjektivSubjectDeleteRequest,
) -> Result<SubjektivSubjectDeleteResponse, RepositoryApiError>;
#[post(
"/api/w/{workspace_id}/subjektiv/staging",
status = 200,
@@ -4955,6 +4959,8 @@ pub struct WorkspacePermissionSummary {
pub manage_secrets: bool,
pub manage_runtimes: bool,
pub delete_workspace: bool,
/// Explicit Subject deletion authority for the current authenticated actor.
pub delete_subjects: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
@@ -5350,7 +5356,25 @@ pub struct MemoryConsolidationResponse {
#[serde(rename_all = "snake_case")]
pub enum SubjektivSubjectState {
Active,
Retired,
/// Saved compatibility only; not a current lifecycle operation.
#[serde(rename = "retired")]
LegacyRetired,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct SubjektivSubjectDeleteRequest {
/// Stable identity retained for every retry, including unknown outcomes.
pub operation_id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct SubjektivSubjectDeleteResponse {
pub subject_id: String,
pub operation_id: String,
/// False confirms the deletion fence, not external stop or erasure completion.
pub completed: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
@@ -10950,6 +10974,10 @@ pub fn memory_api_typescript() -> String {
export type SubjektivSubjectCreateRequest = { role: string, behavior_md?: string, };
export type SubjektivSubjectDeleteRequest = { operation_id: string, };
export type SubjektivSubjectDeleteResponse = { subject_id: string, operation_id: string, completed: boolean, };
export type SubjektivSubjectBehaviorUpdateRequest = { expected_behavior_md: string, behavior_md: string, };
export type SubjektivSubjectResponse = { id: string, role: string, behavior_md: string, state: SubjektivSubjectState, memory_fingerprint: string, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, };
@@ -13483,7 +13511,8 @@ mod tests {
"manage_repositories": true,
"manage_secrets": true,
"manage_runtimes": true,
"delete_workspace": true
"delete_workspace": true,
"delete_subjects": true
},
"extension_points": {
"store": "sqlite",
@@ -0,0 +1,49 @@
use serde_json::json;
use server_api::{
SubjektivSubjectDeleteRequest, SubjektivSubjectDeleteResponse, SubjektivSubjectState,
};
#[test]
fn subject_delete_api_replaces_retire_and_requires_operation_identity() {
let document: serde_json::Value = serde_json::from_str(
&server_api::canonical_openapi_document()
.unwrap()
.to_json()
.unwrap(),
)
.unwrap();
let paths = &document["paths"];
assert!(
paths
.get("/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire")
.is_none()
);
let operation = &paths["/api/w/{workspace_id}/subjektiv/subjects/{subject_id}"]["delete"];
assert_eq!(operation["operationId"], "subjektiv_subject_delete");
assert_eq!(operation["requestBody"]["required"], true);
assert!(serde_json::from_value::<SubjektivSubjectDeleteRequest>(json!({})).is_err());
assert!(
serde_json::from_value::<SubjektivSubjectDeleteRequest>(
json!({"operation_id":"op", "force":true})
)
.is_err()
);
for completed in [false, true] {
let response = SubjektivSubjectDeleteResponse {
subject_id: "subject".into(),
operation_id: "op".into(),
completed,
};
assert_eq!(
serde_json::to_value(response).unwrap(),
json!({"subject_id":"subject", "operation_id":"op", "completed":completed})
);
}
}
#[test]
fn old_subject_state_is_explicit_saved_compatibility_not_a_new_operation() {
let state: SubjektivSubjectState = serde_json::from_value(json!("retired")).unwrap();
assert_eq!(state, SubjektivSubjectState::LegacyRetired);
assert_eq!(serde_json::to_value(state).unwrap(), json!("retired"));
}
+46
View File
@@ -12,3 +12,49 @@ Hosts retain body ownership across inactivity, reserve actual execution quiescen
- [`api`]: shared Host operations and bounded responses
- [`job`]: consolidation selection rules and result validation
- [`journal`]: individually committed activity operations, source coverage, retry receipts, and bounded reading
## Explicit Subject deletion
`begin_subject_deletion(subject_id, operation_id)` commits a permanent content-free
fence in this store **before** the Host stops execution or erases external content.
The operation ID is unique within this storage scope. Exact retries return current
saved state; a different key for that Subject or reuse for another Subject conflicts.
Pending fences survive restart and a failed purge. Normal reads, writes, saved-receipt
replay, cached Host/Journal contexts and authority lookup cannot use a fenced Subject.
Subject discovery hides fenced records before pagination. SQL INSERT/UPDATE guards
cover direct and indirect owners, checking both OLD and NEW ownership on UPDATE.
Deleting a grant never restores non-Subject fallback: content-free Worker identity
fences remain, and the Host must consult them before fallback authorization.
The Host uses `subject_deletion` / `pending_subject_deletions` for recovery and
`deletion_subject_workers` / `deletion_subject_sessions` / `deletion_subject_jobs`
only as cleanup inventories, not authority or model-visible content. It must fence
creation reservations, quiesce Workers and in-flight delivery, purge Job inputs/results
and live Job execution copies, and clear resident/derived caches. Source Worker/Session
originals and pre-existing archives have independent retention lifecycles: the Workspace
Host archives source originals rather than erasing them. Those archives may contain
previously injected Memory; Workdirs, Tickets, exports and external provider copies are
also outside Subject content erasure. These boundaries must be explained to the user.
The domain rechecks after out-of-DB capture callbacks and before returning Job read
responses; already returned DTOs cannot be revoked by SQLite. Host lifecycle locking
must cover delivery/reinjection of previously fetched content as well as finish.
Only after external cleanup succeeds may the Host call
`finish_subject_deletion(subject_id, operation_id)`. One transaction deletes Subject,
Memory/history, staging/evidence, Surface snapshots/runs, permission sources/receipts,
child grants, writer/session bindings, Journal content/generations/deliveries/receipts,
links, and the Subject's `legacy_record_archive` rows. A purge error rolls back all
those deletions without undoing the separately committed fence. Ordinary history
retention and explicit Journal Session-erasure rules are unchanged for other Subjects.
Retained records contain only deletion operation/time/completion and execution/session
identities needed to prevent reuse; existing content-free Session-erasure markers and
workspace-only migration metadata remain. Other Subjects and storage scopes are untouched.
The deletion migration adds fences without erasing or reactivating existing data. Old `retired`
JSON/SQL records deserialize as `SubjectState::LegacyRetired` and remain readable
until explicitly deleted. There is no current retirement API. Normal Worker deletion
still does not implicitly delete Subject memory.
Completion means logical erasure from domain and Host-managed storage, **not** secure
physical overwriting: SQLite free pages, WAL files, filesystem snapshots, backups and
external copies can retain bytes under their own retention/erasure policies.
+1
View File
@@ -116,6 +116,7 @@ impl HostOperationContext {
"Host operation context belongs to another storage scope".into(),
));
}
Self::require_subject(store, &self.subject_id)?;
Ok(())
}
+42
View File
@@ -452,3 +452,45 @@ fn query_content_condition_ignores_behavior_and_detects_aba_memory_writes() {
Err(OperationError::Conflict(_))
));
}
#[test]
fn subject_deletion_revalidates_cached_body_empty_batch_and_receipt_contexts() {
let temp = tempfile::tempdir().unwrap();
let (_manager, store) = open(temp.path(), "scope");
let subject = store
.create_subject(SubjectRole::new("companion").unwrap())
.unwrap();
let body = HostOperationContext::validated_body(&store, &subject.id, None).unwrap();
let empty =
HostOperationContext::validated_consolidation(&store, &subject.id, Some(vec![]), None)
.unwrap();
store.begin_subject_deletion(&subject.id, "delete").unwrap();
for context in [&body, &empty] {
for operation in [
query(None),
Op::ListCandidates(server_api::SubjektivMemoryCandidateListRequest { limit: None }),
Op::ReceiptStatus(server_api::SubjektivMemoryReceiptStatusRequest {
receipt_id: "receipt".into(),
}),
Op::ReadCandidate(server_api::SubjektivMemoryCandidateReadRequest {
candidate_id: "candidate".into(),
}),
] {
assert!(matches!(
execute(&store, context, operation),
Err(OperationError::SubjectNotFound(_))
));
}
}
store
.finish_subject_deletion(&subject.id, "delete")
.unwrap();
assert!(
execute(
&store,
&empty,
Op::ListCandidates(server_api::SubjektivMemoryCandidateListRequest { limit: None })
)
.is_err()
);
}
+419
View File
@@ -0,0 +1,419 @@
//! Explicit Subject erasure is a two-phase Host operation. Beginning commits a permanent,
//! content-free fence before the Host stops/removes Workers and purges Jobs and execution copies.
//! Source Worker/Session originals remain under their separate archive lifecycle.
//! Failure leaves that fence and the cleanup inventory intact; the Host retries
//! the same operation ID. Finishing asserts that external cleanup succeeded and atomically erases
//! this store's content. This store cannot verify external Runtime/Job/Session cleanup itself.
//!
//! Tombstones contain no behavior, permission vectors, Memory, Journal or Session text. Worker
//! identities remain permanently fenced so removal of a grant cannot become non-Subject fallback
//! authority. Existing Journal Session erasure markers are retained to reject delayed admission;
//! they are content-free and are not a second copy of the deleted material.
use super::*;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SubjectDeletion {
pub subject_id: String,
pub operation_id: String,
pub completed: bool,
pub started_at: String,
}
/// Host-only cleanup identity, never Job input, output, profile or authorization material.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DeletionSubjectJob {
pub job_id: String,
pub attempt_id: String,
}
// All content tables are enumerated deliberately. Indirect tables are erased before their
// owner rows, both for FK checks and so their fencing predicates remain resolvable.
const INDIRECT: &[(&str, &str)] = &[
(
"journal_generation_input_deliveries",
"generation_id IN (SELECT g.id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id={subject})",
),
(
"legacy_record_archive",
"json_extract(row_json, '$.subject_id')={subject}",
),
(
"journal_generation_deliveries",
"generation_id IN (SELECT g.id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id={subject})",
),
(
"journal_operations",
"journal_id IN (SELECT id FROM journals WHERE subject_id={subject})",
),
(
"journal_generations",
"journal_id IN (SELECT id FROM journals WHERE subject_id={subject})",
),
(
"journal_sources",
"journal_id IN (SELECT id FROM journals WHERE subject_id={subject})",
),
(
"journal_session_bindings",
"binding_id IN (SELECT binding_id FROM journal_writer_bindings WHERE subject_id={subject})",
),
];
const DIRECT: &[&str] = &[
"memory_journal_links",
"journal_corrections",
"journal_entries",
"journals",
"journal_writer_bindings",
"subject_session_attributions",
"worker_execution_grants",
"subject_permission_receipts",
"subject_permissions",
"subject_permission_changes",
"candidate_decision_receipts",
"surface_generation_state",
"surface_generation_runs",
"surface_snapshot_refs",
"surface_snapshot_seals",
"surface_snapshots",
"staging_resolution_targets",
"staging_resolution_seals",
"staging_resolutions",
"memory_change_candidates",
"memory_change_derivations",
"memory_change_seals",
"memory_changes",
"memory_records",
"staging_records",
"subjects",
];
fn owner_expression(table: &str, row: &str) -> String {
match table {
"legacy_record_archive" => format!("json_extract({row}.row_json, '$.subject_id')"),
"journal_generation_input_deliveries" | "journal_generation_deliveries" => format!(
"(SELECT j.subject_id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE g.id={row}.generation_id)"
),
"journal_operations" | "journal_generations" | "journal_sources" => {
format!("(SELECT subject_id FROM journals WHERE id={row}.journal_id)")
}
"journal_session_bindings" => format!(
"(SELECT subject_id FROM journal_writer_bindings WHERE binding_id={row}.binding_id)"
),
_ => format!("{row}.subject_id"),
}
}
pub(super) fn migrate(tx: &Transaction<'_>) -> feature_storage::Result<()> {
tx.execute_batch(
"CREATE TABLE subject_deletions (
subject_id TEXT PRIMARY KEY,
operation_id TEXT NOT NULL UNIQUE,
completed INTEGER NOT NULL CHECK(completed IN (0,1)),
started_at TEXT NOT NULL
);
CREATE TABLE subject_deleted_workers (
subject_id TEXT NOT NULL REFERENCES subject_deletions(subject_id) ON DELETE RESTRICT,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
PRIMARY KEY(subject_id,runtime_id,worker_id)
);
CREATE TABLE subject_deleted_sessions (
subject_id TEXT NOT NULL REFERENCES subject_deletions(subject_id) ON DELETE RESTRICT,
session_id TEXT NOT NULL UNIQUE,
PRIMARY KEY(subject_id,session_id)
);
CREATE TRIGGER subject_deleted_sessions_retained BEFORE DELETE ON subject_deleted_sessions
BEGIN SELECT RAISE(ABORT, 'deleted Session fences are permanent'); END;
CREATE TRIGGER subject_deleted_sessions_immutable BEFORE UPDATE ON subject_deleted_sessions
BEGIN SELECT RAISE(ABORT, 'deleted Session fences are immutable'); END;
CREATE INDEX subject_deleted_worker_identity ON subject_deleted_workers(runtime_id,worker_id);
CREATE TRIGGER subject_deletions_retained BEFORE DELETE ON subject_deletions
BEGIN SELECT RAISE(ABORT, 'Subject deletion fences are permanent'); END;
CREATE TRIGGER subject_deletions_identity BEFORE UPDATE ON subject_deletions
WHEN OLD.subject_id<>NEW.subject_id OR OLD.operation_id<>NEW.operation_id
OR OLD.started_at<>NEW.started_at OR OLD.completed>NEW.completed
BEGIN SELECT RAISE(ABORT, 'Subject deletion identity is permanent'); END;
CREATE TRIGGER subject_deleted_workers_retained BEFORE DELETE ON subject_deleted_workers
BEGIN SELECT RAISE(ABORT, 'deleted Worker fences are permanent'); END;
CREATE TRIGGER subject_deleted_workers_immutable BEFORE UPDATE ON subject_deleted_workers
BEGIN SELECT RAISE(ABORT, 'deleted Worker fences are immutable'); END;",
)?;
// Preserve each existing history-retention condition, adding only the explicit Subject
// erasure exception. Frozen earlier migrations are not edited. Normal history deletion
// (including another Subject's history) remains forbidden.
let triggers = tx
.prepare("SELECT name,tbl_name,sql FROM sqlite_master WHERE type='trigger'")?
.query_map([], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
))
})?
.collect::<std::result::Result<Vec<_>, _>>()?;
for (name, table, sql) in triggers {
if !DIRECT.contains(&table.as_str()) && !INDIRECT.iter().any(|(t, _)| *t == table) {
continue;
}
let upper = sql.to_ascii_uppercase();
if !upper.contains("BEFORE DELETE") {
continue;
}
let begin = upper.find("BEGIN").expect("trigger body");
let (header, body) = sql.split_at(begin);
let exception = format!(
"NOT EXISTS(SELECT 1 FROM subject_deletions WHERE subject_id={})",
owner_expression(&table, "OLD")
);
let header = if let Some(when) = header.to_ascii_uppercase().find("WHEN") {
format!(
"{} WHEN ({}) AND {exception} ",
&header[..when],
header[when + 4..].trim()
)
} else {
format!("{header} WHEN {exception} ")
};
tx.execute_batch(&format!("DROP TRIGGER \"{name}\"; {header}{body}"))?;
}
for table in DIRECT
.iter()
.copied()
.chain(INDIRECT.iter().map(|(t, _)| *t))
.chain(["journal_erased_sessions"])
{
for action in ["INSERT", "UPDATE"] {
let new_owner = owner_expression(table, "NEW");
let predicate = if action == "UPDATE" {
format!(
"subject_id={new_owner} OR subject_id={}",
owner_expression(table, "OLD")
)
} else {
format!("subject_id={new_owner}")
};
tx.execute_batch(&format!(
"CREATE TRIGGER deletion_fence_{table}_{action} BEFORE {action} ON {table}
WHEN EXISTS(SELECT 1 FROM subject_deletions WHERE {predicate})
BEGIN SELECT RAISE(ABORT, 'Subject deletion has begun'); END;"
))?;
}
}
// Identity reuse must not reconnect purged content or turn a deleted child into
// an unscoped Worker. Check both old and new on ownership-changing UPDATEs.
for table in [
"subject_session_attributions",
"journal_writer_bindings",
"worker_execution_grants",
] {
for action in ["INSERT", "UPDATE"] {
let rows = if action == "UPDATE" {
vec!["OLD", "NEW"]
} else {
vec!["NEW"]
};
let mut predicates = Vec::new();
for row in rows {
predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id={row}.runtime_id AND worker_id={row}.worker_id)"));
if table == "subject_session_attributions" {
predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_sessions WHERE session_id={row}.session_id)"));
}
if table == "worker_execution_grants" {
predicates.push(format!("EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=json_extract({row}.record_json,'$.origin_runtime_id') AND worker_id=json_extract({row}.record_json,'$.origin_worker_id'))"));
}
}
tx.execute_batch(&format!("CREATE TRIGGER deletion_identity_{table}_{action} BEFORE {action} ON {table} WHEN {} BEGIN SELECT RAISE(ABORT, 'deleted Subject execution identity'); END;", predicates.join(" OR ")))?;
}
}
Ok(())
}
pub(crate) fn is_deleting(connection: &Connection, subject: &str) -> Result<bool> {
// Older migration fixtures deliberately use current Rust reads before upgrade. Production
// opens only registered current stores and must fail closed if the fence table is absent.
#[cfg(test)]
if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deletions' AND type='table')", [], |r|r.get::<_,bool>(0))? {
return Ok(false);
}
Ok(connection.query_row(
"SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE subject_id=?1)",
[subject],
|r| r.get::<_, bool>(0),
)?)
}
pub(crate) fn require_not_deleting(connection: &Connection, subject: &str) -> Result<()> {
if is_deleting(connection, subject)? {
return Err(SubjektivError::SubjectNotFound(subject.into()));
}
Ok(())
}
pub(crate) fn require_live_worker(
connection: &Connection,
runtime: &str,
worker: &str,
) -> Result<()> {
#[cfg(test)]
if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deleted_workers' AND type='table')", [], |r|r.get::<_,bool>(0))? { return Ok(()); }
if connection.query_row(
"SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)",
params![runtime, worker],
|r| r.get::<_, bool>(0),
)? {
return Err(SubjektivError::AuthorizationDenied(
"Worker belongs to a deleted Subject".into(),
));
}
Ok(())
}
pub(crate) fn deleted_session_subject(
connection: &Connection,
session: &str,
) -> Result<Option<String>> {
#[cfg(test)]
if !connection.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deleted_sessions' AND type='table')", [], |r|r.get::<_,bool>(0))? { return Ok(None); }
Ok(connection
.query_row(
"SELECT subject_id FROM subject_deleted_sessions WHERE session_id=?1",
[session],
|r| r.get(0),
)
.optional()?)
}
fn deletion(connection: &Connection, subject: &str) -> Result<Option<SubjectDeletion>> {
Ok(connection.query_row("SELECT subject_id,operation_id,completed,started_at FROM subject_deletions WHERE subject_id=?1", [subject], |r|Ok(SubjectDeletion { subject_id:r.get(0)?, operation_id:r.get(1)?, completed:r.get(2)?, started_at:r.get(3)? })).optional()?)
}
fn require_deletion(connection: &Connection, subject: &str) -> Result<SubjectDeletion> {
deletion(connection, subject)?.ok_or_else(|| SubjektivError::SubjectNotFound(subject.into()))
}
impl SubjektivStore {
/// Commits the read/write/authority fence, even for legacy-retired Subjects. Operation IDs
/// are unique within this store's scope; neither another key nor another Subject may reuse
/// a deletion. Exact retries return the saved current result, including completed deletions.
pub fn begin_subject_deletion(
&self,
subject: &str,
operation: &str,
) -> Result<SubjectDeletion> {
validate_label("subject id", subject)?;
validate_label("deletion operation id", operation)?;
self.database.try_transaction(|tx| {
if let Some(saved) = deletion(tx,subject)? {
return if saved.operation_id == operation { Ok(saved) } else { Err(SubjektivError::SubjectDeletionConflict(operation.into())) };
}
if tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE operation_id=?1)",[operation],|r|r.get::<_,bool>(0))? {
return Err(SubjektivError::SubjectDeletionConflict(operation.into()));
}
require_subject(tx,subject)?;
tx.execute("INSERT INTO subject_deletions(subject_id,operation_id,completed,started_at) VALUES(?1,?2,0,?3)",params![subject,operation,now()])?;
// Capture bodies, historical attribution, delegated children, and grant origins.
// This transaction also fences delayed grants/attributions, so no new identity can
// appear after the inventory is frozen. Runtime create reservations remain Host-owned.
tx.execute("INSERT INTO subject_deleted_workers(subject_id,runtime_id,worker_id)
SELECT subject_id,runtime_id,worker_id FROM subject_session_attributions WHERE subject_id=?1
UNION SELECT subject_id,runtime_id,worker_id FROM journal_writer_bindings WHERE subject_id=?1
UNION SELECT subject_id,runtime_id,worker_id FROM worker_execution_grants WHERE subject_id=?1
UNION SELECT subject_id,json_extract(record_json,'$.origin_runtime_id'),json_extract(record_json,'$.origin_worker_id') FROM worker_execution_grants WHERE subject_id=?1",[subject])?;
tx.execute("INSERT INTO subject_deleted_sessions(subject_id,session_id)
SELECT subject_id,session_id FROM subject_session_attributions WHERE subject_id=?1
UNION SELECT subject_id,source_session_id FROM journals WHERE subject_id=?1
UNION SELECT subject_id,session_id FROM journal_erased_sessions WHERE subject_id=?1", [subject])?;
require_deletion(tx,subject)
})
}
/// Host-only, content-free cleanup inventory. Usable after begin, including exact finish
/// retries. Tombstones survive finish to prevent non-Subject permission fallback.
pub fn deletion_subject_workers(&self, subject: &str) -> Result<Vec<(String, String)>> {
self.database.try_with_connection(|tx| {
require_deletion(tx,subject)?;
Ok(tx.prepare("SELECT runtime_id,worker_id FROM subject_deleted_workers WHERE subject_id=?1 ORDER BY runtime_id,worker_id")?.query_map([subject],|r|Ok((r.get(0)?,r.get(1)?)))?.collect::<std::result::Result<Vec<_>,_>>()?)
})
}
/// Host-only Session identities; retained source rows are never exposed through this path.
/// The Host must erase originals/archives before finish removes attribution.
pub fn deletion_subject_sessions(&self, subject: &str) -> Result<Vec<String>> {
self.database.try_with_connection(|tx| {
require_deletion(tx,subject)?;
Ok(tx.prepare("SELECT session_id FROM subject_deleted_sessions WHERE subject_id=?1 ORDER BY session_id")?.query_map([subject],|r|r.get(0))?.collect::<std::result::Result<Vec<_>,_>>()?)
})
}
/// Host-only Job cleanup inventory. Job storage owns inputs, outcomes and Worker archives;
/// this store retains only these identities until Host cleanup has succeeded.
pub fn deletion_subject_jobs(&self, subject: &str) -> Result<Vec<DeletionSubjectJob>> {
self.database.try_with_connection(|tx| {
require_deletion(tx,subject)?;
Ok(tx.prepare("SELECT job_id,attempt_id FROM surface_generation_runs WHERE subject_id=?1 AND job_id IS NOT NULL AND attempt_id IS NOT NULL UNION SELECT g.job_id,g.attempt_id FROM journal_generations g JOIN journals j ON j.id=g.journal_id WHERE j.subject_id=?1 ORDER BY 1,2")?.query_map([subject],|r|Ok(DeletionSubjectJob {job_id:r.get(0)?,attempt_id:r.get(1)?}))?.collect::<std::result::Result<Vec<_>,_>>()?)
})
}
/// Host-only recovery state; this never exposes Subject content or authority.
pub fn subject_deletion(&self, subject: &str) -> Result<Option<SubjectDeletion>> {
self.database
.try_with_connection(|tx| deletion(tx, subject))
}
/// Durable unfinished operations, including after Host restart. Completed tombstones
/// remain queryable individually but are not returned as pending work.
pub fn pending_subject_deletions(&self) -> Result<Vec<SubjectDeletion>> {
self.database.try_with_connection(|tx| {
Ok(tx.prepare("SELECT subject_id,operation_id,completed,started_at FROM subject_deletions WHERE completed=0 ORDER BY started_at,subject_id")?
.query_map([], |r| Ok(SubjectDeletion { subject_id:r.get(0)?, operation_id:r.get(1)?, completed:r.get(2)?, started_at:r.get(3)? }))?
.collect::<std::result::Result<Vec<_>,_>>()?)
})
}
pub fn worker_subject_deleted(&self, runtime: &str, worker: &str) -> Result<bool> {
self.database.try_with_connection(|tx| Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)",params![runtime,worker],|r|r.get(0))?))
}
/// Host confirmation that external cleanup is complete. All Subject content and completed
/// receipt commit together. Deferred FKs allow cyclic current/history references to disappear
/// without weakening FK enforcement outside this transaction. Failure rolls back all erasure.
pub fn finish_subject_deletion(
&self,
subject: &str,
operation: &str,
) -> Result<SubjectDeletion> {
self.database.try_transaction(|tx| {
let saved = require_deletion(tx, subject)?;
if saved.operation_id != operation {
return Err(SubjektivError::SubjectDeletionConflict(operation.into()));
}
if saved.completed {
return Ok(saved);
}
tx.execute_batch("PRAGMA defer_foreign_keys=ON;")?;
for (table, predicate) in INDIRECT {
tx.execute(
&format!(
"DELETE FROM {table} WHERE {}",
predicate.replace("{subject}", "?1")
),
[subject],
)?;
}
for table in DIRECT {
tx.execute(
&format!("DELETE FROM {table} WHERE subject_id=?1"),
[subject],
)?;
}
tx.execute(
"UPDATE subject_deletions SET completed=1 WHERE subject_id=?1",
[subject],
)?;
require_deletion(tx, subject)
})
}
}
#[cfg(test)]
mod tests;
+489
View File
@@ -0,0 +1,489 @@
use super::*;
use crate::tests::{candidate, legacy_retired_fixture, open_store, role};
use feature_storage::FeatureStorage;
use memory::extract::CandidateKind;
fn draft() -> MemoryDraft {
MemoryDraft::active(
CandidateKind::Lesson,
"private claim",
"private body",
"useful",
"initial",
)
}
fn count(store: &SubjektivStore, table: &str) -> i64 {
store
.database
.try_with_connection::<_, SubjektivError>(|c| {
Ok(c.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0))?)
})
.unwrap()
}
#[test]
fn fence_only_denies_content_reads_writes_and_exact_replay_before_purge() {
let temp = tempfile::tempdir().unwrap();
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
let subject = store
.create_subject_with_behavior(role(), "private behavior".into())
.unwrap();
let id = &subject.id;
let attribution = SubjectSessionAttribution::new(id, "runtime", "worker", "session").unwrap();
store
.record_session_attribution(attribution.clone())
.unwrap();
let staged = candidate(id, "candidate", store.workspace_id());
store.stage_candidate(staged.clone()).unwrap();
let request = CandidateDecisionRequest {
request_id: "decision".into(),
candidate_id: "candidate".into(),
reason: "adopt".into(),
decision: CandidateDecision::Apply {
target: MemoryChangeTarget::Create,
draft: draft(),
},
};
let memory = store
.decide_candidate(id, request.clone())
.unwrap()
.memory
.unwrap();
let generation = store
.prepare_job_surface_generation(id, Some(("job", "attempt")))
.unwrap();
let snapshot = store
.publish_surface_generation(
id,
&generation.id,
vec![SurfacePoint {
body_md: "private surface".into(),
memory_refs: vec![MemoryChangeRef {
memory_id: memory.id.clone(),
change_id: memory.change_id.clone(),
}],
}],
)
.unwrap();
let fence = store.begin_subject_deletion(id, "delete").unwrap();
assert!(!fence.completed);
// The fence must work while every body remains physically present.
assert_eq!(count(&store, "subjects"), 1);
assert_eq!(count(&store, "memory_records"), 1);
assert_eq!(count(&store, "candidate_decision_receipts"), 1);
assert!(store.subject(id).unwrap().is_none());
assert!(store.list_subjects(1).unwrap().items.is_empty());
macro_rules! denied { ($($result:expr),+ $(,)?) => { $(assert!($result.is_err(), stringify!($result));)+ }; }
denied!(
store.session_attribution("session"),
store.subject_session_attributions(id),
store.subject_session_attribution_page(id, None, "2999", None, 1),
store.record_session_attribution(attribution),
store.staging_candidate(id, "candidate"),
store.pending_staging_candidates(id, 1),
store.pending_staging_backlog(id),
store.staging_resolution(id, "candidate"),
store.memory(id, &memory.id),
store.memory_change(id, &memory.id, memory.change_id.clone()),
store.scoped_memory(id, &memory.id),
store.scoped_memory_change(id, &memory.id, memory.change_id.clone()),
store.list_memories(id),
store.list_memory_changes(id, &memory.id),
store.memory_fingerprint(id),
store.surface_snapshot(id, &snapshot.id),
store.resident_surface(id),
store.resident_context(id),
store.require_job_surface_generation(id, &generation.id, "job", "attempt"),
store.validate_job_surface_outcome(
id,
&generation.id,
generation.memory_fingerprint,
"ready",
Some(&snapshot.id),
None,
"job",
"attempt"
),
store.prepare_surface_generation(id),
store.publish_surface_generation(id, &generation.id, vec![]),
store.fail_surface_generation(id, &generation.id, "failed"),
store.update_subject_behavior(id, &subject.behavior_md, "late behavior".into()),
store.stage_candidate(staged),
store.decide_candidate(id, request),
store.create_memory(id, draft()),
store.subject_permissions(id),
store.list_subject_worker_execution_grants(id),
);
assert_eq!(
store.deletion_subject_workers(id).unwrap(),
vec![("runtime".into(), "worker".into())]
);
assert_eq!(
store.deletion_subject_sessions(id).unwrap(),
vec!["session"]
);
assert_eq!(store.deletion_subject_jobs(id).unwrap()[0].job_id, "job");
assert!(store.worker_subject_deleted("runtime", "worker").unwrap());
assert!(
store
.finish_subject_deletion(id, "delete")
.unwrap()
.completed
);
for table in DIRECT {
assert_eq!(count(&store, table), 0, "{table}");
}
assert!(store.worker_execution_grant("runtime", "worker").is_err());
assert_eq!(
store.deletion_subject_sessions(id).unwrap(),
vec!["session"]
);
assert_eq!(
store.finish_subject_deletion(id, "delete").unwrap(),
store.begin_subject_deletion(id, "delete").unwrap()
);
}
#[test]
fn purge_failure_rolls_back_content_but_not_the_committed_fence_and_restart_recovers() {
let temp = tempfile::tempdir().unwrap();
let id: String;
let pending;
{
let (manager, scope, store) = open_store(temp.path(), "workspace-a");
id = store.create_subject(role()).unwrap().id;
store.create_memory(&id, draft()).unwrap();
pending = store.begin_subject_deletion(&id, "delete").unwrap();
store.database.try_with_connection::<_, SubjektivError>(|c| {
c.execute_batch("CREATE TRIGGER injected_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT, 'simulated purge failure'); END;")?;
Ok(())
}).unwrap();
assert!(store.finish_subject_deletion(&id, "delete").is_err());
assert_eq!(count(&store, "memory_records"), 1);
assert_eq!(count(&store, "memory_changes"), 1);
assert_eq!(store.subject_deletion(&id).unwrap(), Some(pending.clone()));
drop(store);
drop(scope);
manager.shutdown().unwrap();
}
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
assert_eq!(
store.pending_subject_deletions().unwrap(),
vec![pending.clone()]
);
assert_eq!(
store.begin_subject_deletion(&id, "delete").unwrap(),
pending
);
assert!(store.list_memories(&id).is_err());
store
.database
.try_with_connection::<_, SubjektivError>(|c| {
c.execute_batch("DROP TRIGGER injected_failure")?;
Ok(())
})
.unwrap();
assert!(
store
.finish_subject_deletion(&id, "delete")
.unwrap()
.completed
);
assert!(store.pending_subject_deletions().unwrap().is_empty());
}
#[test]
fn deletion_identity_conflicts_are_scope_wide_but_do_not_cross_workspaces() {
let temp = tempfile::tempdir().unwrap();
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
let a = store.create_subject(role()).unwrap().id;
let b = store.create_subject(role()).unwrap().id;
let saved = store.begin_subject_deletion(&a, "same-key").unwrap();
assert_eq!(store.begin_subject_deletion(&a, "same-key").unwrap(), saved);
for result in [
store.begin_subject_deletion(&a, "different"),
store.begin_subject_deletion(&b, "same-key"),
store.finish_subject_deletion(&a, "different"),
] {
assert!(matches!(
result,
Err(SubjektivError::SubjectDeletionConflict(_))
));
}
assert!(store.begin_subject_deletion("absent", "new-key").is_err());
assert!(store.finish_subject_deletion(&b, "same-key").is_err());
let page = store.list_subjects(1).unwrap();
assert_eq!(page.items[0].id, b);
assert!(!page.has_more);
store.finish_subject_deletion(&a, "same-key").unwrap();
store.create_memory(&b, draft()).unwrap();
let (_other_manager, _other_scope, other) = open_store(temp.path(), "workspace-b");
assert!(other.subject_deletion(&a).unwrap().is_none());
assert!(other.begin_subject_deletion(&b, "same-key").is_err());
let c = other.create_subject(role()).unwrap().id;
other.begin_subject_deletion(&c, "same-key").unwrap();
assert_eq!(store.list_memories(&b).unwrap().len(), 1);
let fresh = store.create_subject(role()).unwrap().id;
assert!(store.list_memories(&fresh).unwrap().is_empty());
assert!(store.subject_permissions(&fresh).unwrap().is_none());
}
#[test]
fn concurrent_begin_and_write_serialize_and_exact_finish_is_replayable() {
let temp = tempfile::tempdir().unwrap();
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
let id = store.create_subject(role()).unwrap().id;
let barrier = std::sync::Barrier::new(3);
std::thread::scope(|scope| {
let a = scope.spawn(|| {
barrier.wait();
store.begin_subject_deletion(&id, "delete")
});
let b = scope.spawn(|| {
barrier.wait();
store.create_memory(&id, draft())
});
barrier.wait();
assert!(!a.join().unwrap().unwrap().completed);
// A write either precedes begin and is later purged, or loses to the fence.
let _ = b.join().unwrap();
});
assert!(store.create_memory(&id, draft()).is_err());
assert_eq!(store.pending_subject_deletions().unwrap().len(), 1);
store.finish_subject_deletion(&id, "delete").unwrap();
assert_eq!(count(&store, "memory_records"), 0);
}
#[test]
fn every_subject_owned_table_has_old_and_new_sql_fences_and_retention_is_scoped() {
let temp = tempfile::tempdir().unwrap();
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
let a = store.create_subject(role()).unwrap().id;
let b = store.create_subject(role()).unwrap().id;
store
.stage_candidate(candidate(&a, "a", store.workspace_id()))
.unwrap();
store
.stage_candidate(candidate(&b, "b", store.workspace_id()))
.unwrap();
store.begin_subject_deletion(&a, "delete").unwrap();
store.database.try_with_connection::<_, SubjektivError>(|c| {
let tables = c.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")?.query_map([], |r| r.get::<_,String>(0))?.collect::<std::result::Result<Vec<_>,_>>()?;
for table in tables {
let columns = c.prepare(&format!("PRAGMA table_info({table})"))?.query_map([], |r| r.get::<_,String>(1))?.collect::<std::result::Result<Vec<_>,_>>()?;
if !columns.iter().any(|v| v == "subject_id") || table.starts_with("subject_delet") { continue; }
assert!(DIRECT.contains(&table.as_str()) || table == "journal_erased_sessions", "unclassified Subject table: {table}");
}
for table in DIRECT.iter().copied().chain(INDIRECT.iter().map(|v| v.0)).chain(["journal_erased_sessions"]) {
let sql: String = c.query_row("SELECT sql FROM sqlite_master WHERE name=?1", [format!("deletion_fence_{table}_UPDATE")], |r| r.get(0))?;
assert!(sql.contains("OLD.") && sql.contains("NEW."), "{table}: {sql}");
}
// OLD fenced -> NEW unfenced must not evade a guard.
assert!(c.execute("UPDATE subjects SET subject_id=?2 WHERE subject_id=?1", params![a, "new-id"]).is_err());
assert!(c.execute("UPDATE subjects SET subject_id=?2 WHERE subject_id=?1", params![b, a]).is_err());
assert!(c.execute("DELETE FROM staging_records WHERE subject_id=?1", [&b]).is_err());
assert!(c.execute("DELETE FROM subject_deletions WHERE subject_id=?1", [&a]).is_err());
assert!(c.execute("UPDATE subject_deletions SET operation_id='changed' WHERE subject_id=?1", [&a]).is_err());
Ok(())
}).unwrap();
store.finish_subject_deletion(&a, "delete").unwrap();
assert!(store.staging_candidate(&b, "b").unwrap().is_some());
}
#[test]
fn legacy_retired_is_readable_serde_compatible_and_explicitly_deletable() {
let temp = tempfile::tempdir().unwrap();
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
let id = store.create_subject(role()).unwrap().id;
store.create_memory(&id, draft()).unwrap();
legacy_retired_fixture(&store, &id).unwrap();
assert_eq!(
serde_json::from_str::<SubjectState>("\"retired\"").unwrap(),
SubjectState::LegacyRetired
);
assert_eq!(
serde_json::to_string(&SubjectState::LegacyRetired).unwrap(),
"\"retired\""
);
assert_eq!(store.list_memories(&id).unwrap().len(), 1);
assert!(store.create_memory(&id, draft()).is_err());
store.begin_subject_deletion(&id, "delete-legacy").unwrap();
store.finish_subject_deletion(&id, "delete-legacy").unwrap();
assert_eq!(count(&store, "memory_changes"), 0);
}
#[test]
fn migration_preserves_legacy_content_until_explicit_delete_including_archive() {
let temp = tempfile::tempdir().unwrap();
let id: String;
{
let manager = FeatureStorage::new(temp.path());
let scope = manager.workspace("workspace-a").unwrap();
let registration = scope
.register(FeatureRegistration::new(
SUBJEKTIV_FEATURE_ID,
&MIGRATIONS[..6],
))
.unwrap();
let store = SubjektivStore::open(&scope, &registration).unwrap();
id = "legacy-subject".into();
store.database.try_transaction::<_, SubjektivError>(|tx| {
let raw = serde_json::json!({"schema_version":1,"id":id,"role":"workspace_companion","behavior_md":"legacy private behavior","state":"active","created_at":"2026-01-01","updated_at":"2026-01-01"}).to_string();
tx.execute("INSERT INTO subjects(subject_id,role,state,store_revision,record_json,created_at,updated_at) VALUES(?1,'workspace_companion','active',0,?2,'2026-01-01','2026-01-01')", params![id,raw])?;
Ok(())
}).unwrap();
// Build a saved SQL fixture rather than using current-only APIs against an old schema.
store
.database
.try_transaction::<_, SubjektivError>(|tx| {
let raw: String = tx.query_row(
"SELECT record_json FROM subjects WHERE subject_id=?1",
[&id],
|r| r.get(0),
)?;
let mut value: serde_json::Value = serde_json::from_str(&raw)?;
value["state"] = "retired".into();
tx.execute(
"UPDATE subjects SET state='retired',record_json=?2 WHERE subject_id=?1",
params![id, value.to_string()],
)?;
Ok(())
})
.unwrap();
drop(store);
drop(scope);
manager.shutdown().unwrap();
}
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
assert_eq!(
store.subject(&id).unwrap().unwrap().state,
SubjectState::LegacyRetired
);
assert!(count(&store, "legacy_record_archive") > 1);
store.begin_subject_deletion(&id, "delete").unwrap();
store.finish_subject_deletion(&id, "delete").unwrap();
store.database.try_with_connection::<_, SubjektivError>(|c| {
let remaining: i64 = c.query_row("SELECT COUNT(*) FROM legacy_record_archive WHERE json_extract(row_json,'$.subject_id')=?1", [&id], |r|r.get(0))?;
assert_eq!(remaining, 0);
// Workspace-only metadata remains and is not a copy of the Subject's content.
assert_eq!(c.query_row("SELECT COUNT(*) FROM legacy_record_archive WHERE source_table='store_scope'", [], |r|r.get::<_,i64>(0))?, 1);
Ok(())
}).unwrap();
}
#[test]
fn concurrent_deletions_on_independent_connections_save_one_request_identity() {
let temp = tempfile::tempdir().unwrap();
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
let (_second_manager, _second_scope, second) = open_store(temp.path(), "workspace-a");
let id = store.create_subject(role()).unwrap().id;
let barrier = std::sync::Barrier::new(3);
std::thread::scope(|scope| {
let a = scope.spawn(|| {
barrier.wait();
store.begin_subject_deletion(&id, "first")
});
let b = scope.spawn(|| {
barrier.wait();
second.begin_subject_deletion(&id, "second")
});
barrier.wait();
let results = [a.join().unwrap(), b.join().unwrap()];
assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
assert_eq!(
results
.iter()
.filter(|r| matches!(r, Err(SubjektivError::SubjectDeletionConflict(_))))
.count(),
1
);
});
let saved = second.subject_deletion(&id).unwrap().unwrap();
assert_eq!(
store
.begin_subject_deletion(&id, &saved.operation_id)
.unwrap(),
saved
);
std::thread::scope(|scope| {
let a = scope.spawn(|| store.finish_subject_deletion(&id, &saved.operation_id));
let b = scope.spawn(|| second.finish_subject_deletion(&id, &saved.operation_id));
assert_eq!(a.join().unwrap().unwrap(), b.join().unwrap().unwrap());
});
}
#[test]
fn deletion_migration_failure_rolls_back_fences_and_trigger_rewrites_then_retries() {
let temp = tempfile::tempdir().unwrap();
let id;
{
let manager = FeatureStorage::new(temp.path());
let scope = manager.workspace("workspace-a").unwrap();
let registration = scope
.register(FeatureRegistration::new(
SUBJEKTIV_FEATURE_ID,
&MIGRATIONS[..13],
))
.unwrap();
let store = SubjektivStore::open(&scope, &registration).unwrap();
id = store.create_subject(role()).unwrap().id;
store
.stage_candidate(candidate(&id, "preserved", store.workspace_id()))
.unwrap();
store.database.try_with_connection::<_, SubjektivError>(|c| {
// Fail near the end, after retention triggers have already been rewritten.
c.execute_batch("CREATE TRIGGER deletion_fence_journal_sources_INSERT BEFORE INSERT ON journal_sources BEGIN SELECT 1; END;")?;
Ok(())
}).unwrap();
drop(store);
drop(scope);
manager.shutdown().unwrap();
}
let manager = FeatureStorage::new(temp.path());
let scope = manager.workspace("workspace-a").unwrap();
let current = SubjektivStore::register(&scope).unwrap();
assert!(SubjektivStore::open(&scope, &current).is_err());
drop(scope);
manager.shutdown().unwrap();
let manager = FeatureStorage::new(temp.path());
let scope = manager.workspace("workspace-a").unwrap();
// Reopen only the fixture's pre-deletion registration to inspect the failed migration.
let old = scope
.register(FeatureRegistration::new(
SUBJEKTIV_FEATURE_ID,
&MIGRATIONS[..13],
))
.unwrap();
let store = SubjektivStore::open(&scope, &old).unwrap();
store
.database
.try_with_connection::<_, SubjektivError>(|c| {
assert!(!c.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='subject_deletions')",
[],
|r| r.get::<_, bool>(0)
)?);
let sql: String = c.query_row(
"SELECT sql FROM sqlite_master WHERE name='staging_records_no_delete'",
[],
|r| r.get(0),
)?;
assert!(!sql.contains("subject_deletions"));
assert!(
c.execute("DELETE FROM staging_records WHERE subject_id=?1", [&id])
.is_err()
);
c.execute_batch("DROP TRIGGER deletion_fence_journal_sources_INSERT;")?;
Ok(())
})
.unwrap();
drop(store);
drop(scope);
manager.shutdown().unwrap();
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
assert!(store.staging_candidate(&id, "preserved").unwrap().is_some());
assert!(store.pending_subject_deletions().unwrap().is_empty());
store.begin_subject_deletion(&id, "delete").unwrap();
store.finish_subject_deletion(&id, "delete").unwrap();
}
+9
View File
@@ -39,6 +39,7 @@ fn denied() -> JournalError {
}
pub(super) fn check_writer(tx: &Connection, host: &JournalHostContext) -> JournalResult<()> {
crate::require_subject_in_connection(tx, &host.subject_id)?;
let attr = host.attribution.as_ref().ok_or_else(denied)?;
let binding = host.writer_binding_id.as_ref().ok_or_else(denied)?;
if !tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings b JOIN journal_session_bindings s ON s.binding_id=b.binding_id WHERE b.binding_id=?1 AND b.subject_id=?2 AND b.runtime_id=?3 AND b.worker_id=?4 AND b.active=1 AND s.session_id=?5)", params![binding,host.subject_id,attr.runtime_id,attr.worker_id,attr.session_id], |r| r.get::<_,bool>(0))? {
@@ -63,6 +64,7 @@ impl SubjektivStore {
binding: &JournalWriterBinding,
) -> JournalResult<Option<String>> {
self.database.try_with_connection(|tx| {
crate::require_subject_in_connection(tx, &binding.subject_id)?;
Ok(tx.query_row("SELECT j.source_session_id FROM journals j JOIN journal_session_bindings s ON s.session_id=j.source_session_id WHERE j.subject_id=?1 AND s.binding_id=(SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=0 ORDER BY rowid DESC LIMIT 1) ORDER BY j.updated_at DESC,j.id DESC LIMIT 1", [&binding.subject_id], |r| r.get(0)).optional()?)
})
}
@@ -77,6 +79,7 @@ impl SubjektivStore {
worker: &str,
) -> JournalResult<bool> {
self.database.try_with_connection(|tx| {
crate::require_subject_in_connection(tx, subject)?;
Ok(tx.query_row(
"SELECT NOT EXISTS(SELECT 1 FROM journal_writer_bindings
WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3)
@@ -95,12 +98,14 @@ impl SubjektivStore {
worker: &str,
) -> JournalResult<bool> {
self.database.try_with_connection(|tx| {
crate::require_subject_in_connection(tx, subject)?;
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 AND active=0)", params![subject,runtime,worker], |r| r.get(0))?)
})
}
pub fn journal_writer_active(&self, binding: &JournalWriterBinding) -> JournalResult<bool> {
self.database.try_with_connection(|tx| {
crate::require_subject_in_connection(tx, &binding.subject_id)?;
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4 AND active=1)", params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id], |r| r.get(0))?)
})
}
@@ -111,6 +116,7 @@ impl SubjektivStore {
session: &str,
) -> JournalResult<bool> {
self.database.try_with_connection(|tx| {
crate::require_subject_in_connection(tx, &binding.subject_id)?;
Ok(tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_session_bindings WHERE binding_id=?1 AND session_id=?2)", params![binding.binding_id,session], |r| r.get(0))?)
})
}
@@ -135,6 +141,8 @@ impl SubjektivStore {
}
}
self.database.try_transaction(|tx| {
crate::require_subject_in_connection(tx, subject)?;
crate::deletion::require_live_worker(tx, runtime, worker)?;
let state: Option<String> = tx.query_row("SELECT state FROM subjects WHERE subject_id=?1", [subject], |r| r.get(0)).optional()?;
if state.as_deref() != Some("active") { return Err(JournalError::PermissionDenied("Subject is absent or retired".into())); }
let current: Option<JournalWriterBinding> = tx.query_row("SELECT subject_id,runtime_id,worker_id,binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND active=1", [subject], |r| Ok(JournalWriterBinding { subject_id:r.get(0)?,runtime_id:r.get(1)?,worker_id:r.get(2)?,binding_id:r.get(3)? })).optional()?;
@@ -200,6 +208,7 @@ impl SubjektivStore {
/// It never fixes an open entry or declares the represented work complete.
pub fn release_journal_writer(&self, binding: &JournalWriterBinding) -> JournalResult<()> {
self.database.try_transaction(|tx| {
crate::require_subject_in_connection(tx, &binding.subject_id)?;
let active: Option<bool> = tx.query_row("SELECT active FROM journal_writer_bindings WHERE binding_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4",params![binding.binding_id,binding.subject_id,binding.runtime_id,binding.worker_id],|r|r.get(0)).optional()?;
let Some(active) = active else { return Err(denied()); };
if !active { return Ok(()); }
@@ -1158,3 +1158,69 @@ fn failed_link_migration_rolls_back_ddl_and_preserves_existing_data_for_retry()
assert_eq!(linked(&f.links(&selector, true).sources[0]).1, entry);
assert_eq!(f.snapshot(), before);
}
#[test]
fn subject_deletion_denies_all_memory_journal_context_and_reconcile_entry_points() {
let f = Fixture::new();
let staged = f.stage("candidate", &[0]);
let selector = f.apply(&staged);
let entry = f.fixed("entry", 0, vec![]);
f.links(&selector, true);
f.store()
.begin_subject_deletion(&f.host.subject_id, "delete")
.unwrap();
for request in [
MemoryJournalRequest::Links {
selector: selector.clone(),
},
MemoryJournalRequest::Reconcile {
selector: selector.clone(),
},
MemoryJournalRequest::LinksPage {
selector: selector.clone(),
source_offset: 0,
limit: 1,
},
MemoryJournalRequest::ReconcilePage {
selector: selector.clone(),
source_offset: 0,
limit: 1,
},
MemoryJournalRequest::Read {
selector,
source_index: 0,
journal_id: f.journal_id.clone(),
entry_id: entry.clone(),
expected_token: None,
byte_offset: 0,
max_bytes: 8,
},
MemoryJournalRequest::Related {
journal_id: f.journal_id.clone(),
entry_id: entry,
after: None,
limit: 1,
},
] {
assert!(matches!(
f.store().execute_memory_journal(&f.host, request),
Err(JournalError::Subject(
crate::SubjektivError::SubjectNotFound(_)
))
));
}
f.store()
.finish_subject_deletion(&f.host.subject_id, "delete")
.unwrap();
f.store()
.database
.try_with_connection::<_, JournalError>(|c| {
assert_eq!(
c.query_row("SELECT COUNT(*) FROM memory_journal_links", [], |r| r
.get::<_, i64>(0))?,
0
);
Ok(())
})
.unwrap();
}
@@ -63,6 +63,10 @@ impl JournalInput {
// below operates on this value only, never by making another live capture request.
let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope);
input.capture = capture_once(journal.process_order, journal.process_byte, budget)?;
// Capture runs outside SQLite; a delayed callback cannot deliver deleted content.
store
.database
.try_with_connection(|tx| super::store::check_host(store, tx, host, true))?;
input.selected_ranges = capture_ranges(&input.capture)?;
if input.selected_ranges.is_empty() {
return Ok(input);
+5
View File
@@ -469,6 +469,11 @@ fn execute_inner(
Ok::<_, JournalError>(())
})?;
}
// Also check empty source reads and Initial snapshot responses (no delivery write).
store.database.try_with_connection(|tx| {
super::store::job(store, tx, context, false)?;
Ok::<_, JournalError>(())
})?;
Ok(response)
}
+2
View File
@@ -105,6 +105,7 @@ pub(super) fn check_session_host(
"different storage scope".into(),
));
}
crate::require_subject_in_connection(tx, &host.subject_id)?;
let attr = host
.attribution
.as_ref()
@@ -134,6 +135,7 @@ pub(super) fn check_host(
"different storage scope".into(),
));
}
crate::require_subject_in_connection(tx, &host.subject_id)?;
if write {
let state: String = tx.query_row(
"SELECT state FROM subjects WHERE subject_id=?1",
+326 -6
View File
@@ -349,7 +349,7 @@ mod input_delivery {
open_action("must not save"),
f.ranges.clone(),
);
f.store().retire_subject(&f.host.subject_id).unwrap();
crate::tests::legacy_retired_fixture(f.store(), &f.host.subject_id).unwrap();
f.reopen();
assert!(matches!(
service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges),
@@ -1210,6 +1210,21 @@ mod snapshot_inputs {
service::finish(f.store(), &recovery).unwrap(),
saved_completion
);
// Deletion, unlike writer release, invalidates even exact saved input/completion replay.
f.store()
.begin_subject_deletion(&host.subject_id, "delete")
.unwrap();
subject_deleted(
f.store()
.prepare_journal_input(&host, &input, generation.job.clone()),
);
subject_deleted(service::finish(f.store(), &recovery));
subject_deleted(JournalJobContext::validated_recovery(
f.store(),
&host,
&generation.id,
generation.job.clone(),
));
}
#[test]
@@ -2833,7 +2848,7 @@ fn job_service_source_reads_are_generation_bound_and_stop_at_supplied_byte_end()
#[test]
fn explicit_erasure_remains_available_for_retired_subject_without_reopening_recording() {
let f = Fixture::new();
f.store().retire_subject(&f.host.subject_id).unwrap();
crate::tests::legacy_retired_fixture(f.store(), &f.host.subject_id).unwrap();
assert!(f.store().journal_retention_required(&f.host).unwrap());
f.store()
.erase_journal_session(&f.host, "erase-retired")
@@ -4086,10 +4101,7 @@ fn retired_subject_keeps_reader_access_but_rejects_existing_write_context() {
},
vec![],
);
fixture
.store()
.retire_subject(&fixture.host.subject_id)
.unwrap();
crate::tests::legacy_retired_fixture(fixture.store(), &fixture.host.subject_id).unwrap();
assert!(matches!(
fixture.store().execute_journal(&ctx, late),
Err(JournalError::PermissionDenied(_))
@@ -5745,3 +5757,311 @@ fn late_gap_admission_and_partial_updates_reject_stale_source_page_cursors() {
Err(JournalError::InvalidInput(_))
));
}
fn subject_deleted<T: std::fmt::Debug>(result: JournalResult<T>) {
assert!(
matches!(
result,
Err(JournalError::Subject(
crate::SubjektivError::SubjectNotFound(_)
))
),
"unexpected result: {result:?}"
);
}
#[test]
fn subject_deletion_fence_denies_all_journal_read_write_binding_and_recovery_paths() {
use transport::{
JournalJobRequest as Request, JournalSnapshot, JournalTextRead, JournalTextTarget,
};
let mut f = Fixture::new();
let (generation, ctx) = f.generation("delete-job", f.ranges.clone());
let operation = f.operation(
"saved",
open_action("private body"),
vec![f.ranges[0].clone()],
);
let receipt = f.store().execute_journal(&ctx, operation.clone()).unwrap();
let entry = receipt.entry_id.unwrap();
let reader = JournalHostContext::validated_reader(f.store(), &f.host.subject_id).unwrap();
let writer = f
.store()
.claim_journal_writer(&f.host.subject_id, "runtime", "worker")
.unwrap();
let digest = format!("sha256:{}", "a".repeat(64));
service::attest_input_delivery(f.store(), &ctx, &digest, &f.ranges).unwrap();
f.store().database.try_transaction::<_, JournalError>(|tx| {
tx.execute("INSERT INTO journal_generation_deliveries(generation_id,source_order,ranges_json) VALUES(?1,0,'[]')", [&generation.id])?;
Ok(())
}).unwrap();
f.store().database.try_transaction::<_, JournalError>(|tx| {
tx.execute("INSERT INTO legacy_record_archive(source_table,row_position,row_json) VALUES('subjects',0,?1)", [serde_json::json!({"subject_id":f.host.subject_id,"record_json":"archived private body"}).to_string()])?;
Ok(())
}).unwrap();
f.store()
.begin_subject_deletion(&f.host.subject_id, "delete")
.unwrap();
// Exercise actual SQL owner resolution for *every* indirect content table,
// not only table-name inventory. Fence errors must precede PK/immutable/FK errors.
f.store()
.database
.try_with_connection::<_, JournalError>(|tx| {
for (table, field, new_owner) in [
("journal_sources", "journal_id", "other-journal"),
("journal_operations", "journal_id", "other-journal"),
("journal_generations", "journal_id", "other-journal"),
("journal_session_bindings", "binding_id", "other-binding"),
(
"journal_generation_deliveries",
"generation_id",
"other-generation",
),
(
"journal_generation_input_deliveries",
"generation_id",
"other-generation",
),
(
"legacy_record_archive",
"row_json",
"{\"subject_id\":\"other-subject\"}",
),
] {
assert!(
tx.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r
.get::<_, i64>(0))?
> 0,
"{table}"
);
let insert = tx
.execute(&format!("INSERT INTO {table} SELECT * FROM {table}"), [])
.unwrap_err();
assert!(
insert.to_string().contains("Subject deletion has begun"),
"{table}: {insert}"
);
let update = tx
.execute(&format!("UPDATE {table} SET {field}=?1"), [new_owner])
.unwrap_err();
assert!(
update.to_string().contains("Subject deletion has begun"),
"{table}: {update}"
);
}
Ok(())
})
.unwrap();
macro_rules! denied { ($($result:expr),+ $(,)?) => { $(subject_deleted($result);)+ }; }
for host in [&f.host, &reader] {
denied!(
f.store().list_journals(host, None, 1),
f.store().read_journal(host, &f.journal_id),
f.store().list_journal_entries(host, &f.journal_id, None, 1),
f.store().read_journal_text(
host,
&f.journal_id,
Some(&entry),
JournalTextField::Body,
None,
0,
8
),
f.store().read_journal_source(host, &f.journal_id, 0),
f.store().list_journal_sources(host, &f.journal_id, None, 1),
f.store()
.query_journal_operation(host, &f.journal_id, "saved"),
f.store()
.read_journal_operation_status(host, &f.journal_id, "saved"),
f.store()
.read_journal_operation_coverage(host, &f.journal_id, "saved", 0, 1),
f.store()
.list_journal_operations(host, &f.journal_id, None, 1),
f.store().journal_entry_summary(host, &f.journal_id, &entry),
f.store()
.journal_correction_targets(host, &f.journal_id, &entry),
f.store()
.browse_journal_entries(host, None, None, None, None, 1),
f.store()
.journal_has_unreflected_sources(host, &f.journal_id),
f.store().journal_startup_snapshot(host, None),
);
}
denied!(
f.store().journal_compaction_snapshot(&f.host),
f.store().journal_session_erased(&f.host),
f.store().journal_for_session(&f.host),
f.store().journal_retention_required(&f.host),
f.store().erase_journal_session(&f.host, "erase"),
f.store().release_erased_journal_body(&f.host),
f.store().journal_predecessor_session(&writer),
f.store()
.journal_body_is_unpublished(&f.host.subject_id, "runtime", "worker"),
f.store()
.journal_worker_was_released(&f.host.subject_id, "runtime", "worker"),
f.store().journal_writer_active(&writer),
f.store()
.journal_session_writer_matches(&writer, "source-session"),
f.store()
.claim_journal_writer(&f.host.subject_id, "runtime", "worker"),
f.store()
.bind_journal_session(&writer, f.host.attribution.clone().unwrap()),
f.store().release_journal_writer(&writer),
f.store().accept_journal_sources(&f.host, "late", &f.ranges),
f.store()
.mark_journal_sources_unavailable(&f.host, &f.journal_id, &f.ranges),
f.store()
.mark_journal_sources_available(&f.host, &f.journal_id, &f.ranges),
f.store().active_journal_generation(&f.host, &f.journal_id),
f.store().prepare_journal_generation(
&f.host,
&f.journal_id,
generation.job.clone(),
f.ranges.clone()
),
f.store()
.close_journal_generation(&f.host, &generation.id, &generation.job),
f.store().read_journal_generation(&f.host, &generation.id),
f.store().journal_resume_state(&f.host, &f.journal_id),
f.store().execute_journal(&ctx, operation.clone()),
JournalJobContext::validated_recovery(
f.store(),
&f.host,
&generation.id,
generation.job.clone()
),
service::attest_input_delivery(f.store(), &ctx, &digest, &f.ranges),
service::finish(f.store(), &ctx),
service::execute_snapshot(f.store(), &ctx, &digest, Request::ReadState),
service::execute_snapshot(
f.store(),
&ctx,
&digest,
Request::QueryOperation {
request_id: "saved".into()
}
),
service::execute_snapshot(
f.store(),
&ctx,
&digest,
Request::ReadText {
read: JournalTextRead {
target: JournalTextTarget::Position {
snapshot: JournalSnapshot::Initial
},
offset: 0,
limit: 8,
content_token: None
}
}
),
);
// Failed purge and restart must not revive any cached context or exact receipt.
f.sql("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected'); END;").unwrap();
assert!(
f.store()
.finish_subject_deletion(&f.host.subject_id, "delete")
.is_err()
);
f.reopen();
subject_deleted(f.store().execute_journal(&ctx, operation));
f.sql("DROP TRIGGER injected_subject_purge_failure;")
.unwrap();
f.store()
.finish_subject_deletion(&f.host.subject_id, "delete")
.unwrap();
f.store()
.database
.try_with_connection::<_, JournalError>(|tx| {
for table in [
"journals",
"journal_entries",
"journal_corrections",
"journal_sources",
"journal_generations",
"journal_operations",
"journal_generation_deliveries",
"journal_generation_input_deliveries",
"journal_writer_bindings",
"journal_session_bindings",
] {
assert_eq!(
tx.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r
.get::<_, i64>(0))?,
0,
"{table}"
);
}
Ok(())
})
.unwrap();
}
#[test]
fn subject_deletion_during_capture_rejects_even_empty_delayed_input() {
use session_store::capture::CaptureDto;
let f = Fixture::new();
let result = recording::JournalInput::build(f.store(), &f.host, &f.journal_id, |_, _, _| {
f.store()
.begin_subject_deletion(&f.host.subject_id, "delete")?;
Ok(CaptureDto {
version: 1,
session_id: "source-session".into(),
segment_id: "segment".into(),
view_entry_count: 0,
entries: vec![],
})
});
subject_deleted(result);
}
#[test]
fn subject_deletion_during_source_callback_cannot_return_or_attest_cached_material() {
use transport::{JournalJobRequest as Request, JournalTextRead, JournalTextTarget};
let f = Fixture::new();
let (_, ctx) = f.generation("delayed-source", f.ranges.clone());
let result = service::execute_bounded(
f.store(),
&ctx,
Request::ReadText {
read: JournalTextRead {
target: JournalTextTarget::Source {
pointer: f.ranges[0].source.pointer.clone(),
},
offset: 0,
limit: 8,
content_token: None,
},
},
|source, start, _| {
f.store()
.begin_subject_deletion(&f.host.subject_id, "delete")?;
Ok(recording::JournalMaterial {
range: JournalSourceRange {
source: source.clone(),
bytes: JournalByteRange {
start,
end: start + 8,
},
},
text: "private!".into(),
})
},
);
subject_deleted(result);
f.store()
.database
.try_with_connection::<_, JournalError>(|tx| {
assert_eq!(
tx.query_row(
"SELECT COUNT(*) FROM journal_generation_deliveries",
[],
|r| r.get::<_, i64>(0)
)?,
0
);
Ok(())
})
.unwrap();
}
+63 -28
View File
@@ -1,7 +1,9 @@
#![doc = include_str!("../README.md")]
pub mod api;
pub mod deletion;
pub mod job;
pub use deletion::{DeletionSubjectJob, SubjectDeletion};
pub mod journal;
mod legacy_migrations;
pub mod permissions;
@@ -51,7 +53,9 @@ pub enum SubjektivError {
InvalidRecord(String),
#[error("subject `{0}` was not found in this Workspace")]
SubjectNotFound(String),
#[error("subject `{0}` is retired")]
#[error("Subject deletion request `{0}` conflicts with a saved deletion")]
SubjectDeletionConflict(String),
#[error("subject `{0}` is legacy-retired")]
SubjectRetired(String),
#[error("the selected Worker Profile does not enable subjektiv")]
ProfileDisabled,
@@ -146,7 +150,10 @@ impl<'de> Deserialize<'de> for SubjectRole {
#[serde(rename_all = "snake_case")]
pub enum SubjectState {
Active,
Retired,
/// Saved pre-deletion lifecycle state. The old `retired` spelling remains readable
/// and serializable solely to preserve existing data; no retirement operation exists.
#[serde(rename = "retired")]
LegacyRetired,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -644,6 +651,11 @@ static MIGRATIONS: &[FeatureMigration] = &[
"bind Journal generation delivery to immutable Job input digest",
journal::service::migrate_input_delivery,
),
FeatureMigration::new(
14,
"add permanent Subject deletion fences and explicit erasure",
deletion::migrate,
),
];
pub const REGISTRATION: FeatureRegistration =
@@ -772,6 +784,9 @@ impl SubjektivStore {
pub fn subject(&self, subject_id: &str) -> Result<Option<SubjectRecord>> {
self.database.try_with_connection(|connection| {
if deletion::is_deleting(connection, subject_id)? {
return Ok(None);
}
let raw = connection
.query_row(
"SELECT record_json FROM subjects WHERE subject_id = ?1",
@@ -862,11 +877,13 @@ impl SubjektivStore {
self.database.try_with_connection(|connection| {
let sql = if after.is_some() {
"SELECT record_json FROM subjects
WHERE created_at > ?1 OR (created_at = ?1 AND subject_id > ?2)
WHERE NOT EXISTS (SELECT 1 FROM subject_deletions d WHERE d.subject_id=subjects.subject_id)
AND (created_at > ?1 OR (created_at = ?1 AND subject_id > ?2))
ORDER BY created_at ASC, subject_id ASC
LIMIT ?3"
} else {
"SELECT record_json FROM subjects
WHERE NOT EXISTS (SELECT 1 FROM subject_deletions d WHERE d.subject_id=subjects.subject_id)
ORDER BY created_at ASC, subject_id ASC
LIMIT ?1"
};
@@ -895,27 +912,6 @@ impl SubjektivStore {
})
}
/// Retires a Subject independently of stopping Workers. Memory/history remain unedited, undeleted, and readable.
/// Retirement is irreversible; new candidates and Memory changes require an active Subject.
pub fn retire_subject(&self, subject_id: &str) -> Result<SubjectRecord> {
self.database.try_transaction(|transaction| {
let mut subject = require_subject(transaction, subject_id)?;
if subject.state == SubjectState::Retired {
return Ok(subject);
}
subject.state = SubjectState::Retired;
subject.updated_at = now();
let raw = serde_json::to_string(&subject)?;
transaction.execute(
"UPDATE subjects
SET state = 'retired', record_json = ?2, updated_at = ?3
WHERE subject_id = ?1",
params![subject_id, raw, subject.updated_at],
)?;
Ok(subject)
})
}
/// Saves an immutable candidate without changing committed Memory.
/// Replay with the same candidate identity and payload returns the original record; different content conflicts.
/// Extraction failure after saving never revokes the candidate. The Host commits the extraction pointer separately.
@@ -1003,6 +999,10 @@ impl SubjektivStore {
) -> Result<Option<SubjectSessionAttribution>> {
validate_label("session id", session_id)?;
self.database.try_with_connection(|connection| {
if let Some(subject) = deletion::deleted_session_subject(connection, session_id)? {
return Err(SubjektivError::SubjectNotFound(subject));
}
let raw = connection
.query_row(
"SELECT record_json FROM subject_session_attributions
@@ -1011,7 +1011,12 @@ impl SubjektivStore {
|row| row.get::<_, String>(0),
)
.optional()?;
raw.map(|raw| parse_session_attribution(&raw)).transpose()
raw.map(|raw| {
let record = parse_session_attribution(&raw)?;
deletion::require_not_deleting(connection, &record.subject_id)?;
Ok(record)
})
.transpose()
})
}
@@ -1021,6 +1026,7 @@ impl SubjektivStore {
) -> Result<Vec<SubjectSessionAttribution>> {
validate_label("subject id", subject_id)?;
self.database.try_with_connection(|connection| {
require_subject_in_connection(connection, subject_id)?;
let mut statement = connection.prepare(
"SELECT record_json FROM subject_session_attributions
WHERE subject_id = ?1
@@ -1062,6 +1068,7 @@ impl SubjektivStore {
}
let query_limit = limit.saturating_add(1);
self.database.try_with_connection(|connection| {
require_subject_in_connection(connection, subject_id)?;
let raw = match (session_id, after) {
(Some(session_id), _) => {
let mut statement = connection.prepare(
@@ -1128,6 +1135,7 @@ impl SubjektivStore {
candidate_id: &str,
) -> Result<Option<SubjectStagingRecord>> {
self.database.try_with_connection(|connection| {
deletion::require_not_deleting(connection, subject_id)?;
let raw = connection
.query_row(
"SELECT record_json FROM staging_records
@@ -1292,6 +1300,7 @@ impl SubjektivStore {
validate_nonempty("resolution reason", &request.reason)?;
let request_raw = serde_json::to_string(&request)?;
self.database.try_transaction(|transaction| {
deletion::require_not_deleting(transaction, subject_id)?;
if let Some((stored_request, stored_receipt)) = transaction
.query_row(
"SELECT request_json, receipt_json
@@ -1426,6 +1435,7 @@ impl SubjektivStore {
candidate_id: &str,
) -> Result<Option<StagingResolution>> {
self.database.try_with_connection(|connection| {
deletion::require_not_deleting(connection, subject_id)?;
let raw = connection
.query_row(
"SELECT resolution_json FROM staging_resolutions
@@ -1440,6 +1450,7 @@ impl SubjektivStore {
pub fn memory(&self, subject_id: &str, memory_id: &str) -> Result<Option<MemoryRecord>> {
self.database.try_with_connection(|connection| {
deletion::require_not_deleting(connection, subject_id)?;
let raw = connection
.query_row(
"SELECT record_json FROM memory_records
@@ -1459,6 +1470,7 @@ impl SubjektivStore {
change_id: String,
) -> Result<Option<MemoryRecord>> {
self.database.try_with_connection(|connection| {
deletion::require_not_deleting(connection, subject_id)?;
let raw = connection
.query_row(
"SELECT record_json FROM memory_changes
@@ -1854,6 +1866,7 @@ impl SubjektivStore {
snapshot_id: &str,
) -> Result<Option<SurfaceSnapshot>> {
self.database.try_with_connection(|connection| {
deletion::require_not_deleting(connection, subject_id)?;
let raw = connection
.query_row(
"SELECT snapshot_json FROM surface_snapshots
@@ -2427,6 +2440,11 @@ fn write_session_attribution(
record: SubjectSessionAttribution,
raw: String,
) -> Result<SubjectSessionAttribution> {
deletion::require_live_worker(transaction, &record.runtime_id, &record.worker_id)?;
if let Some(subject) = deletion::deleted_session_subject(transaction, &record.session_id)? {
return Err(SubjektivError::SubjectNotFound(subject));
}
let existing = transaction
.query_row(
"SELECT record_json FROM subject_session_attributions WHERE session_id = ?1",
@@ -2621,6 +2639,7 @@ fn require_subject_in_connection(
connection: &rusqlite::Connection,
subject_id: &str,
) -> Result<SubjectRecord> {
deletion::require_not_deleting(connection, subject_id)?;
let raw = connection
.query_row(
"SELECT record_json FROM subjects WHERE subject_id = ?1",
@@ -2637,7 +2656,7 @@ fn require_active_subject(
subject_id: &str,
) -> Result<SubjectRecord> {
let subject = require_subject(transaction, subject_id)?;
if subject.state == SubjectState::Retired {
if subject.state == SubjectState::LegacyRetired {
return Err(SubjektivError::SubjectRetired(subject_id.to_string()));
}
Ok(subject)
@@ -3278,6 +3297,7 @@ fn require_job_generation(
job_id: &str,
attempt_id: &str,
) -> Result<String> {
deletion::require_not_deleting(connection, subject_id)?;
let change_id: Option<String> = connection.query_row(
"SELECT memory_fingerprint FROM surface_generation_runs WHERE subject_id = ?1 AND generation_id = ?2 AND job_id = ?3 AND attempt_id = ?4",
params![subject_id, generation_id, job_id, attempt_id], |row| row.get(0),
@@ -3370,6 +3390,21 @@ mod tests {
use super::*;
use feature_storage::FeatureStorage;
/// Builds saved pre-deletion lifecycle data; not a current product operation.
pub(crate) fn legacy_retired_fixture(
store: &SubjektivStore,
subject_id: &str,
) -> Result<SubjectRecord> {
store.database.try_transaction(|tx| {
let mut subject = require_subject(tx, subject_id)?;
subject.state = SubjectState::LegacyRetired;
subject.updated_at = now();
tx.execute("UPDATE subjects SET state='retired',record_json=?2,updated_at=?3 WHERE subject_id=?1",
params![subject_id,serde_json::to_string(&subject)?,subject.updated_at])?;
Ok(subject)
})
}
pub(super) fn role() -> SubjectRole {
SubjectRole::new("workspace_companion").unwrap()
}
@@ -5047,8 +5082,8 @@ mod tests {
Err(SubjektivError::InvalidStateTransition { .. })
));
let retired = store.retire_subject(&subject.id).unwrap();
assert_eq!(retired.state, SubjectState::Retired);
let retired = crate::tests::legacy_retired_fixture(&store, &subject.id).unwrap();
assert_eq!(retired.state, SubjectState::LegacyRetired);
assert!(matches!(
store.create_memory(&subject.id, draft("after retirement", "invalid")),
Err(SubjektivError::SubjectRetired(_))
+8 -1
View File
@@ -94,6 +94,10 @@ impl SubjektivStore {
pub fn bind_worker_execution_grant(&self, grant: &WorkerExecutionGrant) -> Result<()> {
validate_grant(self.scope_id(), grant)?;
self.database.try_transaction(|tx| {
require_subject(tx, &grant.subject_id)?;
for (runtime, worker) in [(&grant.runtime_id, &grant.worker_id), (&grant.origin_runtime_id, &grant.origin_worker_id)] {
if tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deleted_workers WHERE runtime_id=?1 AND worker_id=?2)", params![runtime,worker], |r|r.get::<_,bool>(0))? { return Err(denied("Worker belongs to a deleted Subject")); }
}
if let Some(existing) = grant_in_connection(tx, &grant.runtime_id, &grant.worker_id)? {
return if existing == *grant { Ok(()) } else {
Err(SubjektivError::WorkerExecutionGrantConflict(format!("{}/{}", grant.runtime_id, grant.worker_id)))
@@ -184,9 +188,12 @@ fn grant_in_connection(
runtime: &str,
worker: &str,
) -> Result<Option<WorkerExecutionGrant>> {
// Absence must never become non-Subject fallback authority after purge.
deletion::require_live_worker(connection, runtime, worker)?;
let raw: Option<String> = connection
.query_row(
"SELECT record_json FROM worker_execution_grants WHERE runtime_id=?1 AND worker_id=?2",
"SELECT record_json FROM worker_execution_grants WHERE runtime_id=?1 AND worker_id=?2 AND NOT EXISTS(SELECT 1 FROM subject_deletions d WHERE d.subject_id=worker_execution_grants.subject_id)",
params![runtime, worker],
|row| row.get(0),
)
+50 -2
View File
@@ -315,7 +315,7 @@ fn redelegation_requires_current_source_and_explicit_retained_parent_ceiling() {
store.worker_execution_grant("runtime", "child").unwrap(),
Some(child)
);
store.retire_subject(&subject).unwrap();
crate::tests::legacy_retired_fixture(&store, &subject).unwrap();
stale.source_change_id = revoked.change_id;
assert!(store.bind_worker_execution_grant(&stale).is_err());
}
@@ -582,7 +582,7 @@ fn subject_grant_listing_is_scoped_retained_ordered_and_fails_on_overflow() {
&update("revoke", Some(&source.change_id), Vec::new()),
)
.unwrap();
store.retire_subject(&subject).unwrap();
crate::tests::legacy_retired_fixture(&store, &subject).unwrap();
let listed = store
.list_subject_worker_execution_grants(&subject)
.unwrap();
@@ -654,3 +654,51 @@ fn failed_schema_migration_rolls_back_new_tables_and_preserves_old_data() {
Ok(())
}).unwrap();
}
#[test]
fn deletion_fences_source_and_child_grant_replays_and_never_falls_back_after_purge() {
let temp = tempfile::tempdir().unwrap();
let (_manager, _scope, store) = open_store(temp.path(), "workspace-a");
let subject = store.create_subject(role()).unwrap().id;
let replacement = update("allow", None, vec![permission("file.read", ".", true)]);
let source = store
.set_subject_permissions(&subject, &replacement)
.unwrap();
let child = grant(&subject, &source.change_id, "child");
store.bind_worker_execution_grant(&child).unwrap();
store.begin_subject_deletion(&subject, "delete").unwrap();
assert!(store.subject_permissions(&subject).is_err());
assert!(
store
.set_subject_permissions(&subject, &replacement)
.is_err()
);
assert!(store.bind_worker_execution_grant(&child).is_err());
assert!(store.worker_execution_grant("runtime", "child").is_err());
assert!(store.worker_subject_deleted("runtime", "body").unwrap());
assert!(store.worker_subject_deleted("runtime", "child").unwrap());
assert_eq!(store.deletion_subject_workers(&subject).unwrap().len(), 2);
store.finish_subject_deletion(&subject, "delete").unwrap();
assert!(store.worker_execution_grant("runtime", "child").is_err());
assert!(store.worker_execution_grant("runtime", "body").is_err());
assert!(
store
.worker_execution_grant("runtime", "unrelated")
.unwrap()
.is_none()
);
let fresh = store.create_subject(role()).unwrap().id;
assert!(store.subject_permissions(&fresh).unwrap().is_none());
assert!(
store
.record_session_attribution(
SubjectSessionAttribution::new(&fresh, "runtime", "child", "new-session").unwrap()
)
.is_err()
);
assert!(
store
.claim_journal_writer(&fresh, "runtime", "body")
.is_err()
);
}
+31
View File
@@ -727,6 +727,23 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
Err("execution backend cannot confirm Subject create execution absence".into())
}
/// Read-only exact execution identity check before Runtime persists exclusion.
fn preflight_subject_delete_worker(
&self,
_request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
Err("execution backend cannot validate Subject Worker identity".into())
}
/// Runtime has durably fenced admission. Confirm shutdown and discard all
/// execution candidates under the backend Worker lock; uncertainty must fail.
fn subject_delete_worker(
&self,
_request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
Err("execution backend cannot confirm Subject Worker shutdown".into())
}
/// Must reserve the actual Controller, not infer Idle from a catalog snapshot.
fn subject_release(
&self,
@@ -1069,6 +1086,20 @@ impl WorkerExecutionBackendRef {
self.backend.recover_subject_create(request)
}
pub(crate) fn preflight_subject_delete_worker(
&self,
request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
self.backend.preflight_subject_delete_worker(request)
}
pub(crate) fn subject_delete_worker(
&self,
request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
self.backend.subject_delete_worker(request)
}
pub(crate) fn subject_release(
&self,
request: &runtime_api::SubjectReleaseRequest,
+69 -7
View File
@@ -43,7 +43,8 @@ use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
#[path = "test_support.rs"]
pub mod test_support;
pub(crate) const SCHEMA_VERSION: u32 = 14;
pub(crate) const SCHEMA_VERSION: u32 = 15;
const RESTORE_REPAIR_SCHEMA_VERSION: u32 = 14;
const RESTORE_DECISION_SCHEMA_VERSION: u32 = 13;
const RESTORE_ADMISSION_SCHEMA_VERSION: u32 = 12;
// Frozen source schemas: do not move their meaning when adding a migration.
@@ -560,6 +561,22 @@ impl FsRuntimeStore {
self.root.join(RUNTIME_FILE)
}
pub(crate) fn worker_originals_absent(
&self,
worker_id: WorkerId,
) -> Result<bool, RuntimeError> {
let path = self.worker_dir(&worker_id);
match fs::symlink_metadata(&path) {
Ok(_) => Ok(false),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true),
Err(source) => Err(RuntimeError::StoreIo {
operation: "confirm Subject Worker original absence",
path,
source,
}),
}
}
fn worker_dir(&self, worker_id: &WorkerId) -> PathBuf {
self.root.join(WORKERS_DIR).join(worker_id.to_string())
}
@@ -659,6 +676,8 @@ pub(crate) struct OpenedFsRuntimeStore {
#[derive(Clone, Debug)]
pub(crate) struct PersistedRuntimeState {
pub(crate) subject_delete_worker_fences:
BTreeMap<WorkerId, crate::runtime::SubjectDeleteWorkerFence>,
pub(crate) subject_create_recovery_fences:
BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
pub(crate) subject_releases: Vec<(
@@ -876,11 +895,12 @@ fn plan_runtime_store_migration(
| SOURCE_SCHEMA_VERSION
| RESTORE_ADMISSION_SCHEMA_VERSION
| RESTORE_DECISION_SCHEMA_VERSION
| RESTORE_REPAIR_SCHEMA_VERSION
) {
return Err(runtime_store_corrupt(
&runtime_path,
format!(
"unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
"unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, {RESTORE_REPAIR_SCHEMA_VERSION}, or {SCHEMA_VERSION}"
),
));
}
@@ -1216,6 +1236,7 @@ fn migrate_worker_record(
| SOURCE_SCHEMA_VERSION
| RESTORE_ADMISSION_SCHEMA_VERSION
| RESTORE_DECISION_SCHEMA_VERSION
| RESTORE_REPAIR_SCHEMA_VERSION
) {
// Admission identity survives migration. Older pending records did not
// distinguish a committed decision from preparation or failed cleanup.
@@ -1229,10 +1250,12 @@ fn migrate_worker_record(
"Worker schema does not match Runtime migration source".into(),
));
}
if source_schema_version != RESTORE_DECISION_SCHEMA_VERSION {
if source_schema_version < RESTORE_DECISION_SCHEMA_VERSION {
migrate_restore_commit_fact(&mut identity_document, identity_path)?;
}
migrate_restore_repair_fact(&mut identity_document, identity_path)?;
if source_schema_version < RESTORE_REPAIR_SCHEMA_VERSION {
migrate_restore_repair_fact(&mut identity_document, identity_path)?;
}
identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION);
let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document)
.map_err(|error| runtime_store_corrupt(identity_path, error.to_string()))?;
@@ -1691,6 +1714,9 @@ fn migrate_runtime_document(
"schema_version".to_string(),
serde_json::Value::from(SCHEMA_VERSION),
);
object
.entry("subject_delete_worker_fences")
.or_insert_with(|| serde_json::json!({}));
// Intermediate schema-11 writers may already have saved irreversible fences.
// Default only a missing field; malformed or existing fences must not be erased.
object
@@ -1698,7 +1724,10 @@ fn migrate_runtime_document(
.or_insert_with(|| serde_json::json!({}));
if !matches!(
source_schema_version,
SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION | RESTORE_DECISION_SCHEMA_VERSION
SOURCE_SCHEMA_VERSION
| RESTORE_ADMISSION_SCHEMA_VERSION
| RESTORE_DECISION_SCHEMA_VERSION
| RESTORE_REPAIR_SCHEMA_VERSION
) {
object.remove("workers");
// This legacy cache is not Runtime lifecycle or restore authority. Bundles
@@ -2249,6 +2278,7 @@ fn migrate_runtime_store_in_place(
SOURCE_SCHEMA_VERSION
| RESTORE_ADMISSION_SCHEMA_VERSION
| RESTORE_DECISION_SCHEMA_VERSION
| RESTORE_REPAIR_SCHEMA_VERSION
)
{
atomic_write_json(
@@ -2288,6 +2318,7 @@ fn migrate_runtime_store_in_place(
#[derive(Clone, Debug, Serialize, Deserialize)]
struct RuntimeSnapshot {
subject_delete_worker_fences: BTreeMap<WorkerId, crate::runtime::SubjectDeleteWorkerFence>,
#[serde(default)]
subject_create_recovery_fences: BTreeMap<WorkerId, crate::runtime::SubjectCreateRecoveryFence>,
#[serde(default)]
@@ -2326,6 +2357,7 @@ fn record_worker_load_diagnostic(
impl RuntimeSnapshot {
fn from_persisted(state: &PersistedRuntimeState) -> Self {
Self {
subject_delete_worker_fences: state.subject_delete_worker_fences.clone(),
subject_create_recovery_fences: state.subject_create_recovery_fences.clone(),
subject_releases: state.subject_releases.clone(),
schema_version: SCHEMA_VERSION,
@@ -2350,6 +2382,33 @@ impl RuntimeSnapshot {
),
});
}
for (worker_id, fence) in &self.subject_delete_worker_fences {
let request = &fence.request;
let response_matches = fence.response.as_ref().is_none_or(|response| {
response.worker_id == request.worker_id
&& response.workspace_id == request.workspace_id
&& response.create_fingerprint == request.create_fingerprint
&& response.operation_id == request.operation_id
});
if *worker_id != request.worker_id
|| request.workspace_id.trim().is_empty()
|| request.create_fingerprint.trim().is_empty()
|| request.create_fingerprint.len() > 512
|| request.operation_id.trim().is_empty()
|| request.operation_id.len() > 512
|| self
.workspace_owners
.get(&request.workspace_id)
.is_none_or(|owner| owner.trim().is_empty())
|| !response_matches
{
return Err(RuntimeError::StoreCorrupt {
operation: "read Subject delete Worker fence",
path: path.to_path_buf(),
message: "invalid deletion identity, receipt or Workspace owner".into(),
});
}
}
for (worker_id, fence) in &self.subject_create_recovery_fences {
let request = &fence.request;
let response_matches = fence.response.as_ref().is_none_or(|response| {
@@ -2392,6 +2451,7 @@ impl RuntimeSnapshot {
workers: BTreeMap<WorkerId, PersistedWorkerRecord>,
) -> PersistedRuntimeState {
PersistedRuntimeState {
subject_delete_worker_fences: self.subject_delete_worker_fences,
subject_create_recovery_fences: self.subject_create_recovery_fences,
subject_releases: self.subject_releases,
display_name: self.display_name,
@@ -3218,8 +3278,7 @@ mod tests {
assert_eq!(fs::read(&worker_path).unwrap(), worker_before);
// The direct reader still rejects a previous schema: migration is
// explicit, not permissive deserialization of a newer format.
let snapshot: RuntimeSnapshot = serde_json::from_value(runtime.clone()).unwrap();
assert!(snapshot.validate(&runtime_path).is_err());
assert!(serde_json::from_value::<RuntimeSnapshot>(runtime.clone()).is_err());
let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap();
let state = opened.state.as_ref().unwrap();
@@ -3231,6 +3290,7 @@ mod tests {
expected_worker
);
runtime["schema_version"] = serde_json::json!(SCHEMA_VERSION);
runtime["subject_delete_worker_fences"] = serde_json::json!({});
if fence_state == "absent" {
runtime["subject_create_recovery_fences"] = serde_json::json!({});
}
@@ -4072,6 +4132,8 @@ mod tests {
);
}
include!("subject_delete_store_tests.rs");
fn canonical_worker_store() -> (
tempfile::TempDir,
FsRuntimeStore,
+17 -1
View File
@@ -2102,7 +2102,11 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s
{
return Some("workers:input");
}
if path == "/v1/subject-create-recovery" && *method == Method::POST {
if matches!(
path,
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
) && *method == Method::POST
{
return Some("workers:delete");
}
if path.ends_with("/stop")
@@ -3332,6 +3336,18 @@ mod tests {
struct AcceptingBackend;
impl WorkerExecutionBackend for AcceptingBackend {
fn preflight_subject_delete_worker(
&self,
_request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
Ok(())
}
fn subject_delete_worker(
&self,
_request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
Ok(())
}
fn recover_subject_create(
&self,
_request: &runtime_api::SubjectCreateRecoveryRequest,
@@ -406,6 +406,18 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
}
}
async fn subject_delete_worker(
&self,
request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, runtime_api::RuntimeApiError> {
let scope = material_scope(&self.state, &request.workspace_id)?;
self.state
.runtime
.subject_delete_worker_scoped(&scope, request)
.map_err(RuntimeHttpRestError::runtime)
.map_err(api_error)
}
async fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
@@ -1222,6 +1222,54 @@ async fn subject_create_recovery_signed_http_requires_delete_and_matching_worksp
}
}
#[tokio::test]
async fn subject_delete_worker_signed_http_requires_delete_and_matching_workspace() {
let f = Fixture::new();
let identity = RuntimeIdentityMaterial::generate("recovery-key").unwrap();
let trust = workspace_trust(&identity);
let app = runtime_http_router_with_workspace_auth(
f.runtime.clone(),
None,
workspace_test_auth(vec![trust.clone()]),
);
let path = "/v1/subject-delete-worker";
let mut request = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: f.scope.workspace_id.clone(),
worker_id: WorkerId::now_v7(),
create_fingerprint: "unpublished-create".into(),
operation_id: "recovery-1".into(),
};
for operation in ["workers:read", "workers:stop", "workers:create"] {
let body = serde_json::to_vec(&request).unwrap();
let mut claims = signed_claims(&trust, "POST", path, &body);
claims.operation = operation.into();
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
}
request.workspace_id = "other-workspace".into();
let body = serde_json::to_vec(&request).unwrap();
let claims = signed_claims(&trust, "POST", path, &body);
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::FORBIDDEN);
request.workspace_id = f.scope.workspace_id.clone();
for _ in 0..2 {
let body = serde_json::to_vec(&request).unwrap();
let claims = signed_claims(&trust, "POST", path, &body);
assert_eq!(claims.operation, "workers:delete");
let response = signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::OK);
let bytes = axum::body::to_bytes(response.into_body(), 16384)
.await
.unwrap();
let receipt: runtime_api::SubjectDeleteWorkerResponse =
serde_json::from_slice(&bytes).unwrap();
assert_eq!(receipt.worker_id, request.worker_id);
assert_eq!(receipt.workspace_id, request.workspace_id);
assert_eq!(receipt.create_fingerprint, request.create_fingerprint);
assert_eq!(receipt.operation_id, request.operation_id);
}
}
#[tokio::test]
async fn explicit_material_erasure_requires_delete_capability_and_matching_scope() {
let f = Fixture::new();
@@ -97,7 +97,7 @@ fn runtime_fixture(root: &Path, schema: u32) -> (WorkerId, SessionId, SegmentId)
"schema_version": schema, "display_name": "retained Runtime",
"backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
"config_bundles": {}, "workspace_owners": {"workspace-a": "owner-a"},
"diagnostics": [], "subject_releases": [],
"diagnostics": [], "subject_releases": [], "subject_delete_worker_fences": {},
"subject_create_recovery_fences": {fenced_worker.to_string(): {
"request": {"workspace_id": "workspace-a", "worker_id": fenced_worker,
"create_fingerprint": "retained-create", "operation_id": "retained-recovery"},
+47 -2
View File
@@ -1,3 +1,5 @@
mod subject_delete;
pub(crate) use subject_delete::SubjectDeleteWorkerFence;
#[cfg(feature = "restore-checkpoints")]
mod restore_checkpoint;
mod restore_continuation;
@@ -5645,6 +5647,7 @@ struct RuntimeState {
// Ephemeral write barrier, not authority: only canonical reread + sync can
// release it. Prevents another mutation from persisting an older projection.
unreconciled_worker_writes: std::collections::BTreeSet<WorkerId>,
subject_delete_worker_fences: BTreeMap<WorkerId, SubjectDeleteWorkerFence>,
subject_create_recovery_fences: BTreeMap<WorkerId, SubjectCreateRecoveryFence>,
subject_release_operations: BTreeMap<
(WorkerId, String),
@@ -5686,6 +5689,7 @@ impl RuntimeState {
next_diagnostic_id: 1,
workers: BTreeMap::new(),
unreconciled_worker_writes: Default::default(),
subject_delete_worker_fences: BTreeMap::new(),
subject_create_recovery_fences: BTreeMap::new(),
subject_release_operations: BTreeMap::new(),
workdir_operations: BTreeMap::new(),
@@ -5722,6 +5726,7 @@ impl RuntimeState {
next_diagnostic_id: 1,
workers: BTreeMap::new(),
unreconciled_worker_writes: Default::default(),
subject_delete_worker_fences: BTreeMap::new(),
subject_create_recovery_fences: BTreeMap::new(),
subject_release_operations: BTreeMap::new(),
workdir_operations: BTreeMap::new(),
@@ -5773,6 +5778,7 @@ impl RuntimeState {
next_diagnostic_id,
workers,
unreconciled_worker_writes: Default::default(),
subject_delete_worker_fences: persisted.subject_delete_worker_fences,
subject_create_recovery_fences: persisted.subject_create_recovery_fences,
subject_release_operations: persisted
.subject_releases
@@ -5804,6 +5810,7 @@ impl RuntimeState {
#[cfg(feature = "fs-store")]
fn persisted_state(&self) -> PersistedRuntimeState {
PersistedRuntimeState {
subject_delete_worker_fences: self.subject_delete_worker_fences.clone(),
subject_create_recovery_fences: self.subject_create_recovery_fences.clone(),
subject_releases: self.subject_release_operations.values().cloned().collect(),
display_name: self.display_name.clone(),
@@ -6027,7 +6034,9 @@ impl RuntimeState {
}
fn ensure_subject_create_not_fenced(&self, worker_id: WorkerId) -> Result<(), RuntimeError> {
if self.subject_create_recovery_fences.contains_key(&worker_id) {
if self.subject_create_recovery_fences.contains_key(&worker_id)
|| self.subject_delete_worker_fences.contains_key(&worker_id)
{
return Err(RuntimeError::InvalidRequest(
"Subject create identity is irreversibly fenced".into(),
));
@@ -6040,6 +6049,14 @@ impl RuntimeState {
worker: &WorkerRef,
allow_disconnected: bool,
) -> Result<(), RuntimeError> {
if allow_disconnected
&& self
.subject_delete_worker_fences
.get(&worker.worker_id)
.is_some_and(|fence| fence.response.is_some())
{
return Ok(());
}
self.ensure_subject_create_not_fenced(worker.worker_id)?;
use runtime_api::SubjectReleaseState as State;
if self
@@ -6110,6 +6127,13 @@ impl RuntimeState {
}
fn forget_workspace_owner_if_unused(&mut self, workspace_id: &str) -> bool {
if self
.subject_delete_worker_fences
.values()
.any(|fence| fence.request.workspace_id == workspace_id)
{
return false;
}
if self
.subject_create_recovery_fences
.values()
@@ -8767,6 +8791,8 @@ mod tests {
#[derive(Default)]
struct TestExecutionBackend {
deletion_confirmed: AtomicBool,
deletion_checks: std::sync::atomic::AtomicUsize,
recovery_absent: AtomicBool,
recovery_checks: std::sync::atomic::AtomicUsize,
spawn_result: Mutex<Option<WorkerExecutionSpawnResult>>,
@@ -8848,6 +8874,23 @@ mod tests {
}
impl WorkerExecutionBackend for TestExecutionBackend {
fn preflight_subject_delete_worker(
&self,
_request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
Ok(())
}
fn subject_delete_worker(
&self,
_request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
self.deletion_checks.fetch_add(1, Ordering::SeqCst);
if self.deletion_confirmed.load(Ordering::SeqCst) {
Ok(())
} else {
Err("shutdown is unconfirmed".into())
}
}
fn recover_subject_create(
&self,
_request: &runtime_api::SubjectCreateRecoveryRequest,
@@ -14074,7 +14117,7 @@ mod tests {
})
.unwrap_err();
assert!(error.to_string().contains(
"unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, or 14"
"unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, 14, or 15"
));
let _ = std::fs::remove_dir_all(root);
@@ -14259,6 +14302,8 @@ mod tests {
let _ = std::fs::remove_dir_all(root);
}
include!("runtime/subject_delete_tests.rs");
#[cfg(feature = "fs-store")]
#[test]
fn subject_create_recovery_is_durable_exact_and_rechecks_absence() {
@@ -0,0 +1,168 @@
use super::*;
/// Permanent exclusion survives WorkerRemoval. A receipt never substitutes for a
/// fresh execution reconciliation on exact retry, including after Runtime reopen.
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct SubjectDeleteWorkerFence {
pub(crate) request: runtime_api::SubjectDeleteWorkerRequest,
pub(crate) response: Option<runtime_api::SubjectDeleteWorkerResponse>,
}
impl Runtime {
pub fn subject_delete_worker_scoped(
&self,
scope: &RuntimeWorkspaceScope,
request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, RuntimeError> {
if request.workspace_id != scope.workspace_id
|| request.create_fingerprint.trim().is_empty()
|| request.operation_id.trim().is_empty()
|| request.create_fingerprint.len() > 512
|| request.operation_id.len() > 512
{
return Err(RuntimeError::InvalidRequest(
"invalid Subject delete Worker identity".into(),
));
}
let lease = self.worker_operation_lock(request.worker_id)?;
let _operation = lease.lock().map_err(|_| RuntimeError::StatePoisoned)?;
let worker_ref = WorkerRef::new(request.worker_id);
let backend = {
let mut state = self.lock()?;
state.ensure_running()?;
if state.backend == RuntimeBackendKind::Memory {
return Err(RuntimeError::InvalidRequest(
"Subject delete requires durable storage".into(),
));
}
state.ensure_workspace_owner(scope, false)?;
if state.workers.contains_key(&request.worker_id) {
state.refresh_uncertain_worker_write(&worker_ref)?;
let worker = state.worker(&worker_ref)?;
if worker.workspace_id.as_deref() != Some(request.workspace_id.as_str())
|| worker.request.as_ref().is_none_or(|create| {
create.create_fingerprint != request.create_fingerprint
})
{
return Err(RuntimeError::InvalidRequest(
"Subject delete Worker identity mismatch".into(),
));
}
}
if let Some(recovery) = state.subject_create_recovery_fences.get(&request.worker_id)
&& (recovery.request.workspace_id != request.workspace_id
|| recovery.request.create_fingerprint != request.create_fingerprint)
{
return Err(RuntimeError::InvalidRequest(
"Subject recovery identity mismatch".into(),
));
}
if state.subject_delete_worker_fences.values().any(|fence| {
fence.request.workspace_id == request.workspace_id
&& fence.request.operation_id == request.operation_id
&& fence.request.worker_id != request.worker_id
}) {
return Err(RuntimeError::InvalidRequest(
"Subject delete operation reused for another Worker".into(),
));
}
state.execution_backend.clone()
};
// Unpublished execution identity is checked without holding the catalog
// mutex. The Worker operation lock excludes Runtime create/restore.
if let Some(backend) = backend.as_ref() {
backend
.preflight_subject_delete_worker(&request)
.map_err(RuntimeError::InvalidRequest)?;
}
{
let mut state = self.lock()?;
state.ensure_workspace_owner(scope, false)?;
if state.subject_delete_worker_fences.values().any(|fence| {
fence.request.workspace_id == request.workspace_id
&& fence.request.operation_id == request.operation_id
&& fence.request.worker_id != request.worker_id
}) {
return Err(RuntimeError::InvalidRequest(
"Subject delete operation reused for another Worker".into(),
));
}
if let Some(saved) = state.subject_delete_worker_fences.get(&request.worker_id) {
if saved.request != request {
return Err(RuntimeError::InvalidRequest(
"different Subject delete Worker request retained".into(),
));
}
} else {
state.ensure_workspace_owner(scope, true)?;
state.subject_delete_worker_fences.insert(
request.worker_id,
SubjectDeleteWorkerFence {
request: request.clone(),
response: None,
},
);
}
// Keep exclusion on persistence uncertainty; never start execution
// cleanup until this durable write has been confirmed.
state.persist_runtime_snapshot()?;
}
let backend = backend.ok_or_else(|| {
RuntimeError::InvalidRequest(
"Subject Worker shutdown cannot be confirmed without execution backend".into(),
)
})?;
backend
.subject_delete_worker(&request)
.map_err(RuntimeError::InvalidRequest)?;
let mut state = self.lock()?;
if let Some(mut worker) = state.workers.get(&request.worker_id).cloned() {
let restore_receipt =
RuntimeWorkerRestoreResult::superseded_by_stop(worker.pending_restore);
worker.pending_stop = None;
worker.pending_restore = None;
if let Some(context) = worker.restore_candidate_context.take() {
context.discard_candidate();
}
worker.restore_guard.rotate();
worker.restore_guard.finish(restore_receipt);
worker.status = WorkerStatus::Stopped;
worker.worker_state = None;
worker.restore_intent = WorkerRestoreIntent::Explicit;
worker.internal_workers.clear();
worker.execution_bound = false;
state.persist_worker_record(&worker)?;
state.workers.insert(request.worker_id, worker);
state.publish_worker_upsert(request.worker_id)?;
}
// This is storage-owner evidence under the same lock that permanently
// excludes create/restore, not an inference from catalog projection.
#[cfg(feature = "fs-store")]
let originals_absent = !state.workers.contains_key(&request.worker_id)
&& state
.fs_store()
.ok_or_else(|| {
RuntimeError::InvalidRequest(
"Subject original absence requires fs storage".into(),
)
})?
.worker_originals_absent(request.worker_id)?;
#[cfg(not(feature = "fs-store"))]
let originals_absent = false;
let response = runtime_api::SubjectDeleteWorkerResponse {
workspace_id: request.workspace_id.clone(),
worker_id: request.worker_id,
create_fingerprint: request.create_fingerprint.clone(),
operation_id: request.operation_id.clone(),
originals_absent,
};
state
.subject_delete_worker_fences
.get_mut(&request.worker_id)
.expect("delete fence retained under Worker operation lock")
.response = Some(response.clone());
state.persist_runtime_snapshot()?;
Ok(response)
}
}
@@ -0,0 +1,274 @@
// Runtime component tests use real filesystem persistence and replace only the
// external execution boundary. Actual Controller shutdown is tested by the adapter.
#[cfg(feature = "fs-store")]
#[test]
fn subject_delete_reserved_id_survives_unknown_reopen_and_rejects_identity_changes() {
let root = tempfile::tempdir().unwrap();
let options =
crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime");
let scoped = scope("workspace-a", "server-a");
let create = scoped_task_request("delayed create", "workspace-a");
let request = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: scoped.workspace_id.clone(),
worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint.clone(),
operation_id: "subject-delete-1".into(),
};
let backend = Arc::new(TestExecutionBackend::default());
let runtime =
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
assert!(
runtime
.subject_delete_worker_scoped(&scoped, request.clone())
.is_err()
);
assert!(
runtime.lock().unwrap().subject_delete_worker_fences[&create.worker_id]
.response
.is_none()
);
assert!(
runtime
.create_worker_scoped(&scoped, create.clone())
.is_err()
);
assert_eq!(backend.deletion_checks.load(Ordering::SeqCst), 1);
drop(runtime);
let runtime =
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
assert!(
runtime
.create_worker_scoped(&scoped, create.clone())
.is_err()
);
assert!(
runtime
.subject_delete_worker_scoped(&scoped, request.clone())
.is_err()
);
backend.deletion_confirmed.store(true, Ordering::SeqCst);
let receipt = runtime
.subject_delete_worker_scoped(&scoped, request.clone())
.unwrap();
let mut expected = serde_json::to_value(&request).unwrap();
expected["originals_absent"] = true.into();
assert_eq!(serde_json::to_value(&receipt).unwrap(), expected);
for index in 0..4 {
let mut changed = request.clone();
match index {
0 => changed.worker_id = WorkerId::now_v7(),
1 => changed.operation_id.push_str("-changed"),
2 => changed.create_fingerprint.push_str("-changed"),
_ => changed.workspace_id.push_str("-changed"),
}
assert!(
runtime
.subject_delete_worker_scoped(&scoped, changed)
.is_err()
);
}
assert!(
runtime
.subject_delete_worker_scoped(&scope("workspace-a", "server-b"), request.clone())
.is_err()
);
assert!(
runtime
.subject_delete_worker_scoped(&scope("workspace-b", "server-a"), request.clone())
.is_err()
);
assert!(
!runtime
.lock()
.unwrap()
.forget_workspace_owner_if_unused("workspace-a")
);
drop(runtime);
// Even a saved receipt must not manufacture fresh shutdown proof on reopen.
let backend = Arc::new(TestExecutionBackend::default());
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
assert!(
runtime
.subject_delete_worker_scoped(&scoped, request.clone())
.is_err()
);
backend.deletion_confirmed.store(true, Ordering::SeqCst);
assert_eq!(
runtime
.subject_delete_worker_scoped(&scoped, request)
.unwrap(),
receipt
);
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
}
#[cfg(feature = "fs-store")]
#[test]
fn subject_delete_existing_stopped_or_active_worker_preserves_record_and_allows_canonical_removal()
{
for stopped in [false, true] {
let root = tempfile::tempdir().unwrap();
let backend = Arc::new(TestExecutionBackend::default());
backend.deletion_confirmed.store(true, Ordering::SeqCst);
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path())
.with_runtime_id("test-runtime");
let runtime =
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let scoped = scope("workspace-a", "server-a");
let create = scoped_task_request("subject body", "workspace-a");
let worker = runtime
.create_worker_scoped(&scoped, create.clone())
.unwrap();
if stopped {
runtime.stop_worker(&worker.worker_ref, None).unwrap();
}
let request = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: scoped.workspace_id.clone(),
worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint.clone(),
operation_id: "delete-body".into(),
};
let mut mismatch = request.clone();
mismatch.create_fingerprint.push_str("-wrong");
assert!(
runtime
.subject_delete_worker_scoped(&scoped, mismatch)
.is_err()
);
let mut mismatch = request.clone();
mismatch.workspace_id = "workspace-b".into();
assert!(
runtime
.subject_delete_worker_scoped(&scope("workspace-b", "server-b"), mismatch)
.is_err()
);
assert!(
!runtime
.lock()
.unwrap()
.subject_delete_worker_fences
.contains_key(&create.worker_id)
);
// A lost Worker settlement cannot be reported as completed deletion.
runtime
.test_runtime_store()
.fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename);
assert!(
runtime
.subject_delete_worker_scoped(&scoped, request.clone())
.is_err()
);
let receipt = runtime
.subject_delete_worker_scoped(&scoped, request.clone())
.unwrap();
assert!(!receipt.originals_absent, "a stopped catalog record still has original material");
assert_eq!(
runtime.worker_detail(&worker.worker_ref).unwrap().status,
WorkerStatus::Stopped
);
assert!(
runtime
.restore_worker(
&worker.worker_ref,
runtime.test_restore_request(&worker.worker_ref)
)
.is_err()
);
assert!(
runtime
.create_worker_scoped(&scoped, create.clone())
.is_err()
);
let record = runtime
.test_runtime_store()
.read_worker_record(&worker.worker_ref)
.unwrap();
assert_eq!(record.worker_id, worker.worker_ref.worker_id);
drop(runtime);
let runtime =
Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap();
assert!(
runtime
.restore_worker(
&worker.worker_ref,
runtime.test_restore_request(&worker.worker_ref)
)
.is_err()
);
assert_eq!(
runtime
.subject_delete_worker_scoped(&scoped, request)
.unwrap(),
receipt
);
runtime
.delete_worker_scoped(&scoped, &worker.worker_ref)
.unwrap();
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
assert!(
!runtime
.lock()
.unwrap()
.forget_workspace_owner_if_unused("workspace-a")
);
}
}
#[cfg(feature = "fs-store")]
#[test]
fn subject_delete_unavailable_backend_never_issues_receipt_but_preserves_exclusion() {
let root = tempfile::tempdir().unwrap();
let options =
crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime");
let scoped = scope("workspace-a", "server-a");
let create = scoped_task_request("reserved without backend", "workspace-a");
let request = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: scoped.workspace_id.clone(),
worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint.clone(),
operation_id: "delete-offline".into(),
};
let runtime = Runtime::with_fs_store(options.clone()).unwrap();
assert!(
runtime
.subject_delete_worker_scoped(&scoped, request.clone())
.is_err()
);
assert!(
runtime.lock().unwrap().subject_delete_worker_fences[&create.worker_id]
.response
.is_none()
);
drop(runtime);
let backend = Arc::new(TestExecutionBackend::default());
backend.deletion_confirmed.store(true, Ordering::SeqCst);
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend).unwrap();
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
runtime
.subject_delete_worker_scoped(&scoped, request)
.unwrap();
}
#[cfg(feature = "fs-store")]
#[test]
fn subject_delete_missing_catalog_does_not_claim_residual_originals_absent() {
let root = tempfile::tempdir().unwrap();
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("test-runtime");
let backend = Arc::new(TestExecutionBackend::default());
backend.deletion_confirmed.store(true, Ordering::SeqCst);
let runtime = Runtime::with_fs_store_and_execution_backend(options, backend).unwrap();
let scoped = scope("workspace-a", "server-a");
let create = scoped_task_request("unpublished original", "workspace-a");
let original = root.path().join("workers").join(create.worker_id.to_string()).join("session/original.jsonl");
std::fs::create_dir_all(original.parent().unwrap()).unwrap();
std::fs::write(&original, "preserve original text").unwrap();
let receipt = runtime.subject_delete_worker_scoped(&scoped, runtime_api::SubjectDeleteWorkerRequest {
workspace_id: scoped.workspace_id.clone(), worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint.clone(), operation_id: "delete-unpublished".into(),
}).unwrap();
assert!(!receipt.originals_absent);
assert_eq!(std::fs::read_to_string(original).unwrap(), "preserve original text");
assert!(runtime.create_worker_scoped(&scoped, create).is_err());
}
@@ -0,0 +1,285 @@
// Integration of Runtime's durable fence with the real adapter and Controller;
// MockFactory replaces only model/provider/profile construction for a hanging run.
#[test]
#[cfg(feature = "fs-store")]
#[serial_test::serial(worker_allocation)]
fn subject_delete_running_controller_confirms_shutdown_and_preserves_history() {
let root = tempfile::tempdir().unwrap();
let client = MockClient::sequential(vec![
MockResponse::Complete(simple_text_events()),
MockResponse::Hang(vec![]),
]);
let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(MockFactory {
client: client.clone(),
runtime_base: root.path().join("control"),
cwd: root.path().to_path_buf(),
store_dir: root.path().join("sessions"),
worker_metadata_dir: root.path().join("metadata"),
observed_cwds: Arc::default(),
observed_workspace_clients: Arc::default(),
})
.unwrap(),
);
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
crate::fs_store::FsRuntimeStoreOptions::new(root.path().join("catalog"))
.with_runtime_id("delete-test"),
backend.clone(),
)
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "server-test");
let mut create = create_request("running Subject body");
create.workspace_api = Some(WorkspaceApiRef {
workspace_id: scope.workspace_id.clone(),
base_url: "https://workspace.invalid".into(),
});
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: scope.workspace_id.clone(),
language: "English".into(),
});
let worker = runtime
.create_worker_scoped(&scope, create.clone())
.unwrap();
runtime
.send_input(&worker.worker_ref, WorkerInput::user("settled history"))
.unwrap();
let execution = backend.workers.lock().unwrap()[&worker.worker_ref].clone();
backend
.run_on_adapter_runtime({
let handle = execution.handle.clone();
let client = client.clone();
async move {
tokio::time::timeout(Duration::from_secs(5), async {
while client.call_count.load(Ordering::SeqCst) < 1
|| handle.shared_state.catalog_status() != WorkerStatus::Idle
{
tokio::task::yield_now().await;
}
})
.await
.map_err(|error| error.to_string())?;
Ok(())
}
})
.unwrap();
let before = persisted_files(&root.path().join("sessions"));
assert!(!before.is_empty());
runtime
.send_input(&worker.worker_ref, WorkerInput::user("run until stopped"))
.unwrap();
backend
.run_on_adapter_runtime({
let handle = execution.handle.clone();
let client = client.clone();
async move {
tokio::time::timeout(Duration::from_secs(5), async {
while client.call_count.load(Ordering::SeqCst) < 2 {
tokio::task::yield_now().await;
}
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Running);
})
.await
.map_err(|error| error.to_string())?;
Ok(())
}
})
.unwrap();
let request = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: scope.workspace_id.clone(),
worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint.clone(),
operation_id: "delete-running".into(),
};
runtime
.subject_delete_worker_scoped(&scope, request.clone())
.unwrap();
assert!(execution.handle.protocol_is_closed());
assert!(execution.tasks.tasks.lock().unwrap().is_empty());
assert!(backend.workers.lock().unwrap().is_empty());
assert!(backend.pending_restores.lock().unwrap().is_empty());
assert_eq!(
runtime.worker_detail(&worker.worker_ref).unwrap().status,
crate::catalog::WorkerStatus::Stopped
);
// Preserve the settled prefix. Ordinary Controller empty-turn rollback remains unchanged.
let after = persisted_files(&root.path().join("sessions"));
for (path, bytes) in before {
let entries = |bytes: &[u8]| {
bytes
.split(|b| *b == b'\n')
.filter(|line| !line.is_empty())
.map(|line| serde_json::from_slice::<serde_json::Value>(line).unwrap())
.collect::<Vec<_>>()
};
assert!(
entries(&after[&path]).starts_with(&entries(&bytes)),
"{path:?} BEFORE={:?} AFTER={:?}",
entries(&bytes),
entries(&after[&path])
);
}
assert!(runtime.create_worker_scoped(&scope, create).is_err());
assert!(
runtime
.restore_worker(
&worker.worker_ref,
runtime.test_restore_request(&worker.worker_ref)
)
.is_err()
);
runtime
.subject_delete_worker_scoped(&scope, request)
.unwrap();
assert_eq!(client.call_count.load(Ordering::SeqCst), 2);
}
#[test]
#[serial_test::serial(worker_allocation)]
#[cfg(feature = "http-server")]
fn subject_delete_unregistered_profile_controller_retries_unknown_shutdown_without_erasing_session()
{
let root = tempfile::tempdir().unwrap();
let (backend, request) = profile_stop_fixture(root.path());
let execution = backend.workers.lock().unwrap()[&request.worker_ref].clone();
let deletion = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: request
.workspace_scope
.as_ref()
.unwrap()
.workspace_id
.clone(),
worker_id: request.worker_ref.worker_id,
create_fingerprint: request.request.create_fingerprint.clone(),
operation_id: "delete-unregistered".into(),
};
let mut bad = deletion.clone();
bad.create_fingerprint.push_str("-wrong");
assert!(backend.preflight_subject_delete_worker(&bad).is_err());
assert!(backend.subject_delete_worker(&bad).is_err());
let mut bad = deletion.clone();
bad.workspace_id.push_str("-wrong");
assert!(backend.subject_delete_worker(&bad).is_err());
assert!(
backend.admissions.lock().unwrap()[&request.worker_ref]
.deletion
.is_none()
);
let preparing = execution.clone();
backend
.run_on_adapter_runtime(async move {
prepare_rejected_shutdown(&preparing, false, None).await;
Ok(())
})
.unwrap();
let session_root = root
.path()
.join("runtime/workers")
.join(request.worker_ref.worker_id.to_string())
.join("session");
let before = persisted_files(&session_root);
assert!(!before.is_empty());
assert!(
backend
.subject_delete_worker(&deletion)
.unwrap_err()
.contains("Conflict")
);
assert!(
backend
.workers
.lock()
.unwrap()
.contains_key(&request.worker_ref)
);
assert!(matches!(
backend.restore_worker(request.clone()),
WorkerExecutionSpawnResult::Rejected(_)
));
backend.subject_delete_worker(&deletion).unwrap();
backend.subject_delete_worker(&deletion).unwrap();
assert!(execution.handle.protocol_is_closed());
assert!(execution.tasks.tasks.lock().unwrap().is_empty());
assert!(backend.workers.lock().unwrap().is_empty());
assert_eq!(persisted_files(&session_root), before);
assert!(backend.preflight_restore(&request).is_err());
assert!(
backend
.activate_restored_worker(request.operation_id, &request.worker_ref)
.is_err()
);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn subject_delete_pending_restore_fences_late_controller_and_never_restarts_it() {
let root = tempfile::tempdir().unwrap();
let restores = Arc::new(AtomicUsize::new(0));
let release = Arc::new(tokio::sync::Notify::new());
let handles = Arc::new(Mutex::new(Vec::new()));
let backend = WorkerRuntimeExecutionBackend::new(LateControllerFactory {
inner: MockFactory {
client: MockClient::new(vec![]),
runtime_base: root.path().join("control"),
cwd: root.path().to_path_buf(),
store_dir: root.path().join("sessions"),
worker_metadata_dir: root.path().join("metadata"),
observed_cwds: Arc::default(),
observed_workspace_clients: Arc::default(),
},
restores: restores.clone(),
release: release.clone(),
actual_handles: handles.clone(),
})
.unwrap()
.with_spawn_restore_timeout(Duration::from_millis(1));
let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7()));
request.workspace_scope = Some(crate::runtime::RuntimeWorkspaceScope::new(
"ws-test",
"server-test",
));
request.request.workspace_api = Some(WorkspaceApiRef {
workspace_id: "ws-test".into(),
base_url: "https://workspace.invalid".into(),
});
assert!(matches!(
backend.restore_worker(request.clone()),
WorkerExecutionSpawnResult::ReconciliationRequired { .. }
));
let deletion = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: "ws-test".into(),
worker_id: request.worker_ref.worker_id,
create_fingerprint: request.request.create_fingerprint.clone(),
operation_id: "delete-pending".into(),
};
// Unfinished factory ownership is unknown, never shutdown success.
assert!(
backend
.subject_delete_worker(&deletion)
.unwrap_err()
.contains("pending")
);
assert!(
backend
.pending_restores
.lock()
.unwrap()
.contains_key(&request.worker_ref)
);
assert!(
backend
.activate_restored_worker(request.operation_id, &request.worker_ref)
.is_err()
);
release.notify_one();
backend.subject_delete_worker(&deletion).unwrap();
assert!(backend.pending_restores.lock().unwrap().is_empty());
assert!(backend.workers.lock().unwrap().is_empty());
assert_eq!(restores.load(Ordering::SeqCst), 1);
assert_eq!(handles.lock().unwrap().len(), 1);
assert!(handles.lock().unwrap()[0].protocol_is_closed());
assert!(matches!(
backend.restore_worker(request),
WorkerExecutionSpawnResult::Rejected(_)
));
}
@@ -0,0 +1,101 @@
// Actual create factory future remains behind Runtime's Worker operation lock;
// deletion must wait for that owner, then permanently reject the delayed retry.
struct SubjectDeleteDeferredCreate {
entered: mpsc::Sender<()>,
release: Arc<tokio::sync::Notify>,
}
#[async_trait]
impl RuntimeWorkerFactory for SubjectDeleteDeferredCreate {
async fn prepare_restore(
&self,
_: WorkerExecutionRestoreRequest,
) -> Result<Box<dyn RuntimeRestoreCandidate>, RuntimeRestoreFailure> {
panic!("fenced create must never restore")
}
async fn spawn_controller(
&self,
_: WorkerExecutionSpawnRequest,
) -> Result<RuntimeWorkerController, String> {
self.entered.send(()).unwrap();
self.release.notified().await;
Err("create failed before any Controller".into())
}
}
#[test]
fn subject_delete_serializes_inflight_create_and_rejects_all_late_admission() {
let root = tempfile::tempdir().unwrap();
let (entered, entry) = mpsc::channel();
let release = Arc::new(tokio::sync::Notify::new());
let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(SubjectDeleteDeferredCreate {
entered,
release: release.clone(),
})
.unwrap(),
);
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
crate::fs_store::FsRuntimeStoreOptions::new(root.path()).with_runtime_id("delete-test"),
backend.clone(),
)
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "server-test");
let mut create = create_request("inflight create");
create.workspace_api = Some(WorkspaceApiRef {
workspace_id: scope.workspace_id.clone(),
base_url: "https://workspace.invalid".into(),
});
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: scope.workspace_id.clone(),
language: "English".into(),
});
let creating = {
let runtime = runtime.clone();
let scope = scope.clone();
let create = create.clone();
std::thread::spawn(move || runtime.create_worker_scoped(&scope, create))
};
entry.recv_timeout(Duration::from_secs(5)).unwrap();
let request = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: scope.workspace_id.clone(),
worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint.clone(),
operation_id: "delete-creating".into(),
};
let deleting = {
let runtime = runtime.clone();
let scope = scope.clone();
let request = request.clone();
std::thread::spawn(move || runtime.subject_delete_worker_scoped(&scope, request))
};
release.notify_one();
assert!(creating.join().unwrap().is_err());
deleting.join().unwrap().unwrap();
assert!(
runtime
.create_worker_scoped(&scope, create.clone())
.is_err()
);
assert!(backend.workers.lock().unwrap().is_empty());
let worker_ref = WorkerRef::new(create.worker_id);
let spawn = WorkerExecutionSpawnRequest {
worker_ref: worker_ref.clone(),
request: create.clone(),
workspace_scope: Some(scope.clone()),
context: test_execution_context(worker_ref.clone()),
workdir_attachments: BTreeMap::new(),
config_bundle: None,
};
assert!(matches!(
backend.spawn_worker(spawn),
WorkerExecutionSpawnResult::Rejected(_)
));
let mut restore = direct_restore_request(worker_ref);
restore.request = create;
restore.workspace_scope = Some(scope);
assert!(matches!(
backend.restore_worker(restore),
WorkerExecutionSpawnResult::Rejected(_)
));
}
@@ -0,0 +1,104 @@
#[test]
fn subject_delete_schema14_migration_adds_exclusion_store_without_changing_worker_or_receipts() {
for with_fence in [false, true] {
let (_temp, store, worker, path) = canonical_worker_store();
let root = store.root.clone();
let mut legacy =
serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap();
legacy["schema_version"] = serde_json::json!(RESTORE_REPAIR_SCHEMA_VERSION);
fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap();
let id = WorkerId::now_v7();
let request = serde_json::json!({"workspace_id": "workspace-a", "worker_id": id,
"create_fingerprint": "fenced-create", "operation_id": "delete-1"});
let mut response = request.clone();
response["originals_absent"] = false.into();
let fence =
serde_json::json!({ id.to_string(): {"request": request, "response": response} });
let mut runtime = serde_json::json!({"schema_version": RESTORE_REPAIR_SCHEMA_VERSION,
"display_name": "retained", "backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
"workspace_owners": {"workspace-a": "server-a"}, "diagnostics": [],
"config_bundles": {}, "subject_create_recovery_fences": {}, "subject_releases": []});
if with_fence {
runtime["subject_delete_worker_fences"] = fence.clone();
}
fs::write(
root.join(RUNTIME_FILE),
serde_json::to_vec(&runtime).unwrap(),
)
.unwrap();
drop(store);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("test-runtime");
let before = fs::read(&path).unwrap();
assert!(
FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
assert_eq!(fs::read(&path).unwrap(), before);
let opened = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap();
legacy["schema_version"] = serde_json::json!(SCHEMA_VERSION);
assert_eq!(
read_json::<serde_json::Value>(&path, "read migrated Worker").unwrap(),
legacy
);
let expected = if with_fence {
fence
} else {
serde_json::json!({})
};
assert_eq!(
serde_json::to_value(&opened.state.as_ref().unwrap().subject_delete_worker_fences)
.unwrap(),
expected
);
opened
.store
.write_runtime_snapshot(opened.state.as_ref().unwrap())
.unwrap();
drop(opened);
let reopened = FsRuntimeStore::open_or_create(root, "test-runtime").unwrap();
assert_eq!(
serde_json::to_value(&reopened.state.unwrap().subject_delete_worker_fences).unwrap(),
expected
);
}
}
#[test]
fn subject_delete_store_rejects_missing_exclusion_field_malformed_receipt_and_workspace_owner() {
for invalid in 0..4 {
let (_temp, store, _worker, _path) = canonical_worker_store();
let root = store.root.clone();
let id = WorkerId::now_v7();
let request = serde_json::json!({"workspace_id": "workspace-a", "worker_id": id,
"create_fingerprint": "fenced-create", "operation_id": "delete-1"});
let mut response = request.clone();
response["originals_absent"] = false.into();
let mut runtime = serde_json::json!({"schema_version": SCHEMA_VERSION,
"display_name": "retained", "backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
"workspace_owners": {"workspace-a": "server-a"}, "diagnostics": [],
"subject_delete_worker_fences": { id.to_string(): {"request": request, "response": response} }});
match invalid {
0 => {
runtime
.as_object_mut()
.unwrap()
.remove("subject_delete_worker_fences");
}
1 => {
runtime["subject_delete_worker_fences"][id.to_string()]["response"]["create_fingerprint"] =
"different".into()
}
2 => runtime["workspace_owners"] = serde_json::json!({}),
_ => {
runtime["subject_delete_worker_fences"][id.to_string()]["request"]["worker_id"] =
WorkerId::now_v7().to_string().into()
}
}
let bytes = serde_json::to_vec(&runtime).unwrap();
fs::write(root.join(RUNTIME_FILE), &bytes).unwrap();
drop(store);
assert!(FsRuntimeStore::open_or_create(root.clone(), "test-runtime").is_err());
assert_eq!(fs::read(root.join(RUNTIME_FILE)).unwrap(), bytes);
}
}
+3 -1
View File
@@ -215,7 +215,8 @@ impl StoppedWorkerTemplate {
snapshot
.validate(&root.join(RUNTIME_FILE))
.map_err(|_| reject("unsupported Runtime schema"))?;
if !snapshot.subject_create_recovery_fences.is_empty()
if !snapshot.subject_delete_worker_fences.is_empty()
|| !snapshot.subject_create_recovery_fences.is_empty()
|| !snapshot.subject_releases.is_empty()
{
return Err(reject("source has lifecycle operations"));
@@ -427,6 +428,7 @@ impl StoppedWorkerTemplate {
)?;
self.validate_session(&session)?;
store.write_runtime_snapshot(&PersistedRuntimeState {
subject_delete_worker_fences: BTreeMap::new(),
subject_create_recovery_fences: BTreeMap::new(),
subject_releases: Vec::new(),
// Match normal store initialization: this is catalog availability,
+176
View File
@@ -3415,6 +3415,14 @@ struct SubjectControllerReservation {
state: runtime_api::SubjectReleaseState,
}
// Admission identity is retained even when execution publication or stop settlement
// failed. Runtime owns durable exclusion; this guards the adapter's delayed calls.
struct WorkerAdmission {
workspace_id: Option<String>,
create_fingerprint: String,
deletion: Option<runtime_api::SubjectDeleteWorkerRequest>,
}
/// Worker execution backend backed by real worker crate Workers.
pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
#[cfg(test)]
@@ -3438,6 +3446,7 @@ pub struct WorkerRuntimeExecutionBackend<F = ProfileRuntimeWorkerFactory> {
>,
>,
worker_locks: Mutex<HashMap<WorkerRef, Arc<Mutex<()>>>>,
admissions: Mutex<HashMap<WorkerRef, WorkerAdmission>>,
spawn_restore_timeout: Duration,
subject_reservations: Mutex<HashMap<(WorkerRef, String), SubjectControllerReservation>>,
shutdown_admission_timeout: Duration,
@@ -3464,12 +3473,92 @@ where
pending_restores: Mutex::new(HashMap::new()),
completed_restore_failures: Mutex::new(HashMap::new()),
worker_locks: Mutex::new(HashMap::new()),
admissions: Mutex::new(HashMap::new()),
spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT,
subject_reservations: Mutex::new(HashMap::new()),
shutdown_admission_timeout: Duration::from_secs(5),
})
}
// Caller holds the adapter Worker lock.
fn admit_worker(
&self,
worker: &WorkerRef,
workspace: Option<&str>,
fingerprint: &str,
) -> Result<(), String> {
let mut admissions = self
.admissions
.lock()
.map_err(|_| "worker admission lock poisoned")?;
if let Some(saved) = admissions.get(worker) {
if saved.deletion.is_some() {
return Err("Subject Worker permanently excluded".into());
}
if saved.workspace_id.as_deref() != workspace || saved.create_fingerprint != fingerprint
{
return Err("Worker admission identity mismatch".into());
}
} else {
admissions.insert(
worker.clone(),
WorkerAdmission {
workspace_id: workspace.map(str::to_string),
create_fingerprint: fingerprint.into(),
deletion: None,
},
);
}
Ok(())
}
fn validate_subject_delete_identity(
&self,
request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
let admissions = self
.admissions
.lock()
.map_err(|_| "worker admission lock poisoned")?;
if let Some(saved) = admissions.get(&WorkerRef::new(request.worker_id)) {
if saved.workspace_id.as_deref() != Some(request.workspace_id.as_str())
|| saved.create_fingerprint != request.create_fingerprint
|| saved
.deletion
.as_ref()
.is_some_and(|saved| saved != request)
{
return Err("Subject delete execution identity mismatch".into());
}
}
if let Some(pending) = self.pending_restore(&WorkerRef::new(request.worker_id))? {
if pending
.request
.workspace_scope
.as_ref()
.map(|scope| scope.workspace_id.as_str())
!= Some(request.workspace_id.as_str())
|| pending.request.request.create_fingerprint != request.create_fingerprint
{
return Err("Subject delete pending execution identity mismatch".into());
}
}
Ok(())
}
fn ensure_not_subject_deleted(&self, worker: &WorkerRef) -> Result<(), String> {
if self
.admissions
.lock()
.map_err(|_| "worker admission lock poisoned")?
.get(worker)
.is_some_and(|admission| admission.deletion.is_some())
{
return Err("Subject Worker permanently excluded".into());
}
Ok(())
}
pub fn with_backend_id(mut self, backend_id: impl Into<String>) -> Self {
self.backend_id = backend_id.into();
self
@@ -4739,6 +4828,58 @@ where
&self.backend_id
}
fn preflight_subject_delete_worker(
&self,
request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
let worker = WorkerRef::new(request.worker_id);
let lock = self.worker_lock(&worker)?;
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
self.validate_subject_delete_identity(request)
}
fn subject_delete_worker(
&self,
request: &runtime_api::SubjectDeleteWorkerRequest,
) -> Result<(), String> {
let worker = WorkerRef::new(request.worker_id);
let lock = self.worker_lock(&worker)?;
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
self.validate_subject_delete_identity(request)?;
self.admissions
.lock()
.map_err(|_| "worker admission lock poisoned")?
.insert(
worker.clone(),
WorkerAdmission {
workspace_id: Some(request.workspace_id.clone()),
create_fingerprint: request.create_fingerprint.clone(),
deletion: Some(request.clone()),
},
);
let result = self.stop_worker_under_lock(&worker);
if !result.is_accepted() {
return Err(result.message_or_default());
}
// A live Controller and failed unpublished candidates can coexist. The
// ordinary stop's live branch joins the Controller; also reconcile any
// factory-owned cache/artifacts before issuing deletion proof.
let factory = self.factory.clone();
let stopped = worker.clone();
self.run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move {
factory.reconcile_stopped_worker(&stopped).await
})?;
self.subject_reservations
.lock()
.map_err(|_| "Subject reservations poisoned")?
.retain(|(saved, _), _| saved != &worker);
self.completed_restore_failures
.lock()
.map_err(|_| "completed restores poisoned")?
.remove(&worker);
Ok(())
}
fn recover_subject_create(
&self,
request: &runtime_api::SubjectCreateRecoveryRequest,
@@ -5178,6 +5319,19 @@ where
));
}
};
if let Err(message) = self.admit_worker(
&request.worker_ref,
request
.workspace_scope
.as_ref()
.map(|scope| scope.workspace_id.as_str()),
&request.request.create_fingerprint,
) {
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected(
WorkerExecutionOperation::Spawn,
message,
));
}
match self.pending_restore(&request.worker_ref) {
Ok(None) => {}
Ok(Some(_)) => {
@@ -5405,6 +5559,10 @@ where
));
}
};
self.ensure_not_subject_deleted(&request.worker_ref)
.map_err(|message| {
WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, message)
})?;
if self
.pending_restore(&request.worker_ref)
.map_err(|message| {
@@ -5499,6 +5657,19 @@ where
));
}
};
if let Err(message) = self.admit_worker(
&request.worker_ref,
request
.workspace_scope
.as_ref()
.map(|scope| scope.workspace_id.as_str()),
&request.request.create_fingerprint,
) {
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected(
WorkerExecutionOperation::Restore,
message,
));
}
if let Some(result) =
self.completed_restore_failure(request.operation_id, &request.worker_ref)
{
@@ -6182,6 +6353,7 @@ where
return Err(message);
}
};
self.ensure_not_subject_deleted(worker_ref)?;
if let Some(pending) = self.pending_restore(worker_ref)? {
if pending.request.operation_id != operation_id {
return Err("restored Worker operation identity does not match".into());
@@ -6820,6 +6992,10 @@ mod tests {
};
use worker::PreparedWorker;
include!("subject_delete_backend_tests.rs");
#[cfg(feature = "fs-store")]
include!("subject_delete_create_tests.rs");
// Both candidate and failure suites need the same guarded, checkout-owned
// temporary root, independently of Runtime's optional filesystem catalog.
fn restore_failure_tempdir() -> tempfile::TempDir {
@@ -478,7 +478,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
(101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into()),
(103, "Passkey metadata and atomic addition results".into()),
(104, "Converge Passkey results and Job input retention".into())
(104, "Converge Passkey results and Job input retention".into()),
(105, "Durable Subject deletion snapshots and Job admission fences".into())
]
);
for (table, rows) in retained.iter().zip(original.iter()) {
+102 -1
View File
@@ -1292,6 +1292,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
})
}
fn subject_delete_worker(
&self,
_request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
Err("Subject deletion fencing is unavailable on this Runtime".into())
}
fn recover_subject_create(
&self,
_request: runtime_api::SubjectCreateRecoveryRequest,
@@ -2269,6 +2276,21 @@ impl RuntimeRegistry {
runtime.worker_session_attachment(workspace_id, &worker_ref, session_id, attachment_id)
}
pub fn subject_delete_worker(
&self,
runtime_id: &str,
request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, RuntimeRegistryError> {
validate_backend_identifier("runtime_id", runtime_id)?;
self.runtime(runtime_id)?
.subject_delete_worker(request)
.map_err(|message| RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: runtime_id.into(),
code: "subject_delete_worker_unproven".into(),
message,
})
}
pub fn recover_subject_create(
&self,
runtime_id: &str,
@@ -3367,6 +3389,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
})
}
fn subject_delete_worker(
&self,
request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
if request.workspace_id != self.workspace_id {
return Err("Subject deletion Workspace mismatch".into());
}
let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend");
self.runtime
.subject_delete_worker_scoped(&scope, request)
.map_err(|e| e.to_string())
}
fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
@@ -4255,7 +4290,10 @@ fn workspace_runtime_operation(method: &str, path_and_query: &str) -> &'static s
}
if path.contains("/retention/")
|| (path.starts_with("/v1/workers/") && method == "DELETE")
|| (path == "/v1/subject-create-recovery" && method == "POST")
|| (matches!(
path,
"/v1/subject-create-recovery" | "/v1/subject-delete-worker"
) && method == "POST")
{
return "workers:delete";
}
@@ -5656,6 +5694,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
})
}
fn subject_delete_worker(
&self,
request: runtime_api::SubjectDeleteWorkerRequest,
) -> Result<runtime_api::SubjectDeleteWorkerResponse, String> {
self.run_runtime_api(
self.request_timeout,
MAX_REMOTE_RUNTIME_RESPONSE_BYTES,
move |client| async move { client.subject_delete_worker(request).await },
)
.map_err(|diagnostic| diagnostic.message)
}
fn recover_subject_create(
&self,
request: runtime_api::SubjectCreateRecoveryRequest,
@@ -8507,6 +8557,57 @@ mod tests {
);
}
#[test]
fn remote_subject_delete_uses_generated_exact_identity_transport() {
assert_eq!(
workspace_runtime_operation("POST", "/v1/subject-delete-worker"),
"workers:delete"
);
let request = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: "workspace-test".into(),
worker_id: EmbeddedWorkerId::now_v7(),
create_fingerprint: "sha256:reserved-create".into(),
operation_id: "delete-worker-1".into(),
};
let response = runtime_api::SubjectDeleteWorkerResponse {
originals_absent: true,
workspace_id: request.workspace_id.clone(),
worker_id: request.worker_id,
create_fingerprint: request.create_fingerprint.clone(),
operation_id: request.operation_id.clone(),
};
// Only remote transport is substituted; actual Controller shutdown and
// signed admission are covered by worker-runtime's independent suites.
let (base_url, server) = serve_mock_http(vec![mock_response(
"POST",
"/v1/subject-delete-worker",
true,
200,
serde_json::to_string(&response).unwrap(),
)]);
let registry = RuntimeRegistry::new(Vec::new());
registry.register(
RemoteWorkerRuntime::new(
RemoteRuntimeConfig::new(
"remote:delete",
"Deletion transport fixture",
base_url,
Some("secret-token".into()),
),
"workspace-test".into(),
"http://127.0.0.1:8787".into(),
)
.unwrap(),
);
assert_eq!(
registry
.subject_delete_worker("remote:delete", request)
.unwrap(),
response
);
server.join().unwrap();
}
#[test]
fn remote_subject_create_recovery_uses_generated_management_transport() {
assert_eq!(
@@ -367,14 +367,54 @@ async fn implementation_result_rejects_forged_evidence_and_rolls_back_without_no
.unwrap();
}
#[tokio::test]
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
let (dir, store) = tracking_store_fixture().await;
tracking_store_assign(&store, "delegation", "assign");
let worker = routing_worker_assign(&store, "implementation").worker;
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
store.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER notification_mr_event;
// Preserve seeded assignment evidence while rebuilding the actual pre-business
// capture composition. Changing only a receipt leaves schema105 admission fences
// and erasure guards in an otherwise legacy fixture.
fn reconstruct_implementation_schema96(conn: &Connection) -> Result<()> {
assert_eq!(current_schema_version(conn)?, 105);
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("retained96.db");
prepare_retained_schema(&path, 96);
let retained = Connection::open(path)?;
for trigger in [
"notification_sources_no_delete",
"notification_decisions_no_delete",
"notification_requests_no_delete",
"notification_acknowledgements_no_delete",
"notification_failures_no_delete",
"notification_job_targets_no_delete",
"notification_job_legacy_no_delete",
"notification_source_preparation_failures_no_delete",
] {
let definition: String = retained.query_row(
"SELECT sql FROM sqlite_schema WHERE type='trigger' AND name=?1",
[trigger],
|row| row.get(0),
)?;
conn.execute_batch(&format!("DROP TRIGGER {trigger}; {definition};"))?;
}
for trigger in [
"subject_deleted_job_insert",
"subject_deleted_job_result",
"subject_deleted_attempt_insert",
"subject_deleted_attempt_result",
"subject_deleted_worker_reservation",
"subject_deleted_notification_source",
"subject_deleted_notification_payload",
"subject_deleted_notification_send",
] {
conn.execute_batch(&format!("DROP TRIGGER {trigger};"))?;
}
// Children before parents; dropping the tables also drops snapshot guards.
conn.execute_batch(
"DROP TABLE subject_deletion_allocations;
DROP TABLE subject_deletion_jobs;
DROP TABLE subject_deletion_workers;
DROP TABLE subject_deletions;
UPDATE __yoi_schema_migrations SET version=104 WHERE version=105;",
)?;
conn.execute_batch(
"DROP TRIGGER notification_mr_event;
DROP TRIGGER notification_mr_source;
DROP TRIGGER notification_implementation_result;
DROP TRIGGER IF EXISTS implementation_delegation_capture;
@@ -396,7 +436,43 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;",
)?;
assert_eq!(current_schema_version(conn)?, 96);
assert_eq!(conn.query_row(
"SELECT count(*) FROM sqlite_schema WHERE name LIKE 'subject_delet%' OR sql LIKE '%subject_deletion%'",
[], |row| row.get::<_, i64>(0),
)?, 0);
for table in [
"notification_sources",
"notification_decisions",
"notification_requests",
"notification_acknowledgements",
"notification_failures",
"notification_job_targets",
"notification_job_legacy_receipts",
"notification_source_preparation_failures",
"backend_jobs",
"backend_job_attempts",
"worker_create_reservations",
] {
assert_eq!(
state_contract_schema_signature(conn, table),
state_contract_schema_signature(&retained, table),
"schema96 {table}"
);
}
Ok(())
}
#[tokio::test]
async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_notification_evidence() {
let (dir, store) = tracking_store_fixture().await;
tracking_store_assign(&store, "delegation", "assign");
let worker = routing_worker_assign(&store, "implementation").worker;
// Reconstruct schema97 from a fresh current-schema fixture, including its historical capture.
store.with_conn(|conn| {
reconstruct_implementation_schema96(conn)?;
migrate_business_notifications_v96_to_v97(conn)?;
assert_eq!(current_schema_version(conn)?, 97);
Ok(())
@@ -522,24 +598,7 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
store.with_conn(|conn| {
conn.execute("INSERT INTO ticket_assignment_operation_fences VALUES('w','operation-known-implementation','delegation','r','worker','workspace-orchestrator','tracking-binding',?1)",[serde_json::to_string(&tracking_store_source()).unwrap()])?;
// Reconstruct the prior composition before its new business capture existed.
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TRIGGER passkey_metadata_insert;
DROP TABLE passkey_addition_receipts;
DROP TABLE passkey_metadata;
DROP TRIGGER backend_job_input_retirement;
DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations;
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
reconstruct_implementation_schema96(conn)?;
conn.execute_batch("INSERT INTO merge_requests VALUES('w','historical-mr','main','open','refs/heads/work','refs/heads/main','3','3');
INSERT INTO merge_request_ticket_relations VALUES('w','historical-mr','tracking','implements','3');
INSERT INTO merge_request_thread_events VALUES('w','historical-mr','approved-event',1,'review','{\"subject_ref\":\"source-a\",\"decision\":\"approve\"}',NULL,'3');
@@ -691,24 +750,7 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
];
store.with_conn(|conn| {
if historical {
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
DROP TRIGGER passkey_metadata_insert;
DROP TABLE passkey_addition_receipts;
DROP TABLE passkey_metadata;
DROP TRIGGER backend_job_input_retirement;
DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
DROP TABLE subject_create_recoveries;
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
DROP TABLE subject_worker_releases;
UPDATE __yoi_schema_migrations SET version=100 WHERE version=101;
DROP TABLE user_activations;
UPDATE __yoi_schema_migrations SET version=99 WHERE version=100;
DROP TABLE hosting_administrators; DROP TABLE hosting_settings;
UPDATE __yoi_schema_migrations SET version=98 WHERE version=99;
UPDATE __yoi_schema_migrations SET version=96 WHERE version=98;")?;
reconstruct_implementation_schema96(conn)?;
}
conn.execute_batch("INSERT INTO merge_requests VALUES('w','actor-mr','main','open','work','main','3','3');
INSERT INTO merge_request_ticket_relations VALUES('w','actor-mr','tracking','implements','3');")?;
@@ -1757,3 +1757,5 @@ WHEN (OLD.runtime_id != NEW.runtime_id OR OLD.worker_id != NEW.worker_id OR OLD.
WHERE workspace_id = OLD.workspace_id AND runtime_id = OLD.runtime_id
AND worker_id = OLD.worker_id)
BEGIN SELECT RAISE(ABORT, 'Worker has unfinished Ticket work'); END;
-- Subject deletion schema is composed by store.rs.
@@ -444,6 +444,28 @@ fn job_result_facts(
pub(super) fn deliver_job_result(api: &WorkspaceApi, acceptance: &BackendJobResultAcceptance) {
use crate::store::JobNotificationFacts;
// A recovered/captured result is not delivery authority. Serialize even the
// Runtime send with deletion, so no prepared bytes escape after its fence.
let subject = subjektiv_deletion::job_subject(&acceptance.job.request);
let gate = match subject
.map(|subject| subjektiv_journal::lifecycle_lock(api, subject))
.transpose()
{
Ok(gate) => gate,
Err(error) => {
tracing::warn!(%error, "Subject Job notification gate unavailable");
return;
}
};
let _guard = match gate.as_ref().map(|gate| gate.lock()).transpose() {
Ok(guard) => guard,
Err(_) => return,
};
if let Some(subject) = subject
&& subjektiv_deletion::require_available(api, subject).is_err()
{
return;
}
let Some(_permit) = api.notification_delivery_lifecycle.begin() else {
return;
};
@@ -849,7 +849,16 @@ async fn notification_routing_fresh_and_upgraded_schema_and_guards_match() {
let snapshot = |conn: &Connection| -> Vec<(String, String)> {
conn.prepare("SELECT name,sql FROM sqlite_schema WHERE name LIKE 'notification_%' OR name LIKE 'ticket_assignment_content%' OR name IN ('ticket_assignment_title_input','ticket_assignment_initial_post_input') ORDER BY name").unwrap().query_map([],|r|Ok((r.get(0)?,r.get(1)?))).unwrap().collect::<rusqlite::Result<_>>().unwrap()
};
let upgraded = snapshot(&conn);
assert_eq!(current_schema_version(&conn).unwrap(), 98);
drop(conn);
// Compare the complete current composition, not upgraded98 against fresh105.
let upgraded_store = SqliteWorkspaceStore::open_for_test(&path).unwrap();
let upgraded = upgraded_store
.with_conn(|conn| {
assert_eq!(current_schema_version(conn)?, 105);
Ok(snapshot(conn))
})
.unwrap();
let (fresh_directory, fresh) = tracking_store_fixture().await;
fresh
.with_conn(|conn| {
+167 -2
View File
@@ -333,6 +333,15 @@ impl SqliteWorkspaceStore {
return Err(StoreError::InvalidInput(if other{"cross-workspace".into()}else{"worker-missing".into()}));
}
};
if crate::store::subject_deletion_worker_kind(&tx, &req.workspace_id, &req.worker)?.is_some() {
let prior = tx.prepare("SELECT plan_id FROM worker_removal_operations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state IN ('executing','failed')")?
.query_map(params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|r.get::<_,String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
for id in prior {
let prior=load_plan(&tx,&id)?.ok_or_else(||StoreError::Store("removal plan disappeared".into()))?;
require_current_subject_deletion_plan(&tx,&prior)?;
}
}
let mut blockers=Vec::new();
if worker.retention_state=="pinned" { blockers.push(WorkerRemovalBlocker::Hold); }
if let Some((assignment_id,ticket_id))=tx.query_row("SELECT a.assignment_id,a.ticket_id FROM ticket_active_worker_assignments c JOIN ticket_worker_assignments a ON a.workspace_id=c.workspace_id AND a.ticket_id=c.ticket_id AND a.assignment_id=c.assignment_id WHERE a.workspace_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3",params![req.workspace_id,req.worker.runtime_id,req.worker.worker_id],|r|Ok((r.get(0)?,r.get(1)?))).optional()? {
@@ -363,6 +372,7 @@ impl SqliteWorkspaceStore {
let mut plan=load_plan(&tx,plan_id)?.ok_or_else(||StoreError::InvalidInput(format!("stale:{plan_id}:plan missing")))?;
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
if plan.input_fingerprint!=fp{return Err(StoreError::InvalidInput(format!("fingerprint:{}",plan.operation_id)));}
require_current_subject_deletion_plan(&tx, &plan)?;
if plan.state==WorkerRemovalPlanState::Succeeded{tx.commit()?;return Ok(plan);}
if plan.state==WorkerRemovalPlanState::Blocked {
return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));
@@ -420,6 +430,8 @@ impl SqliteWorkspaceStore {
WorkerRetentionError::Invalid("Worker id must be a canonical UUIDv7".to_string())
})?;
let removed_at = plan.created_at.clone();
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
.map_err(map_error)?;
let prior_failure_category = plan.failure_category.clone();
Ok(PreparedWorkerRemoval {
runtime_request: WorkerRetentionExecutionRequest {
@@ -459,8 +471,8 @@ impl SqliteWorkspaceStore {
WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3
)
)
ORDER BY CASE state WHEN 'succeeded' THEN 0 ELSE 1 END,
created_at DESC LIMIT 1",
ORDER BY CASE WHEN EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=worker_removal_operations.workspace_id AND d.runtime_id=worker_removal_operations.runtime_id AND d.worker_id=worker_removal_operations.worker_id AND d.prior_removal_operation_id=worker_removal_operations.operation_id) THEN 0 WHEN state='succeeded' THEN 1 ELSE 2 END,
created_at DESC,operation_id DESC LIMIT 1",
params![workspace_id, worker.runtime_id, worker.worker_id],
|row| row.get::<_, String>(0),
)
@@ -474,6 +486,8 @@ impl SqliteWorkspaceStore {
let Some(plan) = plan else {
return Ok(None);
};
self.with_conn(|conn| require_current_subject_deletion_plan(conn, &plan))
.map_err(map_error)?;
let prior_failure_category = plan.failure_category.clone();
let worker_id = plan
.worker
@@ -659,6 +673,7 @@ impl SqliteWorkspaceStore {
let mut plan=load_plan_op(&tx,operation_id)?.ok_or_else(||StoreError::InvalidInput("operation missing".into()))?;
if plan.workspace_id!=workspace_id{return Err(StoreError::InvalidInput("cross-workspace".into()));}
if plan.input_fingerprint!=fp||result.input_fingerprint!=fp||result.operation_id!=operation_id{return Err(StoreError::InvalidInput(format!("fingerprint:{operation_id}")));}
require_current_subject_deletion_plan(&tx, &plan)?;
if plan.state==WorkerRemovalPlanState::Succeeded{
// Repair projection fences left by the pre-atomic implementation and
// return a replayable change for a broadcast missed after commit.
@@ -1024,6 +1039,49 @@ fn load_removal_policy(
let Some(mut policy) = load_policy(c, w)? else {
return Ok(None);
};
// Snapshot authority outranks normal policy and Journal cleanup policy. Body
// and delegated Sessions must never inherit the ephemeral Job exception.
if let Some(is_job) = crate::store::subject_deletion_worker_kind(c, w, worker)? {
let update = WorkerRetentionPolicyUpdate {
policy_id: if is_job {
"subject-deletion-job-v1"
} else {
"subject-deletion-conversation-v1"
}
.into(),
session_disposition: if is_job {
SessionDisposition::Purge
} else {
SessionDisposition::Archive
},
metadata_disposition: MetadataDisposition::Tombstone,
archive_retention: ArchiveRetention::Forever,
// Runtime calls diagnostic archiving Retain, not Archive. Preserve
// the Workspace's duration, or thirty days when it normally purges.
diagnostics_disposition: if is_job {
DiagnosticsDisposition::Purge
} else {
DiagnosticsDisposition::Retain
},
diagnostics_retention_seconds: if is_job {
None
} else {
Some(
policy
.diagnostics_retention_seconds
.unwrap_or(30 * 24 * 60 * 60),
)
},
};
policy.policy_id = update.policy_id.clone();
policy.policy_digest = worker_retention_policy_digest(&update);
policy.session_disposition = update.session_disposition;
policy.metadata_disposition = update.metadata_disposition;
policy.archive_retention = update.archive_retention;
policy.diagnostics_disposition = update.diagnostics_disposition;
policy.diagnostics_retention_seconds = update.diagnostics_retention_seconds;
return Ok(Some(policy));
}
if erased_journal || crate::store::terminal_journal_job_worker(c, w, worker)? {
let update = WorkerRetentionPolicyUpdate {
policy_id: if erased_journal {
@@ -1047,6 +1105,39 @@ fn load_removal_policy(
}
Ok(Some(policy))
}
fn require_current_subject_deletion_plan(
c: &Connection,
plan: &WorkerRemovalPlan,
) -> crate::Result<()> {
if plan.state == WorkerRemovalPlanState::Succeeded
&& crate::store::subject_deletion_prior_removal_matches(
c,
&plan.workspace_id,
&plan.worker,
&plan.operation_id,
)?
{
// An exact successful receipt pinned before Subject deletion owns its
// independent original/archive lifecycle. Never retarget its policy.
return Ok(());
}
if crate::store::subject_deletion_worker_kind(c, &plan.workspace_id, &plan.worker)?.is_some() {
let current = load_removal_policy(c, &plan.workspace_id, &plan.worker, false)?
.ok_or_else(|| StoreError::InvalidInput("Subject deletion policy missing".into()))?;
if plan.session_disposition != current.session_disposition
|| plan.metadata_disposition != current.metadata_disposition
|| plan.archive_retention != current.archive_retention
|| plan.diagnostics_disposition != current.diagnostics_disposition
{
return Err(stale_error(
plan,
"Subject deletion retention differs from saved plan; reconcile canonical Runtime removal before retry",
));
}
}
Ok(())
}
fn load_policy(c: &Connection, w: &str) -> crate::Result<Option<WorkerRetentionPolicy>> {
c.query_row(
"SELECT p.policy_id,p.policy_digest,r.session_disposition,r.metadata_disposition,
@@ -2736,4 +2827,78 @@ mod tests {
);
assert_eq!(recovered.plan.state, WorkerRemovalPlanState::Failed);
}
#[test]
fn subject_deletion_snapshot_overrides_policy_without_changing_normal_removal() {
let store = setup();
let ordinary = store.plan_worker_removal(&req(), &inv()).unwrap();
store
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
.unwrap();
let body = store.plan_worker_removal(&req(), &inv()).unwrap();
assert_eq!(body.session_disposition, SessionDisposition::Archive);
assert_eq!(body.archive_retention, ArchiveRetention::Forever);
assert_eq!(body.metadata_disposition, MetadataDisposition::Tombstone);
assert_eq!(body.diagnostics_disposition, DiagnosticsDisposition::Retain);
assert!(body.diagnostics_retention_seconds.is_some());
assert_ne!(body.policy_id, ordinary.policy_id);
// A saved plan that would discard diagnostics cannot silently stand in
// for the Subject conversation archive plan.
assert!(matches!(
store.begin_worker_removal("w", &ordinary.plan_id, &ordinary.input_fingerprint),
Err(WorkerRetentionError::StalePlan { .. })
));
let ordinary_store = setup();
assert_eq!(
ordinary_store
.plan_worker_removal(&req(), &inv())
.unwrap()
.policy_id,
ordinary.policy_id
);
}
#[test]
fn subject_deletion_job_purge_rejects_recovery_of_old_executing_archive() {
let store = setup();
let old = store.plan_worker_removal(&req(), &inv()).unwrap();
store
.prepare_worker_removal_execution("w", &old.plan_id, &old.input_fingerprint)
.unwrap();
store
.prepare_subject_deletion("w", "subject", "delete", &[req().worker.clone()])
.unwrap();
store
.with_conn(|c| {
c.execute(
"UPDATE subject_deletion_workers SET is_job=1 WHERE workspace_id='w'",
[],
)?;
Ok(())
})
.unwrap();
assert!(matches!(
store.recover_worker_removal_execution("w", &req().worker),
Err(WorkerRetentionError::StalePlan { .. })
));
assert!(matches!(
store.plan_worker_removal(&req(), &inv()),
Err(WorkerRetentionError::StalePlan { .. })
));
let fresh = store
.with_conn(|c| load_removal_policy(c, "w", &req().worker, false))
.unwrap()
.unwrap();
assert_eq!(fresh.session_disposition, SessionDisposition::Purge);
assert_eq!(fresh.diagnostics_disposition, DiagnosticsDisposition::Purge);
assert_eq!(fresh.metadata_disposition, MetadataDisposition::Tombstone);
assert_eq!(
store
.with_conn(|c| load_plan(c, &old.plan_id))
.unwrap()
.unwrap()
.state,
WorkerRemovalPlanState::Executing
);
}
}
@@ -91,10 +91,16 @@ mod schema_convergence {
assert_eq!(plan.current_schema_version, 103);
assert_eq!(
plan.migrations,
vec![WorkspaceSchemaMigrationStep {
version: 104,
name: "Converge Passkey results and Job input retention".into(),
}]
vec![
WorkspaceSchemaMigrationStep {
version: 104,
name: "Converge Passkey results and Job input retention".into(),
},
WorkspaceSchemaMigrationStep {
version: 105,
name: "Durable Subject deletion snapshots and Job admission fences".into(),
},
]
);
assert_eq!(
schema_rows(&conn),
@@ -109,6 +115,7 @@ mod schema_convergence {
assert_eq!(feature_rows(conn, job_variant), rows);
let mut expected = history.clone();
expected.push((104, "Converge Passkey results and Job input retention".into()));
expected.push((105, "Durable Subject deletion snapshots and Job admission fences".into()));
assert_eq!(workspace_schema_migration_history(conn)?, expected);
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
@@ -138,7 +145,7 @@ mod schema_convergence {
assert_eq!(feature_rows(conn, job_variant), rows);
assert_eq!(
workspace_schema_migration_history(conn)?.len(),
history.len() + 1
history.len() + 2
);
Ok(())
})
@@ -176,7 +183,7 @@ mod schema_convergence {
upgraded
.with_conn(|conn| {
assert_eq!(feature_rows(conn, job_variant), rows);
assert_eq!(current_schema_version(conn)?, 104);
assert_eq!(current_schema_version(conn)?, 105);
Ok(())
})
.unwrap();
+132 -48
View File
@@ -28,6 +28,7 @@ use subjektiv::api::*;
mod backend_job_profile;
mod external_operation_audit;
mod subject_authorization;
mod subjektiv_deletion;
mod subjektiv_job;
mod subjektiv_journal;
mod subjektiv_journal_browser;
@@ -4577,6 +4578,12 @@ impl WorkspaceApi {
.run(move || recover_workdir_removals(&workdir_recovery_api))
.await?;
recover_runtime_removals(&api).await;
let deletion_api = api.clone();
let deletion_runtime = tokio::runtime::Handle::current();
api.host_executors
.host()?
.run(move || deletion_runtime.block_on(subjektiv_deletion::recover(&deletion_api)))
.await?;
api.host_executors
.host()?
.run(move || {
@@ -4604,9 +4611,23 @@ impl WorkspaceApi {
) -> Result<BackendJobReservation> {
self.resolve_backend_job_profile(request)?;
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
let reservation =
let reservation = {
let gate = subjektiv_deletion::job_subject(request)
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
.transpose()?;
let _guard = gate
.as_ref()
.map(|gate| {
gate.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
})
.transpose()?;
if let Some(subject) = subjektiv_deletion::job_subject(request) {
subjektiv_deletion::require_available(self, subject)?;
}
self.store
.reserve_backend_job(&self.config.workspace_id, request, &now)?;
.reserve_backend_job(&self.config.workspace_id, request, &now)?
};
let result = self.dispatch_backend_job_reservation(reservation);
if result.is_err() {
self.drain_reserved_backend_jobs();
@@ -4621,9 +4642,27 @@ impl WorkspaceApi {
/// immutable request.
pub fn retry_backend_job(&self, job_id: &str) -> Result<BackendJobReservation> {
let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
let reservation =
let reservation = {
let current = self
.store
.get_backend_job(self.workspace_id(), job_id)?
.ok_or_else(|| Error::InvalidInput("unknown Backend Job".into()))?;
let gate = subjektiv_deletion::job_subject(&current.request)
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
.transpose()?;
let _guard = gate
.as_ref()
.map(|gate| {
gate.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
})
.transpose()?;
if let Some(subject) = subjektiv_deletion::job_subject(&current.request) {
subjektiv_deletion::require_available(self, subject)?;
}
self.store
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?;
.reserve_backend_job_retry(&self.config.workspace_id, job_id, &now)?
};
let result = self.dispatch_backend_job_reservation(reservation);
if result.is_err() {
self.drain_reserved_backend_jobs();
@@ -4637,13 +4676,8 @@ impl WorkspaceApi {
) -> Result<BackendJobReservation> {
// Reread even a terminal/stale reservation before deciding whether it can
// dispatch or replay: an explicit erasure may have retired its saved input.
let journal_gate = reservation
.job
.request
.grants
.subjektiv_journal
.as_ref()
.map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id))
let journal_gate = subjektiv_deletion::job_subject(&reservation.job.request)
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
.transpose()?;
let _journal_guard = journal_gate
.as_ref()
@@ -4652,6 +4686,9 @@ impl WorkspaceApi {
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))
})
.transpose()?;
if let Some(subject) = subjektiv_deletion::job_subject(&reservation.job.request) {
subjektiv_deletion::require_available(self, subject)?;
}
let current_job = self
.store
.get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)?
@@ -5571,6 +5608,27 @@ impl WorkspaceApi {
source_worker: &RuntimeWorkerRef,
submission: &BackendJobResultSubmission,
) -> Result<BackendJobResultAcceptance> {
let result_job = self
.store
.get_backend_job(self.workspace_id(), &submission.job_id)?;
let subject_gate = result_job
.as_ref()
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
.map(|subject| subjektiv_journal::lifecycle_lock(self, subject))
.transpose()?;
let subject_guard = subject_gate
.as_ref()
.map(|gate| {
gate.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))
})
.transpose()?;
if let Some(subject) = result_job
.as_ref()
.and_then(|job| subjektiv_deletion::job_subject(&job.request))
{
subjektiv_deletion::require_available(self, subject)?;
}
let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true);
let mut verified_submission = submission.clone();
if let Some(job) = self
@@ -5696,6 +5754,7 @@ impl WorkspaceApi {
&verified_submission,
&accepted_at,
)?;
drop(subject_guard);
if let Some(job) = self
.store
.get_backend_job(&self.config.workspace_id, &submission.job_id)?
@@ -5899,15 +5958,21 @@ impl WorkspaceApi {
.map(|assignment| assignment.operation_id.clone())
})
.unwrap_or_else(|| format!("manual:{}", WorkerId::now_v7()));
let subject = singleton_key
let body_subject = singleton_key
.as_deref()
.and_then(|key| key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX));
if let Some(subject) = subject
if let Some(subject) = body_subject
&& let Err(error) = subjektiv_journal::recover_failed_subject_create(self, subject)
{
let diagnostics = cleanup_spawn_created_workdirs(self, &spawned_workdir_ids);
return Err(ApiError::with_diagnostics(error, diagnostics));
}
let subject = body_subject.or_else(|| {
request
.resolved_execution_grant
.as_ref()
.map(|grant| grant.subject_id.as_str())
});
let subject_gate = subject
.map(|s| subjektiv_journal::lifecycle_lock(self, s))
.transpose()?;
@@ -5919,6 +5984,9 @@ impl WorkspaceApi {
})
.transpose()?;
if let Some(subject) = subject {
subjektiv_deletion::require_available(self, subject)?;
}
if let Some(subject) = body_subject {
subjektiv_journal::check_start_allowed(self, subject)?;
}
let reservation = match self.config_store.reserve_worker_create(
@@ -5943,11 +6011,11 @@ impl WorkspaceApi {
return Err(ApiError::with_diagnostics(error, diagnostics));
}
};
drop(subject_guard);
let worker_id = reservation.worker_id;
if let Some(grant) = request.resolved_execution_grant.as_ref() {
subject_authorization::bind_reserved_grant(self, grant, &worker_id.to_string())?;
}
drop(subject_guard);
// A completed exact-source allocation is historical acceptance, not a new
// launch. Source/grant authorization and the full fingerprint were checked
// above. Do not reserve its already-linked attachments again, redispatch
@@ -7458,7 +7526,7 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou
.merge(server_api::server_api_axum::subjektiv_memory_changes(
service.clone(),
))
.merge(server_api::server_api_axum::subjektiv_subject_retire(
.merge(server_api::server_api_axum::subjektiv_subject_delete(
service.clone(),
))
.merge(server_api::server_api_axum::subjektiv_stage_candidate(
@@ -9724,22 +9792,25 @@ impl server_api::ServerApi for ServerApiContractService {
.map_err(ApiError::into_repository_api_error)
}
async fn subjektiv_subject_retire(
async fn subjektiv_subject_delete(
&self,
context: server_api::ServerRequestContext,
workspace_id: String,
subject_id: String,
) -> std::result::Result<server_api::SubjektivSubjectResponse, server_api::RepositoryApiError>
{
scoped_retire_subjektiv_subject(
State(self.workspace_api()?.clone()),
AxumPath(ScopedSubjektivSubjectPath {
workspace_id,
subject_id,
}),
)
.await
.map(|Json(response)| response)
.map_err(ApiError::into_repository_api_error)
request: server_api::SubjektivSubjectDeleteRequest,
) -> std::result::Result<
server_api::SubjektivSubjectDeleteResponse,
server_api::RepositoryApiError,
> {
let api = self.workspace_api()?.clone();
let actor = subject_authorization::administration_actor(&context)
.map_err(ApiError::into_repository_api_error)?;
require_workspace_owner(&api, &workspace_id, actor, "Subject deletion")
.await
.map_err(ApiError::into_repository_api_error)?;
subjektiv_deletion::delete(api, workspace_id, subject_id, request)
.await
.map_err(ApiError::into_repository_api_error)
}
async fn subjektiv_stage_candidate(
@@ -22674,7 +22745,9 @@ fn subjektiv_subject_response(
memory_fingerprint,
state: match subject.state {
crate::subjektiv::SubjectState::Active => server_api::SubjektivSubjectState::Active,
crate::subjektiv::SubjectState::Retired => server_api::SubjektivSubjectState::Retired,
crate::subjektiv::SubjectState::LegacyRetired => {
server_api::SubjektivSubjectState::LegacyRetired
}
},
created_at: subject.created_at,
updated_at: subject.updated_at,
@@ -22962,24 +23035,6 @@ async fn scoped_list_subjektiv_memory_changes(
.await
}
async fn scoped_retire_subjektiv_subject(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedSubjektivSubjectPath>,
) -> ApiResult<Json<server_api::SubjektivSubjectResponse>> {
api.host_executors
.host()?
.run(move || {
validate_workspace_scope(&api, &path.workspace_id)?;
let subject = open_subjektiv_store(&api)?
.retire_subject(&path.subject_id)
.map_err(subjektiv_store_error)?;
// Retirement deliberately does not stop or remove the current Worker. The
// Worker lease and subject lifecycle are independent authorities.
Ok(Json(subjektiv_subject_response(&api, subject)?))
})
.await
}
async fn scoped_stage_subjektiv_candidate(
State(api): State<WorkspaceApi>,
AxumPath(path): AxumPath<ScopedWorkspacePath>,
@@ -24564,6 +24619,7 @@ fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error {
Error::WorkspacePermissionDenied(error.to_string())
}
crate::subjektiv::SubjektivError::SubjectPermissionsConflict(_)
| crate::subjektiv::SubjektivError::SubjectDeletionConflict(_)
| crate::subjektiv::SubjektivError::PermissionOperationConflict(_)
| crate::subjektiv::SubjektivError::WorkerExecutionGrantConflict(_) => {
Error::RepositoryConflict(error.to_string())
@@ -31543,6 +31599,7 @@ async fn workspace_response(
manage_secrets: is_owner,
manage_runtimes: is_owner,
delete_workspace: is_owner,
delete_subjects: is_owner,
},
extension_points: WorkspaceExtensionPoints {
store: "sqlite".to_string(),
@@ -37777,6 +37834,7 @@ mod tests {
mod restore_workdir_tests;
mod restored_parent_control_tests;
mod subject_authorization_tests;
mod subject_deletion_tests;
mod subject_spawn_tests;
mod subjektiv_jobs_tests;
mod subjektiv_journal_tests;
@@ -46241,7 +46299,13 @@ mod tests {
let retired = store
.create_subject(crate::subjektiv::SubjectRole::new("retired").unwrap())
.unwrap();
store.retire_subject(&retired.id).unwrap();
// Frozen lifecycle fixture: migration preserves old inactive Subjects; there is
// intentionally no public operation that creates this historical state anymore.
api.feature_storage.open(&api.subjektiv_registration).unwrap()
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
tx.execute("UPDATE subjects SET state='retired', record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&retired.id])?;
Ok(())
}).unwrap();
let request = |subject_id: Option<&str>, profile: &str, singleton_key: Option<&str>| {
CreateWorkspaceWorkerRequest {
@@ -48843,6 +48907,7 @@ mod tests {
>,
materializer: worker_runtime::working_directory::RuntimeGitMaterializer,
spawn_failure: std::sync::Mutex<Option<String>>,
subject_delete_failure: std::sync::Mutex<Option<String>>,
recovery_failure: std::sync::Mutex<Option<String>>,
recovery_requests: std::sync::Mutex<Vec<runtime_api::SubjectCreateRecoveryRequest>>,
recovery_hook: std::sync::Mutex<Option<Box<dyn FnOnce() + Send>>>,
@@ -48887,6 +48952,7 @@ mod tests {
std::env::temp_dir().join(unique),
),
spawn_failure: std::sync::Mutex::new(None),
subject_delete_failure: std::sync::Mutex::new(None),
recovery_failure: std::sync::Mutex::new(None),
recovery_requests: std::sync::Mutex::new(Vec::new()),
recovery_hook: std::sync::Mutex::new(None),
@@ -49242,6 +49308,24 @@ mod tests {
.map(|execution| execution.snapshot())
}
fn preflight_subject_delete_worker(
&self,
_request: &runtime_api::SubjectDeleteWorkerRequest,
) -> std::result::Result<(), String> {
Ok(())
}
fn subject_delete_worker(
&self,
request: &runtime_api::SubjectDeleteWorkerRequest,
) -> std::result::Result<(), String> {
if let Some(message) = self.subject_delete_failure.lock().unwrap().clone() {
return Err(message);
}
self.stop_worker(&worker_runtime::identity::WorkerRef::new(request.worker_id));
Ok(())
}
fn stop_worker(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
@@ -149,6 +149,13 @@ pub(super) fn worker_permissions(
api: &WorkspaceApi,
worker: &RuntimeWorkerRef,
) -> ApiResult<Option<Vec<ScopedPermission>>> {
if subjektiv_deletion::deleting_worker(api, worker)?
|| open_subjektiv_store(api)?
.worker_subject_deleted(&worker.runtime_id, &worker.worker_id)
.map_err(subjektiv_store_error)?
{
return Err(denied("Worker belonged to an explicitly deleted Subject"));
}
if let Some(grant) = live_child_grant(api, worker)? {
return Ok(Some(grant.permissions));
}
@@ -226,6 +233,8 @@ pub(super) fn prepare_delegation(
singleton_key: Option<&str>,
subject_connection: bool,
) -> ApiResult<Option<WorkerExecutionGrant>> {
// A deleted grant must never become an ordinary non-Subject delegation source.
worker_permissions(api, source)?;
let child = live_child_grant(api, source)?;
let body = if child.is_none() {
current_body_subject(api, source)?
@@ -0,0 +1,288 @@
//! Explicit Subject destruction is independent of ordinary Worker retention. The domain
//! fence commits before external effects, remains after failures, and rejects cached grants.
//! Worker originals are archived; Job execution copies are purged. Existing archives, Workdirs,
//! Tickets and other Subjects are not erased. Archive contents can include previously injected
//! Memory: their independent retention/access policy still applies. This is logical content
//! deletion, not a promise to erase backups, external providers or already exported material.
//!
//! Child removal/create-fence receipts, not acceptance, Idle or missing projections, prove
//! quiescence. A retry carries the same scope/Subject/operation; no new child identity is issued.
//! The domain receipt is authoritative completion. Backend inventories retain only identifiers
//! needed for cleanup and fail-closed authorization, never a copy of Subject content.
use super::*;
use rusqlite::params;
pub(super) fn job_subject(request: &BackendJobRequest) -> Option<&str> {
request
.grants
.subjektiv_journal
.as_ref()
.map(|g| g.subject_id.as_str())
.or_else(|| {
request
.grants
.subjektiv_consolidation
.as_ref()
.map(|g| g.subject_id.as_str())
})
}
pub(super) fn require_available(api: &WorkspaceApi, subject: &str) -> Result<()> {
let store = open_subjektiv_store(api).map_err(|error| error.error)?;
if store
.subject(subject)
.map_err(|e| Error::Store(e.to_string()))?
.is_none()
{
return Err(Error::SubjektivSubjectNotFound(subject.into()));
}
Ok(())
}
pub(super) fn pending(api: &WorkspaceApi, subject: &str) -> Result<bool> {
api.config_store.with_conn(|connection| {
Ok(connection.query_row(
"SELECT EXISTS(SELECT 1 FROM subject_deletions WHERE workspace_id=?1 AND subject_id=?2)",
params![api.workspace_id(), subject], |row| row.get(0),
)?)
})
}
/// This is internal cleanup evidence, never a user-supplied removal flag. It exists only
/// after the independently durable domain fence and exact-target inventory commit.
pub(super) fn deleting_worker(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Result<bool> {
api.config_store.with_conn(|connection| {
Ok(connection.query_row(
"SELECT EXISTS(SELECT 1 FROM subject_deletion_workers WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3)",
params![api.workspace_id(), worker.runtime_id, worker.worker_id], |row| row.get(0),
)?)
})
}
pub(super) async fn delete(
api: WorkspaceApi,
workspace: String,
subject: String,
request: server_api::SubjektivSubjectDeleteRequest,
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
let handle = tokio::runtime::Handle::current();
api.host_executors
.host()?
.run(move || {
validate_workspace_scope(&api, &workspace)?;
handle.block_on(resume(&api, &subject, &request.operation_id))
})
.await
}
pub(super) async fn recover(api: &WorkspaceApi) -> Result<()> {
let pending = open_subjektiv_store(api)
.map_err(|error| error.error)?
.pending_subject_deletions()
.map_err(|e| Error::Store(e.to_string()))?;
for deletion in pending {
if let Err(error) = resume(api, &deletion.subject_id, &deletion.operation_id).await {
tracing::warn!(%error.error, subject_id=%deletion.subject_id,
"Subject deletion restart recovery remains incomplete; same request can resume");
}
}
Ok(())
}
async fn resume(
api: &WorkspaceApi,
subject: &str,
operation: &str,
) -> ApiResult<server_api::SubjektivSubjectDeleteResponse> {
let store = open_subjektiv_store(api)?;
let gate = subjektiv_journal::lifecycle_lock(api, subject)?;
let receipt = {
let _guard = gate
.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
let receipt = store
.begin_subject_deletion(subject, operation)
.map_err(subjektiv_store_error)?;
if !receipt.completed {
let delegated = store
.deletion_subject_workers(subject)
.map_err(subjektiv_store_error)?
.into_iter()
.map(|(runtime, worker)| RuntimeWorkerRef::new(runtime, worker))
.collect::<Vec<_>>();
api.config_store.prepare_subject_deletion(
api.workspace_id(),
subject,
operation,
&delegated,
)?;
}
receipt
};
let response = |completed| server_api::SubjektivSubjectDeleteResponse {
subject_id: subject.into(),
operation_id: operation.into(),
completed,
};
if receipt.completed {
return Ok(response(true));
}
// Never hold the Host callback gate across a Runtime call. A create/restore can be
// waiting for the callback that now has to reject this permanently fenced Subject.
let workers =
api.config_store
.list_subject_deletion_workers(api.workspace_id(), subject, operation)?;
for (worker, _is_job) in workers {
if let Err(error) = remove_worker(api, subject, operation, &worker).await {
tracing::warn!(%error.error, subject_id=subject, operation_id=operation, "Subject deletion remains incomplete; exact retry required");
return Ok(response(false));
}
}
{
let _guard = gate
.lock()
.map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?;
api.config_store
.erase_subject_deleted_jobs(api.workspace_id(), subject, operation)?;
// Finish Backend cleanup before the authoritative domain receipt. If domain
// purge fails, its separately committed fence and pending operation survive.
// No retry can report domain completion while occupancy/Job cleanup is unproven.
api.config_store
.finish_subject_deletion_record(api.workspace_id(), subject, operation)?;
store
.finish_subject_deletion(subject, operation)
.map_err(subjektiv_store_error)?;
}
Ok(response(true))
}
async fn remove_worker(
api: &WorkspaceApi,
subject: &str,
operation: &str,
worker: &RuntimeWorkerRef,
) -> ApiResult<()> {
let request = api.config_store.subject_deletion_worker_request(
api.workspace_id(),
subject,
operation,
worker,
)?;
let fence = api
.runtime
.subject_delete_worker(&worker.runtime_id, request.clone())
.map_err(RuntimeRegistryError::into_error)?;
if fence.workspace_id != request.workspace_id
|| fence.worker_id != request.worker_id
|| fence.create_fingerprint != request.create_fingerprint
|| fence.operation_id != request.operation_id
{
return Err(Error::RepositoryConflict(
"Subject Worker deletion fence receipt mismatch".into(),
)
.into());
}
// Replay the canonical child even when physical removal already happened:
// its retained receipt can finish the Backend catalog midpoint. The Runtime
// fence above also prevents an older create from arriving after that removal.
if api
.config_store
.recover_worker_removal_execution(api.workspace_id(), worker)
.map_err(|error| Error::Store(error.to_string()))?
.is_some()
{
return execute_removal(api, operation, worker).await;
}
match api.runtime.worker(worker) {
Ok(detail) => {
if !detail.state.eq_ignore_ascii_case("stopped") {
let stopped = api
.runtime
.stop_worker(
worker,
WorkerLifecycleRequest {
reason: Some(format!("Explicit Subject deletion {operation}")),
ticket_assignment: None,
},
)
.map_err(RuntimeRegistryError::into_error)?;
if stopped.state != InternalWorkerOperationState::Accepted {
return Err(Error::RepositoryConflict(
"Subject Worker stop was not accepted".into(),
)
.into());
}
}
let actual = api
.runtime
.worker(worker)
.map_err(RuntimeRegistryError::into_error)?;
if !actual.state.eq_ignore_ascii_case("stopped") {
return Err(
Error::RepositoryConflict("Subject Worker stop is unconfirmed".into()).into(),
);
}
if api
.store
.get_worker_registry(api.workspace_id(), worker)?
.is_none()
{
// The Runtime fence authenticated this exact reserved fingerprint. A create
// can have reached Runtime before its Backend publication failed. Publish
// only its stopped record so the normal archive/Purge owner can remove it.
record_worker_summary(
api,
&actual,
&actual.display_name,
actual.profile.clone(),
WorkerRegistryDisplayNamePolicy::PreserveExisting,
)?;
}
}
Err(RuntimeRegistryError::UnknownWorker { .. }) => {
if api
.store
.get_worker_registry(api.workspace_id(), worker)?
.is_some()
{
return Err(Error::RepositoryConflict(
"Runtime object missing before canonical original archival".into(),
)
.into());
}
api.config_store.confirm_subject_deletion_absence(
api.workspace_id(),
subject,
operation,
worker,
&fence,
)?;
return Ok(());
}
Err(error) => return Err(error.into_error().into()),
}
execute_removal(api, operation, worker).await
}
async fn execute_removal(
api: &WorkspaceApi,
operation: &str,
worker: &RuntimeWorkerRef,
) -> ApiResult<()> {
let response = WorkerRemovalService::new(api)
.execute_target_removal_on_host(
api.runtime.as_ref(),
worker,
&format!("Explicit Subject deletion {operation}"),
None,
)
.await
.map_err(Error::Store)?;
if response.status != StatusCode::OK.as_u16() {
return Err(Error::RepositoryConflict(
"Subject Worker archival/Purge remains unconfirmed".into(),
)
.into());
}
Ok(())
}
@@ -446,6 +446,7 @@ pub(super) fn bind_host_session(
/// Runs under the same Subject gate as final revoke/singleton release. A failed create or
/// legacy removal may have released its Backend reservation but cannot hide a durable writer.
pub(super) fn check_start_allowed(api: &WorkspaceApi, subject: &str) -> Result<()> {
subjektiv_deletion::require_available(api, subject)?;
let store = store(api)?;
let key = crate::subjektiv::subject_worker_singleton_key(subject)
.map_err(|e| Error::InvalidInput(e.to_string()))?;
@@ -1747,6 +1748,13 @@ pub(super) fn prepare_worker_removal(
runtime: &RuntimeRegistry,
target: &RuntimeWorkerRef,
) -> Result<Option<crate::retention::ErasedJournalSession>> {
if let Some(api) = service.journal_dispatch_api.upgrade()
&& subjektiv_deletion::deleting_worker(&api, target)?
{
// Full Subject deletion has a permanent domain fence. It deliberately drops
// Journal continuity, but retains source originals through its archive policy.
return Ok(None);
}
let Some(attr) = removal_attribution(service, target)? else {
if service
.store
@@ -1809,6 +1817,13 @@ pub(super) fn erased_archive_removal_proof(
target: &RuntimeWorkerRef,
prepared: &crate::retention::PreparedWorkerRemoval,
) -> Result<Option<crate::retention::RuntimeJournalErasureAck>> {
if let Some(api) = service.journal_dispatch_api.upgrade()
&& subjektiv_deletion::deleting_worker(&api, target)?
{
// Resume the exact canonical retention request after full Subject fencing.
// Subject deletion retains originals; it is not per-Session source erasure.
return Ok(None);
}
let Some(attr) = removal_attribution(service, target)? else {
return Ok(None);
};
@@ -1852,6 +1867,13 @@ pub(super) fn validate_removal_retention(
target: &RuntimeWorkerRef,
prepared: &crate::retention::PreparedWorkerRemoval,
) -> Result<()> {
if let Some(api) = service.journal_dispatch_api.upgrade()
&& subjektiv_deletion::deleting_worker(&api, target)?
{
// The plan-local retention policy and canonical Runtime receipt remain required.
// Neither unfinished Journals nor a stopped old Controller are deletion blockers.
return Ok(());
}
let Some(attr) = removal_attribution(service, target)? else {
if service
.store
@@ -2066,6 +2088,9 @@ pub(super) fn session_erased(api: &WorkspaceApi, request: &BackendJobRequest) ->
let Some(grant) = &request.grants.subjektiv_journal else {
return Ok(false);
};
if subjektiv_deletion::pending(api, &grant.subject_id)? {
return Ok(true);
}
let store = store(api)?;
store
.journal_session_erased(&host(api, &store, grant)?)
@@ -0,0 +1,729 @@
//! Real router/authentication, feature store and Backend DB boundaries. Runtime
//! execution/unknown and restart fences are covered in worker-runtime and the
//! canonical removal suites, rather than claiming HTTP acceptance proves shutdown.
use super::*;
async fn delete_request(
api: &WorkspaceApi,
subject: &str,
key: &str,
token: Option<&str>,
workspace: &str,
) -> (StatusCode, Value) {
let mut builder = Request::builder()
.method("DELETE")
.uri(format!("/api/w/{workspace}/subjektiv/subjects/{subject}"))
.header(CONTENT_TYPE, "application/json");
if let Some(token) = token {
builder = builder.header("authorization", format!("Bearer {token}"));
}
let response = build_router(api.clone())
.oneshot(
builder
.body(Body::from(json!({"operation_id":key}).to_string()))
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let bytes = to_bytes(response.into_body(), 2 * 1024 * 1024)
.await
.unwrap();
let body = serde_json::from_slice(&bytes)
.unwrap_or_else(|_| json!({"text":String::from_utf8_lossy(&bytes)}));
(status, body)
}
#[tokio::test]
async fn deletion_owner_scope_exact_retry_and_subject_isolation() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("delete me").unwrap(),
"private behavior".into(),
)
.unwrap();
let other = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("keep me").unwrap(),
"unchanged".into(),
)
.unwrap();
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-owner").await;
let stranger = seed_test_api_token(api.store.as_ref(), "subject-delete-stranger");
for (token, workspace, expected) in [
(None, TEST_WORKSPACE_ID, StatusCode::UNAUTHORIZED),
(
Some(stranger.as_str()),
TEST_WORKSPACE_ID,
StatusCode::FORBIDDEN,
),
(
Some(owner.as_str()),
"another-workspace",
StatusCode::NOT_FOUND,
),
] {
let (status, body) =
delete_request(&api, &subject.id, "same-operation", token, workspace).await;
assert_eq!(status, expected, "{body}");
assert!(store.subject(&subject.id).unwrap().is_some());
}
let (status, body) = delete_request(
&api,
&subject.id,
"same-operation",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
assert!(store.subject(&subject.id).unwrap().is_none());
assert_eq!(
store.subject(&other.id).unwrap().unwrap().behavior_md,
"unchanged"
);
assert_eq!(store.list_subjects(100).unwrap().items.len(), 1);
let (status, replay) = delete_request(
&api,
&subject.id,
"same-operation",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{replay}");
assert_eq!(replay, body);
let (status, _) = delete_request(
&api,
&subject.id,
"different-operation",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::CONFLICT);
let (status, _) = delete_request(
&api,
&other.id,
"same-operation",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::CONFLICT);
assert!(store.subject(&other.id).unwrap().is_some());
let fresh = store
.create_subject(crate::subjektiv::SubjectRole::new("fresh").unwrap())
.unwrap();
assert_ne!(fresh.id, subject.id);
assert!(store.subject_permissions(&fresh.id).unwrap().is_none());
assert_eq!(
store
.pending_staging_candidates(&fresh.id, 100)
.unwrap()
.len(),
0
);
}
#[tokio::test]
async fn legacy_inactive_subject_is_preserved_until_explicit_delete_and_retire_route_is_gone() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("legacy fixture").unwrap(),
"retained original".into(),
)
.unwrap();
api.feature_storage.open(&api.subjektiv_registration).unwrap()
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
tx.execute("UPDATE subjects SET state='retired',record_json=json_set(record_json,'$.state','retired') WHERE subject_id=?1", [&subject.id])?;
Ok(())
}).unwrap();
assert_eq!(
store.subject(&subject.id).unwrap().unwrap().state,
crate::subjektiv::SubjectState::LegacyRetired
);
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-legacy").await;
let request = Request::builder()
.method("POST")
.uri(format!(
"/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/retire",
subject.id
))
.header("authorization", format!("Bearer {owner}"))
.body(Body::empty())
.unwrap();
let old = build_router(api.clone()).oneshot(request).await.unwrap();
assert_eq!(old.status(), StatusCode::METHOD_NOT_ALLOWED);
assert_eq!(
store.subject(&subject.id).unwrap().unwrap().behavior_md,
"retained original"
);
let (status, body) = delete_request(
&api,
&subject.id,
"explicit-legacy-delete",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn unconfirmed_worker_shutdown_is_not_completion_and_exact_retry_converges() {
let dir = tempfile::tempdir().unwrap();
init_clean_git_workspace(dir.path());
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("delete connected body").unwrap(),
"private behavior".into(),
)
.unwrap();
let Json(created) = create_workspace_worker(
State(api.clone()),
HeaderMap::new(),
Json(CreateWorkspaceWorkerRequest {
runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(),
display_name: "Subject deletion boundary".into(),
singleton_key: None,
profile: Some("builtin:companion".into()),
ticket_assignment: None,
initial_submit: vec![],
workdir_attachments: vec![],
feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest {
subjektiv: Some(server_api::SubjektivWorkerConnectionRequest {
subject_id: subject.id.clone(),
}),
},
control_operation_id: None,
execution_grant: None,
}),
)
.await
.unwrap();
let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id);
seed_worker_session_for_cleanup(dir.path(), &worker);
let owner = seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unknown").await;
*execution.subject_delete_failure.lock().unwrap() = Some("shutdown completion unknown".into());
let (status, body) = delete_request(
&api,
&subject.id,
"unknown-delete",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], false);
assert!(store.subject(&subject.id).unwrap().is_none());
assert!(
!store
.subject_deletion(&subject.id)
.unwrap()
.unwrap()
.completed
);
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
assert!(
api.store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
.unwrap()
.is_some()
);
*execution.subject_delete_failure.lock().unwrap() = None;
api.config_store.with_conn(|conn| {
conn.execute_batch("CREATE TRIGGER injected_worker_catalog_failure BEFORE DELETE ON worker_registry BEGIN SELECT RAISE(ABORT,'injected catalog failure'); END;")?;
Ok(())
}).unwrap();
let (status, midpoint) = delete_request(
&api,
&subject.id,
"unknown-delete",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{midpoint}");
assert_eq!(midpoint["completed"], false);
assert!(matches!(
api.runtime.worker(&worker),
Err(RuntimeRegistryError::UnknownWorker { .. })
));
assert!(
api.store
.get_worker_registry(TEST_WORKSPACE_ID, &worker)
.unwrap()
.is_some()
);
api.config_store
.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER injected_worker_catalog_failure;")?;
Ok(())
})
.unwrap();
let (status, body) = delete_request(
&api,
&subject.id,
"unknown-delete",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true, "{body}");
assert!(
api.store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &format!("subjektiv:{}", subject.id))
.unwrap()
.is_none()
);
let removal = api
.config_store
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
.unwrap()
.unwrap();
assert_eq!(
removal.plan.state,
crate::retention::WorkerRemovalPlanState::Succeeded
);
assert_eq!(
removal.runtime_request.session_disposition,
worker_runtime::retention::SessionDisposition::Archive
);
assert!(
store
.subject_deletion(&subject.id)
.unwrap()
.unwrap()
.completed
);
}
#[tokio::test]
async fn captured_subject_job_result_cannot_deliver_after_domain_fence() {
let dir = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject(crate::subjektiv::SubjectRole::new("notification fence").unwrap())
.unwrap();
let target = spawn_ticket_check_source(&api, "subject-notification-target");
let worker = spawn_ticket_check_source(&api, "subject-notification-result");
let request = BackendJobRequest {
job_id: "captured-subject-notification".into(),
purpose: "subject_notification_contract".into(),
input_ref: "subject:notification".into(),
input: json!({"private":"input"}),
instruction: "Read input".into(),
profile: "builtin:backend-job".into(),
grants: crate::backend_job::BackendJobGrants {
subjektiv_consolidation: Some(crate::backend_job::SubjektivConsolidationGrant {
subject_id: subject.id.clone(),
candidate_ids: vec!["candidate".into()],
}),
..Default::default()
},
serialization_key: None,
source_worker: Some(target.clone()),
notification_target: Some(target),
limits: Default::default(),
};
// Enter at the committed result boundary: execution/result validation is
// outside this notification test, but routing and Runtime transport are real.
let now = Utc::now().to_rfc3339();
let reserved = api
.store
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &now)
.unwrap();
let attempt = &reserved.attempt.attempt_id;
api.store
.claim_backend_job_attempt_dispatch(TEST_WORKSPACE_ID, &request.job_id, attempt, &now)
.unwrap();
api.store
.bind_backend_job_attempt_worker(
TEST_WORKSPACE_ID,
&request.job_id,
attempt,
&worker,
Some("run"),
&now,
)
.unwrap();
let captured = api
.store
.accept_backend_job_result(
TEST_WORKSPACE_ID,
&worker,
&BackendJobResultSubmission {
job_id: request.job_id.clone(),
attempt_id: attempt.clone(),
input_digest: request.input_digest().unwrap(),
result: json!({"private":"result"}),
},
&now,
)
.unwrap();
let delivery = crate::backend_job::delivery_id(&request.job_id, attempt);
assert!(
api.store
.job_notification_source(TEST_WORKSPACE_ID, &request.job_id, attempt)
.is_ok()
);
execution.take_inputs();
store
.begin_subject_deletion(&subject.id, "notification-delete")
.unwrap();
notification_delivery::deliver_job_result(&api, &captured);
notification_delivery::recover_job_notifications(&api);
assert!(
execution.take_inputs().is_empty(),
"captured or recovered result must not escape the domain fence"
);
assert!(
api.store
.job_notification_request(TEST_WORKSPACE_ID, &delivery)
.unwrap()
.is_none()
);
}
#[tokio::test]
async fn domain_purge_failure_keeps_fence_and_backend_reopen_resumes_exact_operation() {
let dir = tempfile::tempdir().unwrap();
let api = test_api(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("retry purge").unwrap(),
"do not resurrect".into(),
)
.unwrap();
let other = store
.create_subject_with_behavior(
crate::subjektiv::SubjectRole::new("untouched").unwrap(),
"keep".into(),
)
.unwrap();
let database = api
.feature_storage
.open(&api.subjektiv_registration)
.unwrap();
database.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
tx.execute_batch("CREATE TRIGGER injected_subject_purge_failure BEFORE DELETE ON subjects BEGIN SELECT RAISE(ABORT,'injected domain purge failure'); END;")?;
Ok(())
}).unwrap();
let owner =
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-purge-failure").await;
let (status, body) = delete_request(
&api,
&subject.id,
"purge-retry",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert!(status.is_server_error(), "{status}: {body}");
assert!(
!store
.subject_deletion(&subject.id)
.unwrap()
.unwrap()
.completed
);
assert!(store.subject(&subject.id).unwrap().is_none());
assert_eq!(
store.subject(&other.id).unwrap().unwrap().behavior_md,
"keep"
);
database
.try_transaction::<_, feature_storage::FeatureStorageError>(|tx| {
tx.execute_batch("DROP TRIGGER injected_subject_purge_failure;")?;
Ok(())
})
.unwrap();
let mut reopened = api.clone();
let config_store =
Arc::new(SqliteWorkspaceStore::open_for_test(&api.config.database_path).unwrap());
reopened.config_store = config_store.clone();
reopened.store = config_store;
let recovery_api = reopened.clone();
let handle = tokio::runtime::Handle::current();
reopened
.host_executors
.host()
.unwrap()
.run(move || handle.block_on(subjektiv_deletion::recover(&recovery_api)))
.await
.unwrap();
let recovered = open_subjektiv_store(&reopened)
.unwrap()
.subject_deletion(&subject.id)
.unwrap()
.unwrap();
assert_eq!(recovered.operation_id, "purge-retry");
assert!(recovered.completed);
let (status, body) = delete_request(
&reopened,
&subject.id,
"purge-retry",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
assert_eq!(
store.subject(&other.id).unwrap().unwrap().behavior_md,
"keep"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn pending_consolidation_is_purged_and_captured_dispatch_and_late_result_are_rejected() {
let dir = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let subject = store
.create_subject(crate::subjektiv::SubjectRole::new("delete running Job").unwrap())
.unwrap();
let candidate: memory::extract::StagingRecord = serde_json::from_value(json!({
"schema_version": 2, "id": "delete-job-candidate", "extract_run_id": "extract",
"source": {"segment_id":"segment", "range":[1,2]}, "kind":"constraint",
"claim":"private candidate marker", "why_useful":"retained only until explicit erasure",
"evidence": [], "source_refs": []
}))
.unwrap();
store
.stage_candidate(crate::subjektiv::SubjectStagingRecord::attach(
&subject.id,
candidate,
))
.unwrap();
let started = start_subjektiv_staging_consolidation(
api.clone(),
&subject.id,
MemoryConsolidateStagingOperation { force: true },
)
.await
.unwrap();
assert_eq!(started.status, "started");
let job = api
.store
.find_active_backend_job_for_resource(
TEST_WORKSPACE_ID,
&crate::backend_job::subjektiv_serialization_key(&subject.id),
)
.unwrap()
.unwrap();
let attempt = api
.store
.get_backend_job_attempt(
TEST_WORKSPACE_ID,
&job.request.job_id,
&crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt),
)
.unwrap()
.unwrap();
let worker = attempt.worker.clone().unwrap();
seed_worker_session_for_cleanup(dir.path(), &worker);
let captured = BackendJobReservation {
job: job.clone(),
attempt: attempt.clone(),
replayed: true,
resource_reused: false,
};
let owner =
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-consolidation").await;
*execution.subject_delete_failure.lock().unwrap() = Some("completion unknown".into());
let (status, body) = delete_request(
&api,
&subject.id,
"delete-job",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], false);
assert!(
api.dispatch_backend_job_reservation(captured.clone())
.is_err()
);
assert!(
api.accept_backend_job_result(
&worker,
&BackendJobResultSubmission {
job_id: job.request.job_id.clone(),
attempt_id: attempt.attempt_id.clone(),
input_digest: job.input_digest.clone(),
result: json!({"late":true}),
}
)
.is_err()
);
*execution.subject_delete_failure.lock().unwrap() = None;
let (status, body) = delete_request(
&api,
&subject.id,
"delete-job",
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
assert!(
api.store
.get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id)
.unwrap()
.is_none()
);
assert!(
api.store
.get_backend_job_attempt(TEST_WORKSPACE_ID, &job.request.job_id, &attempt.attempt_id)
.unwrap()
.is_none()
);
assert!(api.dispatch_backend_job_reservation(captured).is_err());
assert!(api.dispatch_backend_job(&job.request).is_err());
let removal = api
.config_store
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
.unwrap()
.unwrap();
assert_eq!(
removal.plan.state,
crate::retention::WorkerRemovalPlanState::Succeeded
);
assert_eq!(
removal.runtime_request.session_disposition,
worker_runtime::retention::SessionDisposition::Purge
);
assert_eq!(
removal.runtime_request.diagnostics_disposition,
worker_runtime::retention::DiagnosticsDisposition::Purge
);
assert!(
!dir.path()
.join(".test-embedded-runtime-store/workers")
.join(&worker.worker_id)
.exists()
);
}
#[tokio::test]
async fn unmaterialized_body_and_job_reservations_finish_only_with_runtime_storage_absence_proof() {
let dir = tempfile::tempdir().unwrap();
let (api, _) = test_api_with_recording_backend(dir.path()).await;
let store = open_subjektiv_store(&api).unwrap();
let owner =
seed_test_workspace_owner_token(api.store.as_ref(), "subject-delete-unmaterialized").await;
for is_job in [false, true] {
let subject = store
.create_subject(crate::subjektiv::SubjectRole::new("reserved before deletion").unwrap())
.unwrap();
let singleton = (!is_job).then(|| format!("subjektiv:{}", subject.id));
let allocation = if is_job {
let request = BackendJobRequest {
job_id: "unmaterialized-subject-job".into(),
purpose: "subject_reservation_contract".into(),
input_ref: "subject:reserved".into(),
input: json!({"private":"reserved input"}),
instruction: "read".into(),
profile: "builtin:backend-job".into(),
grants: crate::backend_job::BackendJobGrants {
subjektiv_consolidation: Some(
crate::backend_job::SubjektivConsolidationGrant {
subject_id: subject.id.clone(),
candidate_ids: vec!["candidate".into()],
},
),
..Default::default()
},
serialization_key: None,
source_worker: None,
notification_target: None,
limits: Default::default(),
};
let reserved = api
.store
.reserve_backend_job(TEST_WORKSPACE_ID, &request, &Utc::now().to_rfc3339())
.unwrap();
crate::backend_job::allocation_key(&reserved.attempt.attempt_id)
} else {
"unmaterialized-subject-body".into()
};
let settings = api
.config_store
.get_workspace_memory_settings(TEST_WORKSPACE_ID)
.unwrap();
let reserved = api
.config_store
.reserve_worker_create(
TEST_WORKSPACE_ID,
EMBEDDED_WORKER_RUNTIME_ID,
&allocation,
"unmaterialized-fingerprint",
singleton.as_deref(),
&settings,
)
.unwrap();
let worker =
RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, reserved.worker_id.to_string());
assert!(matches!(
api.runtime.worker(&worker),
Err(RuntimeRegistryError::UnknownWorker { .. })
));
let (status, body) = delete_request(
&api,
&subject.id,
&format!("delete-reserved-{is_job}"),
Some(&owner),
TEST_WORKSPACE_ID,
)
.await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["completed"], true);
assert!(subject_authorization::worker_permissions(&api, &worker).is_err());
let proved = api.config_store.with_conn(|conn| Ok(conn.query_row("SELECT originals_absent FROM subject_deletion_workers WHERE workspace_id=?1 AND subject_id=?2 AND runtime_id=?3 AND worker_id=?4", rusqlite::params![TEST_WORKSPACE_ID,subject.id,worker.runtime_id,worker.worker_id], |row|row.get::<_,bool>(0))?)).unwrap();
assert!(proved);
assert!(
api.config_store
.recover_worker_removal_execution(TEST_WORKSPACE_ID, &worker)
.unwrap()
.is_none(),
"absence is not a fabricated Purge/archive receipt"
);
assert!(
api.config_store
.reserve_worker_create(
TEST_WORKSPACE_ID,
EMBEDDED_WORKER_RUNTIME_ID,
&allocation,
"unmaterialized-fingerprint",
singleton.as_deref(),
&settings
)
.is_err()
);
if let Some(singleton) = singleton {
assert!(
api.store
.current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton)
.unwrap()
.is_none()
);
}
}
}
@@ -1143,6 +1143,8 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
&Utc::now().to_rfc3339(),
)
.unwrap();
// Discard only the original delivery, before recovery can automatically retry.
f.execution.take_inputs();
f.api.recover_backend_job_worker_cleanups().unwrap();
wait_for_backend_job_worker_cleanup(
&f.api,
@@ -1151,23 +1153,64 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
BackendJobWorkerCleanupState::Completed,
)
.await;
// Cleanup commits before domain reconciliation; wait for its authoritative outcome.
tokio::time::timeout(std::time::Duration::from_secs(10), async {
// Cleanup commits before domain reconciliation. Job Failed is only transient:
// reconcile_unknown fences this attempt, then automatically reserves/queues its
// retry. Observe the immutable attempt outcome and the delivered new generation,
// not the Job's transient state; a manual retry would race the automatic one.
let reconciliation = tokio::time::timeout(std::time::Duration::from_secs(10), async {
loop {
let job = f
.api
.store
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
.unwrap()
.unwrap();
if job.state == BackendJobState::Failed {
break;
let original = f.api.store.get_backend_job_attempt(
TEST_WORKSPACE_ID,
&f.request.job_id,
&dispatched.attempt.attempt_id,
).unwrap().unwrap();
if original.state == BackendJobAttemptState::Failed {
assert_eq!(original.failure_category.as_deref(), Some("journal_attempt_fenced"));
assert_eq!(original.worker_cleanup_state, Some(BackendJobWorkerCleanupState::Completed));
assert!(original.worker_cleanup_completed_at.is_some());
if let Some(generation) = f.store.active_journal_generation(&f.host, &f.id).unwrap()
&& generation.job.attempt_id != original.attempt_id
&& let Some(retry) = f.api.store.get_backend_job_attempt(
TEST_WORKSPACE_ID,
&f.request.job_id,
&generation.job.attempt_id,
).unwrap()
&& retry.state == BackendJobAttemptState::Dispatched
&& let Some(resumed) = &retry.worker
// Worker binding precedes input delivery/attestation. ReadState
// succeeds only once the new generation's delivery is attested.
&& subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok()
{
break (original, retry, generation);
}
}
tokio::task::yield_now().await;
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.unwrap();
.await;
assert!(
reconciliation.is_ok(),
"post-cleanup reconciliation timed out: job={:?}, original_attempt={:?}, active_generation={:?}",
f.api
.store
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
.unwrap(),
f.api
.store
.get_backend_job_attempt(
TEST_WORKSPACE_ID,
&f.request.job_id,
&dispatched.attempt.attempt_id
)
.unwrap(),
f.store.active_journal_generation(&f.host, &f.id).unwrap(),
);
let (fenced, retry, generation) = reconciliation.unwrap();
assert_eq!(fenced.worker, dispatched.attempt.worker);
assert_eq!(retry.attempt, dispatched.attempt.attempt + 1);
assert_eq!(generation.job.job_id, f.request.job_id);
assert_eq!(generation.job.attempt_id, retry.attempt_id);
assert_eq!(retry.input_digest, f.request.input_digest().unwrap());
f.assert_purged(worker);
assert!(f.store.journal_retention_required(&f.host).unwrap());
assert_eq!(
@@ -1187,25 +1230,25 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_
1,
"source pin survives failed/partial Job cleanup"
);
f.execution.take_inputs();
let retry = f
let job = f
.api
.store
.reserve_backend_job_retry(
TEST_WORKSPACE_ID,
&f.request.job_id,
&Utc::now().to_rfc3339(),
)
.get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id)
.unwrap()
.unwrap();
assert_eq!(retry.job.request.input, f.request.input);
assert_eq!(retry.job.input_digest, f.request.input_digest().unwrap());
let retry = f.api.dispatch_backend_job_reservation(retry).unwrap();
let resumed = retry.attempt.worker.as_ref().unwrap();
assert_eq!(job.state, BackendJobState::Pending);
assert_eq!(job.current_attempt, retry.attempt);
assert_eq!(
job.load_executable_request().unwrap().input,
f.request.input
);
assert_eq!(job.input_digest, f.request.input_digest().unwrap());
let resumed = retry.worker.as_ref().unwrap();
let delivered = f.execution.take_inputs();
assert_eq!(delivered.len(), 1);
assert_eq!(
delivered[0].1,
f.request.worker_input(&retry.attempt.attempt_id).unwrap()
f.request.worker_input(&retry.attempt_id).unwrap()
);
assert_eq!(
f.store
+55 -8
View File
@@ -92,6 +92,13 @@ use crate::backend_job::{
use crate::workspace_deletion::WorkspaceDeletionStore;
use crate::{Error, Result};
#[path = "subject_deletion_store.rs"]
mod subject_deletion_store;
pub(crate) use subject_deletion_store::{
prior_removal_matches as subject_deletion_prior_removal_matches,
worker_kind as subject_deletion_worker_kind,
};
#[path = "activation.rs"]
mod activation;
pub use activation::ActivationCodeIssue;
@@ -119,7 +126,7 @@ pub use ticket::{
};
const OLDEST_SCHEMA_VERSION: i64 = 50;
const LATEST_SCHEMA_VERSION: i64 = 104;
const LATEST_SCHEMA_VERSION: i64 = 105;
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
@@ -447,6 +454,11 @@ const MIGRATIONS: &[Migration] = &[
name: "Converge Passkey results and Job input retention",
apply: migrate_schema_103_variants_to_v104,
},
Migration {
version: 105,
name: "Durable Subject deletion snapshots and Job admission fences",
apply: subject_deletion_store::migrate,
},
];
// Both schema-103 identities shipped. Keep the alternate implementation and accept
@@ -3000,6 +3012,7 @@ impl SqliteWorkspaceStore {
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
subject_deletion_store::require_worker_admission(&tx, workspace_id, allocation_key, singleton_key)?;
let fixed_worker_id = prepare(&tx)?;
let existing = tx
.query_row(
@@ -7455,6 +7468,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let deadline = backend_job_deadline(now, request.limits.timeout_seconds)?;
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
subject_deletion_store::require_job_admission(&tx, workspace_id, request)?;
if let Some(job) = read_backend_job(&tx, workspace_id, &request.job_id)? {
if job.intent_fingerprint != fingerprint {
return Err(Error::InvalidInput(format!(
@@ -7532,7 +7546,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
return Ok(job);
}
if reason == JobInputRetirementReason::Completed {
job.load_executable_request()?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))?
.ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?;
if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed
@@ -7562,6 +7577,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let mut job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let previous_id = attempt_id(job_id, job.current_attempt);
let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)?
@@ -7701,6 +7717,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
@@ -7764,6 +7781,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?;
@@ -7827,6 +7845,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)?
.ok_or_else(|| {
@@ -7880,6 +7899,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{}`", submission.job_id)))?;
let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)?
.ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?;
if submission.input_digest != job.input_digest
|| attempt.input_digest != submission.input_digest
@@ -7908,6 +7928,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
tx.commit()?;
return Ok(BackendJobResultAcceptance { job, attempt, replayed: true });
}
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
job.load_executable_request()?;
if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending {
return Err(Error::InvalidInput(
@@ -7972,6 +7993,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let job = read_backend_job(&tx, workspace_id, &expected.job_id)?.ok_or_else(|| Error::InvalidInput("Journal Job missing".into()))?;
subject_deletion_store::require_job_admission(&tx, workspace_id, &job.request)?;
let attempt = read_backend_job_attempt(&tx, workspace_id, &expected.job_id, &expected.attempt_id)?.ok_or_else(|| Error::InvalidInput("Journal attempt missing".into()))?;
// Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence.
if &attempt != expected || job.current_attempt != attempt.attempt
@@ -8277,6 +8299,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
) -> Result<(BackendJobDeliveryRecord, bool)> {
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let deleted:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM subject_deletion_jobs WHERE workspace_id=?1 AND job_id=?2)",params![workspace_id,job_id],|r|r.get(0))?;
if deleted {return Err(Error::RepositoryConflict("subject_deleted: Job delivery fenced".into()));}
if let Some(existing) = read_backend_job_delivery(&tx, workspace_id, delivery_id)? {
if existing.job_id != job_id || existing.target != *target {
return Err(Error::InvalidInput(
@@ -15545,6 +15569,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
notification_routing::jobs::install(conn)?;
notification_routing::install_business_events(conn)?;
notification_routing::upgrade_result_reports(conn)?;
subject_deletion_store::install_notification_erasure(conn)?;
ticket::verify_sqlite_ticket_schema(conn).map_err(|error| {
Error::Store(format!(
"fresh Workspace Ticket schema verification failed: {error}"
@@ -15583,10 +15608,17 @@ fn create_workspace_schema_structure_from_sql(
CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?;
conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?;
conn.execute_batch(&schema_sql.replace(
"-- Ticket-owned assignment schema is composed by store.rs.",
ticket::TICKET_ASSIGNMENT_SCHEMA_SQL,
))?;
conn.execute_batch(
&schema_sql
.replace(
"-- Ticket-owned assignment schema is composed by store.rs.",
ticket::TICKET_ASSIGNMENT_SCHEMA_SQL,
)
.replace(
"-- Subject deletion schema is composed by store.rs.",
include_str!("subject_deletion_schema.sql"),
),
)?;
let composed_sql = r#"
ALTER TABLE typed_tickets RENAME TO typed_tickets_legacy_v66;
CREATE TABLE typed_tickets (
@@ -15809,6 +15841,9 @@ fn verify_workspace_resource_constraints(conn: &Connection) -> Result<()> {
if current_schema_version(conn)? >= 96 {
notification_routing::jobs::verify(conn)?;
}
if current_schema_version(conn)? >= 105 {
subject_deletion_store::verify(conn)?;
}
Ok(())
}
@@ -16223,6 +16258,8 @@ fn current_worker_singleton_owner(
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r JOIN subject_deletion_workers d ON d.workspace_id=r.workspace_id AND d.runtime_id=r.runtime_id AND d.worker_id=r.worker_id WHERE r.workspace_id=owner.workspace_id AND r.runtime_id=owner.runtime_id AND r.worker_id=owner.worker_id AND r.state='succeeded' AND owner.singleton_key='subjektiv:'||d.subject_id AND (d.prior_removal_operation_id=r.operation_id OR (d.prior_removal_operation_id IS NULL AND d.is_job=0 AND r.session_disposition='archive' AND r.archive_retention_kind='forever' AND r.metadata_disposition='tombstone'))) \
AND NOT EXISTS(SELECT 1 FROM subject_deletion_workers d WHERE d.workspace_id=owner.workspace_id AND d.runtime_id=owner.runtime_id AND d.worker_id=owner.worker_id AND owner.singleton_key='subjektiv:'||d.subject_id AND d.originals_absent=1) \
THEN 'created' ELSE reservation.state END \
FROM worker_singleton_owners owner \
LEFT JOIN worker_create_reservations reservation \
@@ -19265,6 +19302,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
WorkspaceSchemaMigrationStep {version:103, name:"Passkey metadata and atomic addition results".into()},
WorkspaceSchemaMigrationStep {version:104, name:"Converge Passkey results and Job input retention".into()},
WorkspaceSchemaMigrationStep {version:105, name:"Durable Subject deletion snapshots and Job admission fences".into()},
]
);
@@ -19379,6 +19417,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(102, "Durable failed Subject creation recovery".to_string()),
(103, "Passkey metadata and atomic addition results".to_string()),
(104, "Converge Passkey results and Job input retention".to_string()),
(105, "Durable Subject deletion snapshots and Job admission fences".to_string()),
]
);
assert!(!table_exists(conn, "trusted_runtime_records")?);
@@ -22664,7 +22703,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
(101, "Durable explicit Subject body release".into()),
(102, "Durable failed Subject creation recovery".into()),
(103, "Passkey metadata and atomic addition results".into()),
(104, "Converge Passkey results and Job input retention".into())]);
(104, "Converge Passkey results and Job input retention".into()),
(105, "Durable Subject deletion snapshots and Job admission fences".into())]);
for (table, rows) in unchanged.iter().zip(before.iter()) {
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
}
@@ -26857,12 +26897,19 @@ INSERT INTO worker_registry (
.reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW)
.unwrap();
store.with_conn(|conn| {
// This fixture deliberately reconstructs schema 102 from fresh.
// Remove the newer Subject fence composition, not just its receipt.
let triggers=conn.prepare("SELECT name,tbl_name FROM sqlite_schema WHERE type='trigger' AND tbl_name LIKE 'notification_%' AND name NOT LIKE 'subject_deleted_%' AND sql LIKE '%subject_deletion_jobs%'")?
.query_map([],|r|Ok((r.get::<_,String>(0)?,r.get::<_,String>(1)?)))?.collect::<rusqlite::Result<Vec<_>>>()?;
for (name,table) in triggers {conn.execute_batch(&format!("DROP TRIGGER {name}; CREATE TRIGGER {name} BEFORE DELETE ON {table} BEGIN SELECT RAISE(ABORT,'Notification evidence is immutable'); END;"))?;}
for name in ["subject_deleted_job_insert","subject_deleted_job_result","subject_deleted_attempt_insert","subject_deleted_attempt_result","subject_deleted_worker_reservation","subject_deleted_notification_source","subject_deleted_notification_payload","subject_deleted_notification_send","subject_deletion_worker_prior_insert","subject_deletion_worker_identity_immutable"] {conn.execute_batch(&format!("DROP TRIGGER {name}"))?;}
conn.execute_batch("DROP TABLE subject_deletion_allocations; DROP TABLE subject_deletion_jobs; DROP TABLE subject_deletion_workers; DROP TABLE subject_deletions;")?;
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
DROP TRIGGER passkey_metadata_insert;
DROP TABLE passkey_addition_receipts;
DROP TABLE passkey_metadata;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
UPDATE __yoi_schema_migrations SET version=102 WHERE version=105;
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
}).unwrap();
drop(store);
@@ -0,0 +1,73 @@
-- Content-free, monotonic main-DB fence. The Host installs the domain fence first
-- and holds its Subject gate across snapshot/admission. No Subject contents live here.
CREATE TABLE subject_deletions (
workspace_id TEXT NOT NULL,
subject_id TEXT NOT NULL,
operation_id TEXT NOT NULL CHECK(length(operation_id)>0),
completed INTEGER NOT NULL DEFAULT 0 CHECK(completed IN (0,1)),
jobs_erased INTEGER NOT NULL DEFAULT 0 CHECK(jobs_erased IN (0,1)),
PRIMARY KEY(workspace_id,subject_id),
UNIQUE(workspace_id,operation_id),
FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE
);
CREATE TABLE subject_deletion_workers (
workspace_id TEXT NOT NULL,
subject_id TEXT NOT NULL,
runtime_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
is_job INTEGER NOT NULL CHECK(is_job IN (0,1)),
-- Exact canonical success already present at prepare, never a later receipt.
prior_removal_operation_id TEXT REFERENCES worker_removal_operations(operation_id),
-- Confirmed by the scoped Runtime storage owner, not catalog absence or Stop ACK.
originals_absent INTEGER NOT NULL DEFAULT 0 CHECK(originals_absent IN (0,1)),
PRIMARY KEY(workspace_id,subject_id,runtime_id,worker_id),
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
);
CREATE TRIGGER subject_deletion_worker_prior_insert BEFORE INSERT ON subject_deletion_workers
WHEN NEW.prior_removal_operation_id IS NOT NULL AND NOT EXISTS(
SELECT 1 FROM worker_removal_operations r WHERE r.operation_id=NEW.prior_removal_operation_id
AND r.workspace_id=NEW.workspace_id AND r.runtime_id=NEW.runtime_id
AND r.worker_id=NEW.worker_id AND r.state='succeeded')
BEGIN SELECT RAISE(ABORT,'Subject deletion prior removal must be an exact canonical success'); END;
CREATE TRIGGER subject_deletion_worker_identity_immutable BEFORE UPDATE OF
workspace_id,subject_id,runtime_id,worker_id,prior_removal_operation_id ON subject_deletion_workers
BEGIN SELECT RAISE(ABORT,'Subject deletion Worker snapshot identity is immutable'); END;
CREATE TRIGGER subject_deletion_original_absence_monotonic BEFORE UPDATE OF originals_absent ON subject_deletion_workers
WHEN OLD.originals_absent=1 AND NEW.originals_absent=0
BEGIN SELECT RAISE(ABORT,'Subject deletion original absence proof is permanent'); END;
-- Job/attempt identities remain after payload erasure, preventing stale allocation
-- or caller-selected Job-id reuse from recreating the deleted Subject's inputs.
CREATE TABLE subject_deletion_jobs (
workspace_id TEXT NOT NULL,
subject_id TEXT NOT NULL,
job_id TEXT NOT NULL,
PRIMARY KEY(workspace_id,job_id),
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
);
CREATE TABLE subject_deletion_allocations (
workspace_id TEXT NOT NULL,
subject_id TEXT NOT NULL,
allocation_key TEXT NOT NULL,
PRIMARY KEY(workspace_id,allocation_key),
FOREIGN KEY(workspace_id,subject_id) REFERENCES subject_deletions(workspace_id,subject_id) ON DELETE CASCADE
);
CREATE TRIGGER subject_deleted_job_insert BEFORE INSERT ON backend_jobs
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
(d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_journal.subject_id') OR
d.subject_id=json_extract(NEW.request_json,'$.grants.subjektiv_consolidation.subject_id')))
OR EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job admission fenced'); END;
CREATE TRIGGER subject_deleted_job_result BEFORE UPDATE OF state,result_json ON backend_jobs
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job result fenced'); END;
CREATE TRIGGER subject_deleted_attempt_insert BEFORE INSERT ON backend_job_attempts
WHEN EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job retry fenced'); END;
CREATE TRIGGER subject_deleted_attempt_result BEFORE UPDATE OF state,result_json,runtime_id,worker_id,runtime_run_id ON backend_job_attempts
WHEN NEW.state!='failed' AND EXISTS(SELECT 1 FROM subject_deletion_jobs d WHERE d.workspace_id=NEW.workspace_id AND d.job_id=NEW.job_id)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Job attempt fenced'); END;
CREATE TRIGGER subject_deleted_worker_reservation BEFORE INSERT ON worker_create_reservations
WHEN EXISTS(SELECT 1 FROM subject_deletions d WHERE d.workspace_id=NEW.workspace_id AND
NEW.singleton_key IN ('subjektiv:'||d.subject_id,'subjektiv-consolidation:'||d.subject_id))
OR EXISTS(SELECT 1 FROM subject_deletion_allocations d WHERE d.workspace_id=NEW.workspace_id AND d.allocation_key=NEW.allocation_key)
BEGIN SELECT RAISE(ABORT,'subject_deleted: Worker allocation fenced'); END;
File diff suppressed because it is too large Load Diff
@@ -94,6 +94,12 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[
"workdir_registry",
"workdir_removal_operations",
"worker_control_grants",
// Snapshot children precede their fence parent and canonical Worker removal
// receipts (subject_deletion_workers.prior_removal_operation_id).
"subject_deletion_allocations",
"subject_deletion_jobs",
"subject_deletion_workers",
"subject_deletions",
"subject_worker_releases",
"subject_create_recoveries",
"worker_create_reservations",
@@ -1276,6 +1282,134 @@ mod tests {
}
}
#[test]
fn workspace_purge_removes_subject_deletion_snapshots_and_preserves_other_workspace() {
let (store, owner, workspace_id) = setup();
let tables = [
"subject_deletion_allocations",
"subject_deletion_jobs",
"subject_deletion_workers",
"subject_deletions",
"worker_removal_operations",
];
// Check the explicit FK ordering as well as execution (the purge defers FKs).
let position = |table| {
WORKSPACE_DELETION_PURGE_TABLES
.iter()
.position(|name| *name == table)
.unwrap()
};
for child in &tables[..3] {
assert!(position(*child) < position("subject_deletions"));
}
assert!(position("subject_deletion_workers") < position("worker_removal_operations"));
for workspace in ["workspace-a", "workspace-b"] {
store.with_conn(|conn| {
conn.execute("INSERT INTO worker_removal_operations(
operation_id,plan_id,input_fingerprint,workspace_id,runtime_id,worker_id,
worker_updated_at,policy_id,policy_digest,session_disposition,metadata_disposition,
archive_retention_kind,diagnostics_disposition,blockers_json,state,reason,created_at,updated_at)
VALUES(?1,?1,'fingerprint',?1,'runtime','worker','1','policy','digest',
'purge','purge','forever','purge','[]','succeeded','removed','1','1')", [workspace])?;
conn.execute("INSERT INTO subject_deletions(workspace_id,subject_id,operation_id,completed,jobs_erased)
VALUES(?1,'subject','subject-delete',1,1)", [workspace])?;
conn.execute("INSERT INTO subject_deletion_workers(workspace_id,subject_id,runtime_id,worker_id,is_job,prior_removal_operation_id,originals_absent)
VALUES(?1,'subject','runtime','worker',1,?1,1)", [workspace])?;
conn.execute("INSERT INTO subject_deletion_jobs VALUES(?1,'subject','job')", [workspace])?;
conn.execute("INSERT INTO subject_deletion_allocations VALUES(?1,'subject','allocation')", [workspace])?;
Ok(())
}).unwrap();
}
let rows = |workspace: &str| {
store
.with_conn(|conn| {
tables
.iter()
.map(|table| {
let mut statement = conn
.prepare(&format!("SELECT * FROM {table} WHERE workspace_id=?1"))?;
let count = statement.column_count();
Ok(statement
.query_map([workspace], |row| {
(0..count)
.map(|index| row.get::<_, rusqlite::types::Value>(index))
.collect::<rusqlite::Result<Vec<_>>>()
})?
.collect::<rusqlite::Result<Vec<_>>>()?)
})
.collect::<Result<Vec<_>>>()
})
.unwrap()
};
let before = rows("workspace-a");
let other = rows("workspace-b");
assert!(before.iter().all(|table| table.len() == 1));
let preflight = store
.workspace_deletion_preflight(&owner, &workspace_id)
.unwrap();
let operation = store
.reserve_workspace_deletion(
&owner,
&workspace_id,
&WorkspaceDeletionRequest {
operation_id: "purge-subject-snapshots".into(),
expected_workspace_updated_at: preflight.expected_workspace_updated_at,
confirmation: "Alpha".into(),
},
)
.unwrap()
.operation;
store
.with_conn(|conn| {
conn.execute_batch(
"CREATE TRIGGER reject_subject_snapshot_purge BEFORE DELETE ON workspaces
BEGIN SELECT RAISE(ABORT,'injected purge failure'); END;",
)?;
Ok(())
})
.unwrap();
assert!(
store
.finalize_workspace_deletion(&operation.operation_id)
.is_err()
);
assert_eq!(
rows("workspace-a"),
before,
"rollback restores snapshots and canonical receipt"
);
assert_eq!(rows("workspace-b"), other);
store
.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER reject_subject_snapshot_purge")?;
Ok(())
})
.unwrap();
let completed = store
.finalize_workspace_deletion(&operation.operation_id)
.unwrap();
assert_eq!(completed.state, WorkspaceDeletionState::Succeeded);
assert!(rows("workspace-a").iter().all(|table| table.is_empty()));
assert_eq!(rows("workspace-b"), other);
assert_eq!(
store
.finalize_workspace_deletion(&operation.operation_id)
.unwrap(),
completed
);
store
.with_conn(|conn| {
assert_eq!(
conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| {
row.get::<_, i64>(0)
})?,
0
);
Ok(())
})
.unwrap();
}
#[test]
fn deletion_is_idempotent_and_removes_workspace_scoped_rows() {
let (store, owner, workspace_id) = setup();
+169 -114
View File
@@ -11127,6 +11127,40 @@
],
"type": "object"
},
"SubjektivSubjectDeleteRequest": {
"additionalProperties": false,
"properties": {
"operation_id": {
"description": "Stable identity retained for every retry, including unknown outcomes.",
"type": "string"
}
},
"required": [
"operation_id"
],
"type": "object"
},
"SubjektivSubjectDeleteResponse": {
"additionalProperties": false,
"properties": {
"completed": {
"description": "False confirms the deletion fence, not external stop or erasure completion.",
"type": "boolean"
},
"operation_id": {
"type": "string"
},
"subject_id": {
"type": "string"
}
},
"required": [
"subject_id",
"operation_id",
"completed"
],
"type": "object"
},
"SubjektivSubjectListQuery": {
"additionalProperties": false,
"properties": {
@@ -11228,11 +11262,19 @@
"type": "object"
},
"SubjektivSubjectState": {
"enum": [
"active",
"retired"
],
"type": "string"
"oneOf": [
{
"enum": [
"active"
],
"type": "string"
},
{
"const": "retired",
"description": "Saved compatibility only; not a current lifecycle operation.",
"type": "string"
}
]
},
"SubjektivWorkerConnectionRequest": {
"additionalProperties": false,
@@ -17129,6 +17171,10 @@
"additionalProperties": false,
"description": "Backend-authoritative permissions for the current Workspace actor.",
"properties": {
"delete_subjects": {
"description": "Explicit Subject deletion authority for the current authenticated actor.",
"type": "boolean"
},
"delete_workspace": {
"type": "boolean"
},
@@ -17146,7 +17192,8 @@
"manage_repositories",
"manage_secrets",
"manage_runtimes",
"delete_workspace"
"delete_workspace",
"delete_subjects"
],
"type": "object"
},
@@ -18302,7 +18349,7 @@
"info": {
"title": "Yoi Server API",
"version": "0.1.0",
"x-yoi-source-digest": "sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4"
"x-yoi-source-digest": "sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e"
},
"openapi": "3.1.0",
"paths": {
@@ -35014,6 +35061,121 @@
}
},
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}": {
"delete": {
"operationId": "subjektiv_subject_delete",
"parameters": [
{
"explode": false,
"in": "path",
"name": "workspace_id",
"required": true,
"schema": {
"$ref": "#/components/schemas/string"
},
"style": "simple"
},
{
"explode": false,
"in": "path",
"name": "subject_id",
"required": true,
"schema": {
"$ref": "#/components/schemas/string"
},
"style": "simple"
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SubjektivSubjectDeleteRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SubjektivSubjectDeleteResponse"
}
}
},
"description": "Successful response"
},
"400": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"401": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"403": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"404": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"409": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"500": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
}
},
"security": [
{
"bearerAuth": []
},
{
"browserSession": []
}
]
},
"get": {
"operationId": "subjektiv_subject_get",
"parameters": [
@@ -35790,113 +35952,6 @@
]
}
},
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/retire": {
"post": {
"operationId": "subjektiv_subject_retire",
"parameters": [
{
"explode": false,
"in": "path",
"name": "workspace_id",
"required": true,
"schema": {
"$ref": "#/components/schemas/string"
},
"style": "simple"
},
{
"explode": false,
"in": "path",
"name": "subject_id",
"required": true,
"schema": {
"$ref": "#/components/schemas/string"
},
"style": "simple"
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SubjektivSubjectResponse"
}
}
},
"description": "Successful response"
},
"400": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"401": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"403": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"404": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"409": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
},
"500": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepositoryApiError"
}
}
},
"description": "Error response"
}
},
"security": [
{
"bearerAuth": []
},
{
"browserSession": []
}
]
}
},
"/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/surface": {
"get": {
"operationId": "subjektiv_resident_surface",
@@ -32,6 +32,14 @@
}
]
},
{
"path": "crates/subjektiv/src/deletion/tests.rs",
"kind": "saved_data",
"reason": "schema 6の固定保存Subject行を投入し、最新migrationが旧retiredとarchive原文を黙って消去・再活性化せず、明示削除だけで消去する検証に必要な旧SQL列名。現行writerや競合制御には使わない。",
"lines": [
{ "sha256": "defca5951b3d48785514b25912044fa3fc252659311b09a31fa2e64af2a19049", "count": 1 }
]
},
{
"path": "crates/subjektiv/src/journal/memory_links/tests.rs",
"kind": "saved_data",
@@ -0,0 +1,162 @@
{
"schemaVersion": 1,
"id": "subject-delete-dark",
"title": "Subject deletion confirmation and retry",
"baseUrl": "http://127.0.0.1:15287",
"colorScheme": "dark",
"reducedMotion": "reduce",
"personas": [
{
"id": "fixture",
"label": "Authenticated owner (synthetic HTTP)",
"auth": {
"kind": "anonymous"
}
}
],
"viewports": [
{
"label": "desktop",
"width": 1440,
"height": 900
},
{
"label": "narrow",
"width": 768,
"height": 900
},
{
"label": "mobile",
"width": 390,
"height": 844
}
],
"routes": [
{
"id": "subject",
"label": "Subject detail",
"path": "/w/memory-review/memory/pending-subject",
"goal": "Review the Subject before irreversible deletion.",
"dataState": "Active Subject, no current Worker; owner permissions from Backend projection.",
"ready": {
"kind": "selector",
"selector": ".subject-overview"
},
"capturePoints": [
{
"id": "initial",
"label": "Subject detail"
},
{
"id": "confirmation",
"label": "Irreversible deletion scope and confirmation",
"interaction": [
{
"action": "click",
"selector": ".subject-delete-open"
}
],
"ready": {
"kind": "selector",
"selector": "[data-delete-state=confirming]"
}
},
{
"id": "pending",
"label": "Fence saved; erasure unconfirmed",
"interaction": [
{
"action": "click",
"selector": ".delete-confirm input"
},
{
"action": "click",
"selector": ".subject-deletion button[type=submit]"
}
],
"ready": {
"kind": "selector",
"selector": "[data-delete-state=pending]"
}
},
{
"id": "completed",
"label": "Deletion confirmed",
"interaction": [
{
"action": "click",
"selector": ".subject-delete-retry"
}
],
"ready": {
"kind": "selector",
"selector": "[data-delete-state=completed]"
}
}
]
},
{
"id": "legacy",
"label": "Legacy ended Subject",
"path": "/w/memory-review/memory/legacy-subject",
"goal": "Delete a legacy ended Subject without reactivating it.",
"dataState": "Saved retired wire state, owner.",
"ready": {
"kind": "selector",
"selector": ".subject-delete-open"
},
"capturePoints": [
{
"id": "initial",
"label": "Legacy ended Subject"
},
{
"id": "confirmation",
"label": "Delete legacy Subject",
"interaction": [
{
"action": "click",
"selector": ".subject-delete-open"
}
],
"ready": {
"kind": "selector",
"selector": "[data-delete-state=confirming]"
}
}
]
},
{
"id": "non-owner",
"label": "Nonowner Subject view",
"path": "/w/memory-review/memory/legacy-subject?persona=reader",
"goal": "Read Subject without presenting unauthorized deletion.",
"dataState": "Backend delete_subjects=false; no deletion controls.",
"ready": {
"kind": "selector",
"selector": ".subject-overview"
},
"capturePoints": [
{
"id": "initial",
"label": "Nonowner details"
}
]
}
],
"processes": [
{
"id": "subject-delete-fixture",
"command": "deno",
"cwd": "..",
"args": [
"run",
"-A",
"../../web/workspace/test/subject-delete-browser/fixture-server.ts",
"15287",
"../../web/workspace/build"
],
"readyUrl": "http://127.0.0.1:15287/health"
}
]
}
@@ -0,0 +1,162 @@
{
"schemaVersion": 1,
"id": "subject-delete-light",
"title": "Subject deletion confirmation and retry",
"baseUrl": "http://127.0.0.1:15287",
"colorScheme": "light",
"reducedMotion": "reduce",
"personas": [
{
"id": "fixture",
"label": "Authenticated owner (synthetic HTTP)",
"auth": {
"kind": "anonymous"
}
}
],
"viewports": [
{
"label": "desktop",
"width": 1440,
"height": 900
},
{
"label": "narrow",
"width": 768,
"height": 900
},
{
"label": "mobile",
"width": 390,
"height": 844
}
],
"routes": [
{
"id": "subject",
"label": "Subject detail",
"path": "/w/memory-review/memory/pending-subject",
"goal": "Review the Subject before irreversible deletion.",
"dataState": "Active Subject, no current Worker; owner permissions from Backend projection.",
"ready": {
"kind": "selector",
"selector": ".subject-overview"
},
"capturePoints": [
{
"id": "initial",
"label": "Subject detail"
},
{
"id": "confirmation",
"label": "Irreversible deletion scope and confirmation",
"interaction": [
{
"action": "click",
"selector": ".subject-delete-open"
}
],
"ready": {
"kind": "selector",
"selector": "[data-delete-state=confirming]"
}
},
{
"id": "pending",
"label": "Fence saved; erasure unconfirmed",
"interaction": [
{
"action": "click",
"selector": ".delete-confirm input"
},
{
"action": "click",
"selector": ".subject-deletion button[type=submit]"
}
],
"ready": {
"kind": "selector",
"selector": "[data-delete-state=pending]"
}
},
{
"id": "completed",
"label": "Deletion confirmed",
"interaction": [
{
"action": "click",
"selector": ".subject-delete-retry"
}
],
"ready": {
"kind": "selector",
"selector": "[data-delete-state=completed]"
}
}
]
},
{
"id": "legacy",
"label": "Legacy ended Subject",
"path": "/w/memory-review/memory/legacy-subject",
"goal": "Delete a legacy ended Subject without reactivating it.",
"dataState": "Saved retired wire state, owner.",
"ready": {
"kind": "selector",
"selector": ".subject-delete-open"
},
"capturePoints": [
{
"id": "initial",
"label": "Legacy ended Subject"
},
{
"id": "confirmation",
"label": "Delete legacy Subject",
"interaction": [
{
"action": "click",
"selector": ".subject-delete-open"
}
],
"ready": {
"kind": "selector",
"selector": "[data-delete-state=confirming]"
}
}
]
},
{
"id": "non-owner",
"label": "Nonowner Subject view",
"path": "/w/memory-review/memory/legacy-subject?persona=reader",
"goal": "Read Subject without presenting unauthorized deletion.",
"dataState": "Backend delete_subjects=false; no deletion controls.",
"ready": {
"kind": "selector",
"selector": ".subject-overview"
},
"capturePoints": [
{
"id": "initial",
"label": "Nonowner details"
}
]
}
],
"processes": [
{
"id": "subject-delete-fixture",
"command": "deno",
"cwd": "..",
"args": [
"run",
"-A",
"../../web/workspace/test/subject-delete-browser/fixture-server.ts",
"15287",
"../../web/workspace/build"
],
"readyUrl": "http://127.0.0.1:15287/health"
}
]
}
+1 -1
View File
@@ -6,7 +6,7 @@
"dev": "deno run -A npm:vite@8.3.2 dev",
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
"check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json",
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts tests/subject-delete.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
"test:journal:backend-contract": "cd ../.. && mkdir -p target && cargo test -p yoi-workspace-server --lib journal_browser_web_contract -- --nocapture > target/journal-browser-contract.log && cd web/workspace && deno run --allow-read=../../target/journal-browser-contract.log test/journal-backend-contract.ts ../../target/journal-browser-contract.log",
"test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts",
"test:browser:activation": "mkdir -p ../../target/web-ux/tmp && TMPDIR=../../target/web-ux/tmp deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/activation-browser/activation-shell.browser.test.ts",
+7
View File
@@ -3,6 +3,7 @@
"specifiers": {
"jsr:@std/assert@*": "1.0.19",
"jsr:@std/internal@^1.0.12": "1.0.14",
"jsr:@std/path@1.1.4": "1.1.4",
"npm:@codemirror/autocomplete@6.20.3": "6.20.3",
"npm:@codemirror/commands@6.11.1": "6.11.1",
"npm:@codemirror/language@6.12.4": "6.12.4",
@@ -42,6 +43,12 @@
},
"@std/internal@1.0.14": {
"integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7"
},
"@std/path@1.1.4": {
"integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5",
"dependencies": [
"jsr:@std/internal"
]
}
},
"npm": {
@@ -1,6 +1,6 @@
// @generated by yoi-companion-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
// Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/companion-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check
@@ -52,6 +52,10 @@ export type WorkspacePermissionSummary = {
manage_secrets: boolean;
manage_runtimes: boolean;
delete_workspace: boolean;
/**
* Explicit Subject deletion authority for the current authenticated actor.
*/
delete_subjects: boolean;
};
export type WorkspaceDeletionState =
@@ -118,6 +118,14 @@ export type SubjektivSubjectCreateRequest = {
behavior_md?: string;
};
export type SubjektivSubjectDeleteRequest = { operation_id: string };
export type SubjektivSubjectDeleteResponse = {
subject_id: string;
operation_id: string;
completed: boolean;
};
export type SubjektivSubjectBehaviorUpdateRequest = {
expected_behavior_md: string;
behavior_md: string;
@@ -1,6 +1,6 @@
// @generated by yoi-repository-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
// Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
// Output: web/workspace/src/lib/generated/repository-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-runtime-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
// Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/runtime-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
// Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/ticket-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-worker-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:7e640cff6bdcf6dc298cfca695c5cb71a520288b20f4f29ec8df8138724b97c4)
// Input contract: 0.1.0 (sha256:fc3064f2d7b0d2c2b11e166007943b0cac0cad0918e34d071e20079cf38d655e)
// Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/worker-launch-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
@@ -482,6 +482,7 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary {
"manage_secrets",
"manage_runtimes",
"delete_workspace",
"delete_subjects",
],
path,
);
@@ -492,6 +493,7 @@ function permissions(value: unknown, path: string): WorkspacePermissionSummary {
),
manage_secrets: boolean(item.manage_secrets, `${path}.manage_secrets`),
manage_runtimes: boolean(item.manage_runtimes, `${path}.manage_runtimes`),
delete_subjects: boolean(item.delete_subjects, `${path}.delete_subjects`),
delete_workspace: boolean(
item.delete_workspace,
`${path}.delete_workspace`,
@@ -0,0 +1,134 @@
<script lang="ts">
import { untrack } from 'svelte';
import { deleteSubjektivSubject, SubjektivSubjectDeleteError } from './api.ts';
import { clearSubjectDeleteAttempt, readSubjectDeleteAttempt, saveSubjectDeleteAttempt, type SubjectDeleteAttempt } from './subject-delete.ts';
import type { SubjektivSubjectResponse } from '#lib/generated/memory-api.ts';
import { workspaceRoute } from '#lib/workspace/api/http.ts';
let { workspaceId, subjectId, subject, onstarted }: {
workspaceId: string;
subjectId: string;
subject: SubjektivSubjectResponse | null;
onstarted: () => void;
} = $props();
let attempt = $state<SubjectDeleteAttempt | null>(null);
let phase = $state<'idle' | 'confirming' | 'running' | 'pending' | 'unknown' | 'failed' | 'completed'>('idle');
let confirmed = $state(false);
let error = $state<string | null>(null);
let storageError = $state(false);
let identity = '';
let generation = 0;
$effect(() => {
const next = JSON.stringify([workspaceId, subjectId]);
untrack(() => {
if (identity === next) return;
identity = next;
generation += 1;
attempt = null;
phase = 'idle';
confirmed = false;
error = null;
storageError = false;
try {
attempt = readSubjectDeleteAttempt(sessionStorage, workspaceId, subjectId);
if (attempt) {
phase = 'unknown';
onstarted();
}
} catch (cause) {
storageError = true;
error = cause instanceof Error ? cause.message : 'Cannot read the saved deletion request. Do not start a new request.';
}
});
});
async function submit(): Promise<void> {
if (phase === 'running' || phase === 'completed' || storageError || (!attempt && (!subject || !confirmed))) return;
const currentGeneration = generation;
const targetWorkspace = workspaceId;
const targetSubject = subjectId;
const request = attempt ?? { subjectId: targetSubject, operationId: crypto.randomUUID() };
const current = () => currentGeneration === generation && targetWorkspace === workspaceId && targetSubject === subjectId;
// 保存に失敗したら送らない。応答喪失後に新しいkeyを作る経路を残さない。
try { saveSubjectDeleteAttempt(sessionStorage, targetWorkspace, request); }
catch {
error = 'The retry identity could not be saved in this tab. No deletion request was sent. Allow session storage and try again.';
return;
}
attempt = request;
phase = 'running';
error = null;
onstarted();
try {
const result = await deleteSubjektivSubject(fetch, targetWorkspace, targetSubject, { operation_id: request.operationId });
// 対象変更後の遅延応答は新対象を更新せず、成功した旧対象の再送入力だけ消去する。
if (result.completed) {
try { clearSubjectDeleteAttempt(sessionStorage, targetWorkspace, targetSubject); }
catch { /* 消去できなくても残るのは同一keyだけ。本文や成功状態は保存しない。 */ }
}
if (!current()) return;
phase = result.completed ? 'completed' : 'pending';
} catch (cause) {
if (!current()) return;
phase = cause instanceof SubjektivSubjectDeleteError && cause.kind === 'rejected' ? 'failed' : 'unknown';
error = cause instanceof Error ? cause.message : 'The deletion outcome is unknown.';
}
}
</script>
{#if subject || attempt || storageError}
<section class="subject-deletion" aria-labelledby="subject-delete-heading" data-delete-state={phase}>
<header>
<h2 id="subject-delete-heading">Delete Subject</h2>
{#if phase === 'idle' && !storageError}
<button type="button" class="workspace-secondary-button subject-delete-open" onclick={() => { phase = 'confirming'; }}>Delete Subject…</button>
{/if}
</header>
{#if phase === 'confirming' || attempt}
<p class="delete-target">Target: <strong>{subject?.role ?? subjectId}</strong> · <code>{subjectId}</code></p>
{/if}
{#if phase === 'confirming'}
<form onsubmit={(event) => { event.preventDefault(); void submit(); }}>
<p><strong>This is irreversible.</strong> This Subject, its behavior, Memory, change history, staging, Surface, Journal, permissions, delegations, Writer/Session associations, and Subject Job inputs/results will be deleted. Connected and delegated Workers and Jobs must stop before completion can be confirmed.</p>
<p>Original Worker/Session text, existing archives and Workdirs, Tickets, and other separately retained resources are <strong>not deleted</strong>. Minimal deletion retry/audit metadata is retained. Other Subjects and Workspaces are not affected.</p>
<label class="delete-confirm"><input type="checkbox" bind:checked={confirmed} /> I have checked the target, deletion scope, retained data, and irreversible effect.</label>
<div class="delete-actions">
<button type="submit" class="workspace-danger-button" disabled={!confirmed}>Permanently delete Subject</button>
<button type="button" class="workspace-secondary-button" onclick={() => { phase = 'idle'; confirmed = false; }}>Cancel</button>
</div>
</form>
{:else if phase === 'running'}
<p role="status">Deleting… Waiting for stop and erasure confirmation. Do not treat this as completed.</p>
{:else if phase === 'pending'}
<p role="status"><strong>Deletion started; completion unconfirmed.</strong> The deletion fence is saved and the Subject is hidden from reads/lists. External stopping or erasure is not yet confirmed. Retry the same request to continue.</p>
{:else if phase === 'unknown'}
<p role="status"><strong>Deletion outcome unknown.</strong> The request may already have taken effect. The saved target and operation ID are retained; retry the same request, not a new deletion.</p>
{:else if phase === 'failed'}
<p role="alert"><strong>Deletion request rejected.</strong> Completion is not confirmed. Resolve the reported condition, then retry the same request.</p>
{:else if phase === 'completed'}
<p role="status"><strong>Subject deletion completed.</strong> Backend confirmed stopping and deletion of Subject-owned data. Separately retained Worker/Session text, archives, Workdirs, and Tickets remain.</p>
<a href={workspaceRoute(workspaceId, '/memory')}>Back to subjects →</a>
{/if}
{#if error}<p class="workspace-callout is-error" role="alert">{error}</p>{/if}
{#if attempt && phase !== 'completed'}
<details><summary>Deletion request identity</summary><code>{attempt.operationId}</code><p>This tab retains the target and same request ID across reloads, even when GET can no longer find the Subject.</p></details>
{#if phase === 'pending' || phase === 'unknown' || phase === 'failed'}
<button type="button" class="workspace-secondary-button subject-delete-retry" onclick={() => { void submit(); }}>Retry same deletion request</button>
{/if}
{/if}
</section>
{/if}
<style>
.subject-deletion { min-width: 0; }
header { display: flex; align-items: center; justify-content: space-between; gap: .75rem; flex-wrap: wrap; }
h2 { font-size: var(--font-size-body); margin: 0; }
p { line-height: 1.6; }
code, .delete-target { overflow-wrap: anywhere; }
.delete-confirm { display: flex; align-items: flex-start; gap: .6rem; line-height: 1.5; }
input { margin-top: .25rem; flex: 0 0 auto; }
.delete-actions { display: flex; gap: .75rem; flex-wrap: wrap; margin-top: 1rem; }
details { margin-block: 1rem; }
summary { cursor: pointer; }
</style>
@@ -29,6 +29,8 @@ import type {
SubjektivResidentSurfaceAvailability,
SubjektivResidentSurfaceResponse,
SubjektivResidentSurfaceSnapshot,
SubjektivSubjectDeleteRequest,
SubjektivSubjectDeleteResponse,
SubjektivSubjectBehaviorUpdateRequest,
SubjektivSubjectCreateRequest,
SubjektivSubjectListResponse,
@@ -158,6 +160,70 @@ function validateSubjectBehavior(behaviorMd: string): void {
}
}
export class SubjektivSubjectDeleteError extends Error {
constructor(
readonly kind: "rejected" | "unknown_outcome",
message: string,
readonly status: number | null = null,
) {
super(message);
this.name = "SubjektivSubjectDeleteError";
}
}
export function parseSubjektivSubjectDeleteResponse(
value: unknown,
subjectId: string,
operationId: string,
): SubjektivSubjectDeleteResponse {
const record = strictRecord(value, ["subject_id", "operation_id", "completed"], "Subject deletion");
const response = {
subject_id: requiredIdentifier(record, "subject_id"),
operation_id: requiredIdentifier(record, "operation_id"),
completed: requiredBoolean(record, "completed"),
};
if (response.subject_id !== subjectId || response.operation_id !== operationId) {
invalid("Subject deletion response does not match the request");
}
return response;
}
/** HTTP success can confirm only the deletion fence. Every retry retains this request. */
export async function deleteSubjektivSubject(
fetchFn: typeof fetch,
workspaceId: string,
subjectId: string,
request: SubjektivSubjectDeleteRequest,
): Promise<SubjektivSubjectDeleteResponse> {
identifier(request.operation_id, "operation_id");
let response: Response;
try {
response = await fetchFn(workspaceApiPath(workspaceId, `/subjektiv/subjects/${encodeURIComponent(subjectId)}`), {
method: "DELETE",
headers: { "content-type": "application/json" },
body: JSON.stringify({ operation_id: request.operation_id }),
});
} catch {
throw new SubjektivSubjectDeleteError("unknown_outcome", "The deletion outcome is unknown. Retry the same request.");
}
if (!response.ok) {
throw new SubjektivSubjectDeleteError(
response.status >= 500 ? "unknown_outcome" : "rejected",
await subjektivSubjectCreateErrorMessage(response),
response.status,
);
}
try {
return parseSubjektivSubjectDeleteResponse(
await readBoundedJson(response, MEMORY_API_LIMITS.maxResponseBytes),
subjectId,
request.operation_id,
);
} catch {
throw new SubjektivSubjectDeleteError("unknown_outcome", "The deletion response could not be confirmed. Retry the same request.");
}
}
export async function createSubjektivSubject(
fetchFn: typeof fetch,
workspaceId: string,
@@ -0,0 +1,51 @@
// 削除開始後はGETから対象が消えるため、同じtabでの再読込みにも依頼IDを保持する。
// Subject本文は保存しない。これは削除結果の正本ではなく、安全な再送用の最小入力。
export type SubjectDeleteAttempt = {
subjectId: string;
operationId: string;
};
export function subjectDeleteStorageKey(
workspaceId: string,
subjectId: string,
): string {
return `yoi.subject-delete:${JSON.stringify([workspaceId, subjectId])}`;
}
export function readSubjectDeleteAttempt(
storage: Pick<Storage, "getItem">,
workspaceId: string,
subjectId: string,
): SubjectDeleteAttempt | null {
const raw = storage.getItem(subjectDeleteStorageKey(workspaceId, subjectId));
if (raw === null) return null;
const value = JSON.parse(raw);
if (
!value || value.subjectId !== subjectId ||
typeof value.operationId !== "string" ||
value.operationId.length === 0 ||
Object.keys(value).some((key) =>
key !== "subjectId" && key !== "operationId"
)
) {
throw new Error(
"The saved deletion request cannot be read. Do not start another deletion request.",
);
}
return value;
}
export function saveSubjectDeleteAttempt(
storage: Pick<Storage, "setItem">,
workspaceId: string,
attempt: SubjectDeleteAttempt,
): void {
storage.setItem(
subjectDeleteStorageKey(workspaceId, attempt.subjectId),
JSON.stringify(attempt),
);
}
export function clearSubjectDeleteAttempt(
storage: Pick<Storage, "removeItem">,
workspaceId: string,
subjectId: string,
): void {
storage.removeItem(subjectDeleteStorageKey(workspaceId, subjectId));
}
@@ -38,7 +38,7 @@
}
function subjectStateLabel(state: SubjektivSubjectResponse['state']): string {
return state === 'active' ? 'Available' : 'Retired';
return state === 'active' ? 'Available' : 'Legacy ended Subject';
}
async function openCreateForm(): Promise<void> {
@@ -6,6 +6,7 @@
// 実行中Workerの履歴へ反映済みであることの証明には使わない。診断値は詳細へ分離する。
import { untrack } from 'svelte';
import DocumentMarkdown from '#lib/workspace/markdown/DocumentMarkdown.svelte';
import SubjectDelete from '#lib/workspace/memory/SubjectDelete.svelte';
import { formatDate, workspaceRoute } from '#lib/workspace/api/http.ts';
import {
SubjektivSubjectCreateError,
@@ -26,6 +27,7 @@
let behaviorSaving = $state(false);
let behaviorError = $state<string | null>(null);
let behaviorSaveConfirmed = $state(false);
let deletionStarted = $state(false);
let routeGeneration = 0;
let subjectWorkspaceId = untrack(() => data.workspaceId);
@@ -49,6 +51,7 @@
behaviorSaving = false;
behaviorError = null;
behaviorSaveConfirmed = false;
deletionStarted = false;
});
});
@@ -149,7 +152,7 @@
}
function subjectStateLabel(state: 'active' | 'retired'): string {
return state === 'active' ? 'Available' : 'Retired';
return state === 'active' ? 'Available' : 'Legacy ended Subject';
}
</script>
@@ -167,9 +170,16 @@
<h1 id="subject-heading">{subject?.role ?? data.subjectId}</h1>
<code class="subject-id" title={data.subjectId}>{data.subjectId}</code>
</div>
<span class="read-only-label">Behavior editable</span>
{#if !deletionStarted}<span class="read-only-label">Behavior editable</span>{/if}
</header>
{#if data.workspace.permissions.delete_subjects}
{#key JSON.stringify([data.workspaceId, data.subjectId])}
<SubjectDelete workspaceId={data.workspaceId} subjectId={data.subjectId} {subject} onstarted={() => { deletionStarted = true; }} />
{/key}
{/if}
{#if !deletionStarted}
<nav aria-label="Subjectの記録"><a href={workspaceRoute(data.workspaceId, `/memory/${encodeURIComponent(data.subjectId)}/journal`)}>Journal · 最近の出来事とSessionを探す →</a></nav>
{#if subject}
@@ -388,6 +398,7 @@
<div class="memory-state" role="status"><p>Current Memory data is unavailable.</p></div>
{/if}
</section>
{/if}
</section>
<style>
@@ -0,0 +1,142 @@
// 本番Web buildと合成HTTP境界を接続する。実認証・削除の永続性はBackend suiteの責務。
import { join, resolve } from "jsr:@std/path@1.1.4";
const [portText, build, mode] = Deno.args;
const port = Number(portText);
const root = resolve(import.meta.dirname!, "../../../..");
const upstreamPort = port + 1;
const upstream = new Deno.Command(Deno.execPath(), {
args: [
"run",
"-A",
join(root, "tools/web-ux/browser-tests/memory_document_fixture_server.ts"),
String(upstreamPort),
resolve(build),
"3",
],
stdout: "null",
stderr: "inherit",
}).spawn();
const stop = () => {
try {
upstream.kill();
} catch { /* already stopped */ }
};
Deno.addSignalListener("SIGTERM", () => {
stop();
Deno.exit();
});
Deno.addSignalListener("SIGINT", () => {
stop();
Deno.exit();
});
const attempts = new Map<string, string>();
const hidden = new Set<string>();
const user = {
user_id: "fixture-user",
account_id: "owner",
handle: "owner",
display_name: "Memory owner",
};
const apiRoot = "/api/w/memory-review";
Deno.serve({ hostname: "127.0.0.1", port }, async (request) => {
const url = new URL(request.url);
if (url.pathname === "/health") {
try {
return await fetch(`http://127.0.0.1:${upstreamPort}/health`);
} catch {
return new Response(null, { status: 503 });
}
}
if (url.pathname === "/api/auth/whoami") {
return Response.json({
authentication_reference: "subject-delete-fixture",
actor: { ...user, auth_method: "browser_session" },
session: { kind: "authenticated", expires_at: "2099-01-01T00:00:00Z" },
});
}
const sessionId =
request.headers.get("cookie")?.match(/delete-review-session=([^;]+)/)
?.[1] ?? crypto.randomUUID();
const reader =
request.headers.get("cookie")?.includes("subject-delete-persona=reader") ??
false;
const match = url.pathname.match(/\/subjektiv\/subjects\/([^/]+)(?:\/|$)/);
const id = match ? decodeURIComponent(match[1]) : null;
const key = JSON.stringify([sessionId, id]);
if (request.method === "DELETE" && id) {
if (reader) {
return Response.json({ error: "forbidden", message: "Owner required" }, {
status: 403,
});
}
const body = await request.json();
const prior = attempts.get(key);
if (prior && prior !== body.operation_id) {
return Response.json(
{ error: "conflict", message: "Changed operation" },
{ status: 409 },
);
}
attempts.set(key, body.operation_id);
hidden.add(key);
if (!prior && id === "unknown-subject") {
return Response.json({
error: "internal",
message: "Response lost after fence",
}, { status: 500 });
}
return Response.json({
subject_id: id,
operation_id: body.operation_id,
completed: !!prior,
});
}
if (id && request.method === "GET" && hidden.has(key)) {
return Response.json(
{ error: "not_found", message: "Subject unavailable" },
{ status: 404 },
);
}
const mapped = id &&
["pending-subject", "unknown-subject", "legacy-subject"].includes(id);
const target = new URL(request.url);
target.port = String(upstreamPort);
if (mapped) {
target.pathname = target.pathname.replace(
encodeURIComponent(id!),
"empty-subject",
);
}
let response = await fetch(new Request(target, request));
if (url.pathname === `${apiRoot}/workspace` || mapped) {
const data = await response.json();
if (url.pathname === `${apiRoot}/workspace` && mode !== "before") {
data.permissions.delete_subjects = !reader;
}
if (mapped) {
if ("id" in data) {
data.id = id;
data.role = `Deletion review · ${id}`;
if (id === "legacy-subject") data.state = "retired";
}
if ("subject_id" in data) data.subject_id = id;
}
response = Response.json(data, { status: response.status });
}
response = new Response(response.body, {
status: response.status,
headers: response.headers,
});
const persona = url.searchParams.get("persona");
if (persona) {
response.headers.set(
"set-cookie",
`subject-delete-persona=${persona}; Path=/; HttpOnly; SameSite=Lax`,
);
}
response.headers.append(
"set-cookie",
`delete-review-session=${sessionId}; Path=/; HttpOnly; SameSite=Lax`,
);
return response;
});
@@ -0,0 +1,257 @@
import { assert, assertEquals } from "@std/assert";
import { chromium, type Page } from "playwright";
import { join, resolve } from "@std/path";
import {
assertNoPageWideOverflow,
freePort,
waitForServer,
} from "../../../../tools/web-ux/browser-tests/memory_document_harness.ts";
const root = resolve(import.meta.dirname!, "../../../..");
const artifact = join(root, "target/web-ux/t786-browser");
async function confirm(page: Page) {
await page.locator(".subject-delete-open").click();
const button = page.getByRole("button", {
name: "Permanently delete Subject",
exact: true,
});
assert(await button.isDisabled());
await page.getByRole("checkbox").check();
await button.click();
}
Deno.test({
name:
"Subject deletion shell: owner confirmation, pending/reload/retry, unknown/rejected and route fencing",
sanitizeOps: false,
sanitizeResources: false,
fn: async () => {
const build = await new Deno.Command(Deno.execPath(), {
args: ["task", "build"],
cwd: join(root, "web/workspace"),
stdout: "null",
stderr: "piped",
}).output();
assert(build.success, new TextDecoder().decode(build.stderr));
const port = await freePort();
const server = new Deno.Command(Deno.execPath(), {
args: [
"run",
"-A",
join(import.meta.dirname!, "fixture-server.ts"),
String(port),
join(root, "web/workspace/build"),
],
stdout: "null",
stderr: "inherit",
}).spawn();
const origin = `http://127.0.0.1:${port}`;
await Deno.mkdir(artifact, { recursive: true });
const browser = await chromium.launch();
try {
await waitForServer(`${origin}/health`);
for (const theme of ["light", "dark"] as const) {
for (const width of [1440, 768, 390]) {
const context = await browser.newContext({
viewport: { width, height: 900 },
colorScheme: theme,
});
const page = await context.newPage();
const pageErrors: string[] = [];
page.on("pageerror", (error) => pageErrors.push(error.message));
const requests: unknown[] = [];
let release!: () => void;
const held = new Promise<void>((resolve) => {
release = resolve;
});
await page.route(
"**/subjektiv/subjects/pending-subject",
async (route) => {
if (route.request().method() !== "DELETE") {
return route.continue();
}
requests.push(route.request().postDataJSON());
if (requests.length === 1) await held;
await route.continue();
},
);
await page.goto(`${origin}/w/memory-review/memory/pending-subject`);
await page.locator(".subject-delete-open").waitFor();
await page.screenshot({
path: join(artifact, `${theme}-${width}-initial.png`),
});
await page.locator(".subject-delete-open").click();
assert(
await page.getByText("This is irreversible.", { exact: true })
.isVisible(),
);
await assertNoPageWideOverflow(page);
await page.screenshot({
path: join(artifact, `${theme}-${width}-confirmation.png`),
});
await page.getByRole("button", { name: "Cancel", exact: true })
.click();
assertEquals(requests.length, 0);
await confirm(page);
await page.locator('[data-delete-state="running"]').waitFor();
assertEquals(await page.locator(".subject-overview").count(), 0);
await page.screenshot({
path: join(artifact, `${theme}-${width}-running.png`),
});
release();
await page.locator('[data-delete-state="pending"]').waitFor();
assertEquals(requests.length, 1);
const saved = await page.evaluate(() =>
Object.keys(sessionStorage).filter((key) =>
key.startsWith("yoi.subject-delete:")
).map((key) => JSON.parse(sessionStorage.getItem(key)!))
);
assertEquals(saved, [{
subjectId: "pending-subject",
operationId: (requests[0] as { operation_id: string }).operation_id,
}]);
await page.screenshot({
path: join(artifact, `${theme}-${width}-pending.png`),
});
await page.reload();
await page.locator('[data-delete-state="unknown"]').waitFor();
assertEquals(await page.locator(".subject-overview").count(), 0);
await page.locator(".subject-delete-retry").click();
await page.locator('[data-delete-state="completed"]').waitFor();
assertEquals(requests.length, 2);
assertEquals(requests[0], requests[1]);
assertEquals(
await page.evaluate(() =>
Object.keys(sessionStorage).filter((key) =>
key.startsWith("yoi.subject-delete:")
)
),
[],
);
await page.screenshot({
path: join(artifact, `${theme}-${width}-completed.png`),
});
await assertNoPageWideOverflow(page);
assertEquals(pageErrors, []);
await context.close();
}
}
for (const subject of ["unknown-subject", "legacy-subject"]) {
const page = await browser.newPage();
const requests: unknown[] = [];
page.on("request", (request) => {
if (request.method() === "DELETE") {
requests.push(request.postDataJSON());
}
});
await page.goto(`${origin}/w/memory-review/memory/${subject}`);
await page.locator(".subject-delete-open").waitFor();
if (subject === "legacy-subject") {
assert(
await page.getByText("Legacy ended Subject", { exact: true })
.isVisible(),
);
}
await confirm(page);
await page.locator(
`[data-delete-state="${
subject === "unknown-subject" ? "unknown" : "pending"
}"]`,
).waitFor();
await page.locator(".subject-delete-retry").click();
await page.locator('[data-delete-state="completed"]').waitFor();
assertEquals(requests.length, 2);
assertEquals(requests[0], requests[1]);
await page.close();
}
// 4xxでもkeyを捨てない。権限の変化を新しい依頼へ置換しない。
{
const page = await browser.newPage();
const requests: unknown[] = [];
await page.route(
"**/subjektiv/subjects/pending-subject",
async (route) => {
if (route.request().method() !== "DELETE") return route.continue();
requests.push(route.request().postDataJSON());
if (requests.length === 1) {
await route.fulfill({
status: 403,
json: { error: "forbidden", message: "Permission changed" },
});
} else {await route.fulfill({
json: {
subject_id: "pending-subject",
operation_id: (requests[0] as any).operation_id,
completed: true,
},
});}
},
);
await page.goto(`${origin}/w/memory-review/memory/pending-subject`);
await confirm(page);
await page.locator('[data-delete-state="failed"]').waitFor();
await page.screenshot({ path: join(artifact, "rejected.png") });
await page.locator(".subject-delete-retry").click();
await page.locator('[data-delete-state="completed"]').waitFor();
assertEquals(requests[0], requests[1]);
await page.close();
}
// 同一SPA内で他対象へ移動してから旧応答が届いても、新対象を消さない。
{
const page = await browser.newPage();
let release!: () => void;
const held = new Promise<void>((resolve) => {
release = resolve;
});
await page.route(
"**/subjektiv/subjects/pending-subject",
async (route) => {
if (route.request().method() !== "DELETE") return route.continue();
const body = route.request().postDataJSON();
await held;
await route.fulfill({
json: {
subject_id: "pending-subject",
operation_id: body.operation_id,
completed: true,
},
});
},
);
await page.goto(`${origin}/w/memory-review/memory/pending-subject`);
await confirm(page);
await page.locator('[data-delete-state="running"]').waitFor();
await page.getByRole("link", { name: "← All subjects", exact: true })
.click();
await page.locator(".subject-list a").first().click();
await page.locator(".subject-overview").waitFor();
const target = page.url();
const response = page.waitForResponse((r) =>
r.request().method() === "DELETE"
);
release();
await response;
assertEquals(page.url(), target);
assertEquals(
await page.locator('[data-delete-state="completed"]').count(),
0,
);
assertEquals(await page.locator(".subject-overview").count(), 1);
await page.close();
}
{
const page = await browser.newPage();
await page.goto(
`${origin}/w/memory-review/memory/legacy-subject?persona=reader`,
);
await page.locator(".subject-overview").waitFor();
assertEquals(await page.locator(".subject-deletion").count(), 0);
await page.screenshot({ path: join(artifact, "non-owner.png") });
await page.close();
}
} finally {
await browser.close();
server.kill("SIGTERM");
await server.status;
}
},
});
+152
View File
@@ -0,0 +1,152 @@
import {
deleteSubjektivSubject,
parseSubjektivSubjectDeleteResponse,
SubjektivSubjectDeleteError,
} from "../src/lib/workspace/memory/api.ts";
import {
clearSubjectDeleteAttempt,
readSubjectDeleteAttempt,
saveSubjectDeleteAttempt,
subjectDeleteStorageKey,
} from "../src/lib/workspace/memory/subject-delete.ts";
declare const Deno: {
test(name: string, fn: () => void | Promise<void>): void;
};
function equal(actual: unknown, expected: unknown) {
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
throw new Error(
`Expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,
);
}
}
Deno.test("Subject deletion uses DELETE with the same request after fence-only success", async () => {
const calls: unknown[] = [];
const fetcher = ((input, init) => {
calls.push([input, init]);
return Promise.resolve(
Response.json({
subject_id: "s/1",
operation_id: "same-operation",
completed: calls.length > 1,
}),
);
}) as typeof fetch;
for (const completed of [false, true]) {
equal(
await deleteSubjektivSubject(fetcher, "w/1", "s/1", {
operation_id: "same-operation",
}),
{ subject_id: "s/1", operation_id: "same-operation", completed },
);
}
equal(calls[0], calls[1]);
equal(calls[0], ["/api/w/w%2F1/subjektiv/subjects/s%2F1", {
method: "DELETE",
headers: { "content-type": "application/json" },
body: '{"operation_id":"same-operation"}',
}]);
});
Deno.test("Subject deletion never confirms a malformed or wrong-target response", () => {
for (
const payload of [
{ subject_id: "other", operation_id: "op", completed: true },
{ subject_id: "s", operation_id: "other", completed: true },
{ subject_id: "s", operation_id: "op", completed: "true" },
{ subject_id: "s", operation_id: "op" },
{ subject_id: "s", operation_id: "op", completed: true, extra: 1 },
]
) {
try {
parseSubjektivSubjectDeleteResponse(payload, "s", "op");
} catch {
continue;
}
throw new Error("Accepted an unconfirmed response");
}
});
Deno.test("Subject deletion classifies rejection separately from unknown transport/server/response outcomes", async () => {
for (
const item of [
{
fetcher: () => Promise.reject(new Error("lost")),
kind: "unknown_outcome",
},
{
fetcher: () =>
Promise.resolve(
Response.json({ error: "forbidden" }, { status: 403 }),
),
kind: "rejected",
},
{
fetcher: () =>
Promise.resolve(
Response.json({ error: "conflict" }, { status: 409 }),
),
kind: "rejected",
},
{
fetcher: () =>
Promise.resolve(
Response.json({ error: "uncertain" }, { status: 500 }),
),
kind: "unknown_outcome",
},
{
fetcher: () =>
Promise.resolve(
Response.json({
subject_id: "wrong",
operation_id: "op",
completed: true,
}),
),
kind: "unknown_outcome",
},
]
) {
try {
await deleteSubjektivSubject(item.fetcher as typeof fetch, "w", "s", {
operation_id: "op",
});
} catch (error) {
if (!(error instanceof SubjektivSubjectDeleteError)) throw error;
equal(error.kind, item.kind);
continue;
}
throw new Error("Unexpected success");
}
});
Deno.test("Retry input survives reload independently of Subject GET and is isolated by Workspace/Subject", () => {
const values = new Map<string, string>();
const storage = {
getItem: (key: string) => values.get(key) ?? null,
setItem: (key: string, value: string) => {
values.set(key, value);
},
removeItem: (key: string) => {
values.delete(key);
},
};
const attempt = {
subjectId: "s",
operationId: "same-op",
};
saveSubjectDeleteAttempt(storage, "w", attempt);
equal(readSubjectDeleteAttempt(storage, "w", "s"), attempt);
equal(readSubjectDeleteAttempt(storage, "other", "s"), null);
equal(readSubjectDeleteAttempt(storage, "w", "other"), null);
saveSubjectDeleteAttempt(storage, "other", attempt);
clearSubjectDeleteAttempt(storage, "w", "s");
equal(readSubjectDeleteAttempt(storage, "w", "s"), null);
equal(readSubjectDeleteAttempt(storage, "other", "s"), attempt);
values.set(subjectDeleteStorageKey("w", "s"), '{"subjectId":"other"}');
try {
readSubjectDeleteAttempt(storage, "w", "s");
} catch {
return;
}
throw new Error("Corrupt retry input allowed new deletion");
});